From a954887fd5de2d0feb26dfac82b5e85dcd7ca259 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 20 Jul 2017 17:05:45 +0300 Subject: [PATCH 001/103] latin1 tests with TEXT columns --- localtests/latin1text/create.sql | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 localtests/latin1text/create.sql diff --git a/localtests/latin1text/create.sql b/localtests/latin1text/create.sql new file mode 100644 index 0000000..58837cb --- /dev/null +++ b/localtests/latin1text/create.sql @@ -0,0 +1,25 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + t text charset latin1 collate latin1_swedish_ci, + primary key(id) +) auto_increment=1 charset latin1 collate latin1_swedish_ci; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, md5(rand())); + insert into gh_ost_test values (null, 'átesting'); + insert into gh_ost_test values (null, 'ádelete'); + insert into gh_ost_test values (null, 'testátest'); + update gh_ost_test set t='áupdated' order by id desc limit 1; + update gh_ost_test set t='áupdated1' where t='áupdated' order by id desc limit 1; + delete from gh_ost_test where t='ádelete'; +end ;; From 0fd4603ae6ef33f3ffe39cc21bda31bcf6f3dfd6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 1 Nov 2017 09:51:45 +0200 Subject: [PATCH 002/103] minor updates to test --- localtests/datetime-submillis/create.sql | 2 +- localtests/test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/localtests/datetime-submillis/create.sql b/localtests/datetime-submillis/create.sql index b4e0b0b..6c04adb 100644 --- a/localtests/datetime-submillis/create.sql +++ b/localtests/datetime-submillis/create.sql @@ -17,7 +17,7 @@ create event gh_ost_test starts current_timestamp ends current_timestamp + interval 60 second on completion not preserve - enable + disable on slave do begin insert into gh_ost_test values (null, 11, now(), now(), now(), 0); diff --git a/localtests/test.sh b/localtests/test.sh index b901ce3..10cf796 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -164,6 +164,7 @@ test_single() { diff $orig_content_output_file $ghost_content_output_file echo "diff $orig_content_output_file $ghost_content_output_file" + return 1 fi } From 48e12814ca404e62f39665a998da2e14313314b0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 13 Mar 2018 07:48:24 +0200 Subject: [PATCH 003/103] Documenting limitation: encrypted binlogs --- doc/requirements-and-limitations.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index e12f3f0..66d3adb 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -52,3 +52,5 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - If you have en `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520) - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles. + +- [Encrypted binary logs](https://www.percona.com/blog/2018/03/08/binlog-encryption-percona-server-mysql/) are not supported. From 21d455013e9d8ce3b4892e44e3750aaf4fb58d51 Mon Sep 17 00:00:00 2001 From: MOON_CLJ Date: Fri, 27 Apr 2018 14:58:07 +0800 Subject: [PATCH 004/103] minor changes and typo correction --- go/binlog/binlog_entry.go | 4 ++-- go/binlog/gomysql_reader.go | 2 +- go/logic/applier.go | 10 +++++++--- go/logic/inspect.go | 3 +++ go/logic/migrator.go | 2 +- go/mysql/utils.go | 4 ---- go/sql/builder.go | 3 +-- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index bb70bc5..5650acc 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -26,7 +26,7 @@ func NewBinlogEntry(logFile string, logPos uint64) *BinlogEntry { return binlogEntry } -// NewBinlogEntry creates an empty, ready to go BinlogEntry object +// NewBinlogEntryAt creates an empty, ready to go BinlogEntry object func NewBinlogEntryAt(coordinates mysql.BinlogCoordinates) *BinlogEntry { binlogEntry := &BinlogEntry{ Coordinates: coordinates, @@ -41,7 +41,7 @@ func (this *BinlogEntry) Duplicate() *BinlogEntry { return binlogEntry } -// Duplicate creates and returns a new binlog entry, with some of the attributes pre-assigned +// String() returns a string representation of this binlog entry func (this *BinlogEntry) String() string { return fmt.Sprintf("[BinlogEntry at %+v; dml:%+v]", this.Coordinates, this.DmlEvent) } diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index a479c99..b86bd72 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -111,7 +111,7 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven binlogEntry.DmlEvent.WhereColumnValues = sql.ToColumnValues(row) } } - // The channel will do the throttling. Whoever is reding from the channel + // The channel will do the throttling. Whoever is reading from the channel // decides whether action is taken synchronously (meaning we wait before // next iteration) or asynchronously (we keep pushing more events) // In reality, reads will be synchronous diff --git a/go/logic/applier.go b/go/logic/applier.go index afdc45f..0de5da4 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -126,7 +126,6 @@ func (this *Applier) readTableColumns() (err error) { // showTableStatus returns the output of `show table status like '...'` command func (this *Applier) showTableStatus(tableName string) (rowMap sqlutils.RowMap) { - rowMap = nil query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(this.migrationContext.DatabaseName), tableName) sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error { rowMap = m @@ -482,6 +481,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected if err != nil { return nil, err } + defer tx.Rollback() sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') @@ -1001,15 +1001,19 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { if err != nil { return err } + rollback := func(err error) error { + tx.Rollback() + return err + } sessionQuery := `SET SESSION time_zone = '+00:00', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') ` if _, err := tx.Exec(sessionQuery); err != nil { - return err + return rollback(err) } if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil { - return err + return rollback(err) } if err := tx.Commit(); err != nil { return err diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 2568afe..4d25ab1 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -699,14 +699,17 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum for _, ghostColumn := range ghostColumns.Names() { if strings.EqualFold(originalColumn, ghostColumn) { isSharedColumn = true + break } if strings.EqualFold(columnRenameMap[originalColumn], ghostColumn) { isSharedColumn = true + break } } for droppedColumn := range this.migrationContext.DroppedColumnsMap { if strings.EqualFold(originalColumn, droppedColumn) { isSharedColumn = false + break } } if isSharedColumn { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f52e2b8..f536de8 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -78,7 +78,7 @@ type Migrator struct { rowCopyCompleteFlag int64 // copyRowsQueue should not be buffered; if buffered some non-damaging but - // excessive work happens at the end of the iteration as new copy-jobs arrive befroe realizing the copy is complete + // excessive work happens at the end of the iteration as new copy-jobs arrive before realizing the copy is complete copyRowsQueue chan tableWriteFunc applyEventsQueue chan *applyEventStruct diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4..2ca8cc2 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -83,9 +83,6 @@ func GetMasterKeyFromSlaveStatus(connectionConfig *ConnectionConfig) (masterKey } defer db.Close() - if err != nil { - return nil, err - } err = sqlutils.QueryRowsMap(db, `show slave status`, func(rowMap sqlutils.RowMap) error { // We wish to recognize the case where the topology's master actually has replication configuration. // This can happen when a DBA issues a `RESET SLAVE` instead of `RESET SLAVE ALL`. @@ -98,7 +95,6 @@ func GetMasterKeyFromSlaveStatus(connectionConfig *ConnectionConfig) (masterKey slaveIORunning := rowMap.GetString("Slave_IO_Running") slaveSQLRunning := rowMap.GetString("Slave_SQL_Running") - // if slaveIORunning != "Yes" || slaveSQLRunning != "Yes" { return fmt.Errorf("Replication on %+v is broken: Slave_IO_Running: %s, Slave_SQL_Running: %s. Please make sure replication runs before using gh-ost.", connectionConfig.Key, diff --git a/go/sql/builder.go b/go/sql/builder.go index c3a6229..2c5a7ae 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -140,13 +140,12 @@ func BuildRangeComparison(columns []string, values []string, args []interface{}, comparisons := []string{} for i, column := range columns { - // value := values[i] rangeComparison, err := BuildValueComparison(column, value, comparisonSign) if err != nil { return "", explodedArgs, err } - if len(columns[0:i]) > 0 { + if i > 0 { equalitiesComparison, err := BuildEqualsComparison(columns[0:i], values[0:i]) if err != nil { return "", explodedArgs, err From 14eda7efe06f5eed297c0f243bd870aa6b6de8b4 Mon Sep 17 00:00:00 2001 From: MOON_CLJ Date: Sat, 28 Apr 2018 12:20:28 +0800 Subject: [PATCH 005/103] fix GetReplicationLag not used args --- go/logic/inspect.go | 3 +-- go/logic/throttler.go | 4 ++-- go/mysql/utils.go | 5 ++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 4d25ab1..db6254d 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -758,9 +758,8 @@ func (this *Inspector) getMasterConnectionConfig() (applierConfig *mysql.Connect } func (this *Inspector) getReplicationLag() (replicationLag time.Duration, err error) { - replicationLag, err = mysql.GetReplicationLag( + replicationLag, err = mysql.GetReplicationLagFromSlaveStatus( this.informationSchemaDb, - this.migrationContext.InspectorConnectionConfig, ) return replicationLag, err } diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 624956a..6f9f4bb 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -140,8 +140,8 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica { // when running on replica, the heartbeat injection is also done on the replica. // This means we will always get a good heartbeat value. - // When runnign on replica, we should instead check the `SHOW SLAVE STATUS` output. - if lag, err := mysql.GetReplicationLag(this.inspector.informationSchemaDb, this.inspector.connectionConfig); err != nil { + // When running on replica, we should instead check the `SHOW SLAVE STATUS` output. + if lag, err := mysql.GetReplicationLagFromSlaveStatus(this.inspector.informationSchemaDb); err != nil { return log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 2ca8cc2..c96aebd 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -57,9 +57,8 @@ func GetDB(migrationUuid string, mysql_uri string) (*gosql.DB, bool, error) { return knownDBs[cacheKey], exists, nil } -// GetReplicationLag returns replication lag for a given connection config; either by explicit query -// or via SHOW SLAVE STATUS -func GetReplicationLag(informationSchemaDb *gosql.DB, connectionConfig *ConnectionConfig) (replicationLag time.Duration, err error) { +// GetReplicationLagFromSlaveStatus returns replication lag for a given db; via SHOW SLAVE STATUS +func GetReplicationLagFromSlaveStatus(informationSchemaDb *gosql.DB) (replicationLag time.Duration, err error) { err = sqlutils.QueryRowsMap(informationSchemaDb, `show slave status`, func(m sqlutils.RowMap) error { slaveIORunning := m.GetString("Slave_IO_Running") slaveSQLRunning := m.GetString("Slave_SQL_Running") From 48e01ad39b950a04ca45202baed279890a396b23 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 29 Oct 2018 10:09:15 +0200 Subject: [PATCH 006/103] Adding binary/varbinary tests --- localtests/varbinary/create.sql | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 localtests/varbinary/create.sql diff --git a/localtests/varbinary/create.sql b/localtests/varbinary/create.sql new file mode 100644 index 0000000..9d85200 --- /dev/null +++ b/localtests/varbinary/create.sql @@ -0,0 +1,40 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id binary(16) NOT NULL, + info varchar(255) COLLATE utf8_unicode_ci NOT NULL, + data binary(8) NOT NULL, + primary key (id), + unique key info_uidx (info) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + replace into gh_ost_test (id, info, data) values (X'12ffffffffffffffffffffffffffff00', 'item 1a', X'12ffffffffffffff'); + replace into gh_ost_test (id, info, data) values (X'34ffffffffffffffffffffffffffffff', 'item 3a', X'34ffffffffffffff'); + replace into gh_ost_test (id, info, data) values (X'90ffffffffffffffffffffffffffffff', 'item 9a', X'90ffffffffffff00'); + + DELETE FROM gh_ost_test WHERE id = X'11ffffffffffffffffffffffffffff00'; + UPDATE gh_ost_test SET info = 'item 2++' WHERE id = X'22ffffffffffffffffffffffffffff00'; + UPDATE gh_ost_test SET info = 'item 3++', data = X'33ffffffffffff00' WHERE id = X'33ffffffffffffffffffffffffffffff'; + DELETE FROM gh_ost_test WHERE id = X'44ffffffffffffffffffffffffffffff'; + UPDATE gh_ost_test SET info = 'item 5++', data = X'55ffffffffffffee' WHERE id = X'55ffffffffffffffffffffffffffffff'; + INSERT INTO gh_ost_test (id, info, data) VALUES (X'66ffffffffffffffffffffffffffff00', 'item 6', X'66ffffffffffffff'); + INSERT INTO gh_ost_test (id, info, data) VALUES (X'77ffffffffffffffffffffffffffffff', 'item 7', X'77ffffffffffff00'); + INSERT INTO gh_ost_test (id, info, data) VALUES (X'88ffffffffffffffffffffffffffffff', 'item 8', X'88ffffffffffffff'); +end ;; + +INSERT INTO gh_ost_test (id, info, data) VALUES + (X'11ffffffffffffffffffffffffffff00', 'item 1', X'11ffffffffffffff'), -- id ends in 00 + (X'22ffffffffffffffffffffffffffff00', 'item 2', X'22ffffffffffffff'), -- id ends in 00 + (X'33ffffffffffffffffffffffffffffff', 'item 3', X'33ffffffffffffff'), + (X'44ffffffffffffffffffffffffffffff', 'item 4', X'44ffffffffffffff'), + (X'55ffffffffffffffffffffffffffffff', 'item 5', X'55ffffffffffffff'), + (X'99ffffffffffffffffffffffffffffff', 'item 9', X'99ffffffffffff00'); -- data ends in 00 From dd41d9a4b1b4b0511152aea81920c0eead8873a3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 3 Jan 2019 11:18:07 +0200 Subject: [PATCH 007/103] Testing nullable int --- localtests/bigint-change-nullable/create.sql | 21 ++++++++++++++++++++ localtests/bigint-change-nullable/extra_args | 1 + 2 files changed, 22 insertions(+) create mode 100644 localtests/bigint-change-nullable/create.sql create mode 100644 localtests/bigint-change-nullable/extra_args diff --git a/localtests/bigint-change-nullable/create.sql b/localtests/bigint-change-nullable/create.sql new file mode 100644 index 0000000..f4f0548 --- /dev/null +++ b/localtests/bigint-change-nullable/create.sql @@ -0,0 +1,21 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint auto_increment, + val bigint not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 18446744073709551615); + insert into gh_ost_test values (null, 18446744073709551614); + insert into gh_ost_test values (null, 18446744073709551613); +end ;; diff --git a/localtests/bigint-change-nullable/extra_args b/localtests/bigint-change-nullable/extra_args new file mode 100644 index 0000000..784d522 --- /dev/null +++ b/localtests/bigint-change-nullable/extra_args @@ -0,0 +1 @@ +--alter="change val val bigint" From 5f587c3262259c4d6ca5a17e955fc96f8f03e0d5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 14 Jan 2019 13:27:44 +0200 Subject: [PATCH 008/103] Adding --force-named-panic option --- go/base/context.go | 1 + go/cmd/gh-ost/main.go | 1 + go/logic/server.go | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/go/base/context.go b/go/base/context.go index 0043d07..318fdf9 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -123,6 +123,7 @@ type MigrationContext struct { CutOverExponentialBackoff bool ExponentialBackoffMaxInterval int64 ForceNamedCutOverCommand bool + ForceNamedPanicCommand bool PanicFlagFile string HooksPath string HooksHintMessage string diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 3024702..a745c21 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -81,6 +81,7 @@ func main() { flag.BoolVar(&migrationContext.TimestampOldTable, "timestamp-old-table", false, "Use a timestamp in old table name. This makes old table names unique and non conflicting cross migrations") cutOver := flag.String("cut-over", "atomic", "choose cut-over type (default|atomic, two-step)") flag.BoolVar(&migrationContext.ForceNamedCutOverCommand, "force-named-cut-over", false, "When true, the 'unpostpone|cut-over' interactive command must name the migrated table") + flag.BoolVar(&migrationContext.ForceNamedPanicCommand, "force-named-panic", false, "When true, the 'panic' interactive command must name the migrated table") flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running") flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") diff --git a/go/logic/server.go b/go/logic/server.go index 919f178..831fdfa 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -322,6 +322,15 @@ help # This message } case "panic": { + if arg == "" && this.migrationContext.ForceNamedPanicCommand { + err := fmt.Errorf("User commanded 'panic' without specifying table name, but --force-named-panic is set") + return NoPrintStatusRule, err + } + if arg != "" && arg != this.migrationContext.OriginalTableName { + // User explicitly provided table name. This is a courtesy protection mechanism + err := fmt.Errorf("User commanded 'panic' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) + return NoPrintStatusRule, err + } err := fmt.Errorf("User commanded 'panic'. I will now panic, without cleanup. PANIC!") this.migrationContext.PanicAbort <- err return NoPrintStatusRule, err From 6c5805d844935776a0ebdb4260b91f598f878c81 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 14 Jan 2019 13:32:43 +0200 Subject: [PATCH 009/103] documenting new flag --- doc/command-line-flags.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 804ba58..3d96ce4 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -111,6 +111,14 @@ While the ongoing estimated number of rows is still heuristic, it's almost exact Without this parameter, migration is a _noop_: testing table creation and validity of migration, but not touching data. +### force-named-cut-over + +If given, a `cut-over` command must name the migrated table, or else ignored. + +### force-named-panic + +If given, a `panic` command must name the migrated table, or else ignored. + ### force-table-names Table name prefix to be used on the temporary tables. From 5eec74132750f4119b1b761e7607aeee83928a67 Mon Sep 17 00:00:00 2001 From: Will Currie Date: Wed, 16 Jan 2019 06:28:05 +1100 Subject: [PATCH 010/103] Add Why Is "Connect to Replica" mode preferred? --- doc/questions.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/questions.md b/doc/questions.md index be6eab0..7585bc7 100644 --- a/doc/questions.md +++ b/doc/questions.md @@ -28,3 +28,9 @@ It is therefore unlikely that `gh-ost` will support this behavior. Yes. TL;DR if running all on same replica/master, make sure to provide `--replica-server-id`. [Read more](cheatsheet.md#concurrent-migrations) # Why + +### Why Is the "Connect to Replica" mode preferred? + +To avoid placing extra load on the master. `gh-ost` connects as a replication client. Each additional replica adds some load to the master. + +To monitor replication lag from a replica. This makes the replication lag throttle, `--max-lag-millis`, more representative of the lag experienced by other replicas following the master (perhaps N levels deep in a tree of replicas). From afa108f8fa49ec66a3b873c094f1bd09e4aaa169 Mon Sep 17 00:00:00 2001 From: twotwotwo Date: Tue, 22 Jan 2019 12:26:55 -0800 Subject: [PATCH 011/103] inspect: remove redundant join conditions The TABLE_SCHEMA and TABLE_NAME are already guaranteed to have the same value in COLUMNS and UNIQUES because of the WHEREs in each query. Dropping it from the ON clause makes it complete much faster. On (at least) MySQL 5.6 installs with thousands of tables, this query completes in a few seconds without the additional join conditions, but takes more than a minute with it. --- go/logic/inspect.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 5f875a0..5643c58 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -622,8 +622,6 @@ func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](* GROUP BY TABLE_SCHEMA, TABLE_NAME, INDEX_NAME ) AS UNIQUES ON ( - COLUMNS.TABLE_SCHEMA = UNIQUES.TABLE_SCHEMA AND - COLUMNS.TABLE_NAME = UNIQUES.TABLE_NAME AND COLUMNS.COLUMN_NAME = UNIQUES.FIRST_COLUMN_NAME ) WHERE From 3161cd5823ccb3e213f8783e972c082f72ad6571 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 24 Jan 2019 13:06:20 +0200 Subject: [PATCH 012/103] added: --hooks-hint-owner, --hooks-hint-token --- doc/hooks.md | 2 ++ go/base/context.go | 2 ++ go/cmd/gh-ost/main.go | 2 ++ go/logic/hooks.go | 2 ++ 4 files changed, 8 insertions(+) diff --git a/doc/hooks.md b/doc/hooks.md index 1b1958f..93f500f 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -69,6 +69,8 @@ The following variables are available on all hooks: - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` - `GH_OST_HOOKS_HINT` - copy of `--hooks-hint` value +- `GH_OST_HOOKS_HINT_OWNER` - copy of `--hooks-hint-owner` value +- `GH_OST_HOOKS_HINT_TOKEN` - copy of `--hooks-hint-token` value - `GH_OST_DRY_RUN` - whether or not the `gh-ost` run is a dry run The following variable are available on particular hooks: diff --git a/go/base/context.go b/go/base/context.go index 0043d07..22a1fd0 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -126,6 +126,8 @@ type MigrationContext struct { PanicFlagFile string HooksPath string HooksHintMessage string + HooksHintOwner string + HooksHintToken string DropServeSocket bool ServeSocketFile string diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 3024702..9a4407b 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -109,6 +109,8 @@ func main() { flag.StringVar(&migrationContext.HooksPath, "hooks-path", "", "directory where hook files are found (default: empty, ie. hooks disabled). Hook files found on this path, and conforming to hook naming conventions will be executed") flag.StringVar(&migrationContext.HooksHintMessage, "hooks-hint", "", "arbitrary message to be injected to hooks via GH_OST_HOOKS_HINT, for your convenience") + flag.StringVar(&migrationContext.HooksHintOwner, "hooks-hint-owner", "", "arbitrary name of owner to be injected to hooks via GH_OST_HOOKS_HINT_OWNER, for your convenience") + flag.StringVar(&migrationContext.HooksHintToken, "hooks-hint-token", "", "arbitrary token to be injected to hooks via GH_OST_HOOKS_HINT_TOKEN, for your convenience") flag.UintVar(&migrationContext.ReplicaServerId, "replica-server-id", 99999, "server id used by gh-ost process. Default: 99999") diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 6515d7f..be11130 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -64,6 +64,8 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) + env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) + env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) env = append(env, fmt.Sprintf("GH_OST_DRY_RUN=%t", this.migrationContext.Noop)) for _, variable := range extraVariables { From 23617f287fbf0903e8979ec4ba5086b032e0bc23 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 10:03:48 -0600 Subject: [PATCH 013/103] Add initial support for ssl encryption connections to database servers. - Adding a command line option for users to enforce tls/ssl connections for the applier, inspector, and binlog reader. - The user can optionally request server certificate verification through a command line option to specify the ca cert via a file path. - Fixes an existing bug appending the timeout option to the singleton applier connection. --- go/base/context.go | 9 +++++++ go/binlog/gomysql_reader.go | 1 + go/cmd/gh-ost/main.go | 8 ++++++ go/logic/applier.go | 2 +- go/mysql/connection.go | 51 +++++++++++++++++++++++++++++++++---- go/mysql/connection_test.go | 19 +++++++++++++- 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 0043d07..856a29c 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -99,6 +99,8 @@ type MigrationContext struct { ConfigFile string CliUser string CliPassword string + UseTLS bool + TlsCACertificate string CliMasterUser string CliMasterPassword string @@ -695,6 +697,13 @@ func (this *MigrationContext) ApplyCredentials() { } } +func (this *MigrationContext) SetupTLS() error { + if this.UseTLS { + return this.InspectorConnectionConfig.UseTLS(this.TlsCACertificate) + } + return nil +} + // ReadConfigFile attempts to read the config file, if it exists func (this *MigrationContext) ReadConfigFile() error { this.configMutex.Lock() diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 794d545..41d1ead 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -46,6 +46,7 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go Port: uint16(binlogReader.connectionConfig.Key.Port), User: binlogReader.connectionConfig.User, Password: binlogReader.connectionConfig.Password, + TLSConfig: binlogReader.connectionConfig.TLSConfig(), UseDecimal: true, } binlogReader.binlogSyncer = replication.NewBinlogSyncer(binlogSyncerConfig) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 3024702..8bd8c78 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -14,6 +14,7 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" + _ "github.com/go-sql-driver/mysql" "github.com/outbrain/golib/log" "golang.org/x/crypto/ssh/terminal" @@ -54,6 +55,9 @@ func main() { flag.StringVar(&migrationContext.ConfigFile, "conf", "", "Config file") askPass := flag.Bool("ask-pass", false, "prompt for MySQL password") + flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL") + flag.StringVar(&migrationContext.TlsCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") + flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)") @@ -194,6 +198,9 @@ func main() { if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" { log.Fatalf("--master-password requires --assume-master-host") } + if migrationContext.TlsCACertificate != "" && !migrationContext.UseTLS { + log.Fatalf("--ssl-ca requires --ssl") + } if *replicationLagQuery != "" { log.Warningf("--replication-lag-query is deprecated") } @@ -238,6 +245,7 @@ func main() { migrationContext.SetThrottleHTTP(*throttleHTTP) migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() + migrationContext.SetupTLS() if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { log.Errore(err) } diff --git a/go/logic/applier.go b/go/logic/applier.go index 140b1a4..b3de0e1 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -73,7 +73,7 @@ func (this *Applier) InitDBConnections() (err error) { if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, applierUri); err != nil { return err } - singletonApplierUri := fmt.Sprintf("%s?timeout=0", applierUri) + singletonApplierUri := fmt.Sprintf("%s&timeout=0", applierUri) if this.singletonDB, _, err = mysql.GetDB(this.migrationContext.Uuid, singletonApplierUri); err != nil { return err } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index c9c75f2..bfb445a 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -6,8 +6,14 @@ package mysql import ( + "crypto/tls" + "crypto/x509" + "errors" "fmt" + "io/ioutil" "net" + + "github.com/go-sql-driver/mysql" ) // ConnectionConfig is the minimal configuration required to connect to a MySQL server @@ -16,6 +22,7 @@ type ConnectionConfig struct { User string Password string ImpliedKey *InstanceKey + tlsConfig *tls.Config } func NewConnectionConfig() *ConnectionConfig { @@ -29,9 +36,10 @@ func NewConnectionConfig() *ConnectionConfig { // DuplicateCredentials creates a new connection config with given key and with same credentials as this config func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig { config := &ConnectionConfig{ - Key: key, - User: this.User, - Password: this.Password, + Key: key, + User: this.User, + Password: this.Password, + tlsConfig: this.tlsConfig, } config.ImpliedKey = &config.Key return config @@ -42,13 +50,42 @@ func (this *ConnectionConfig) Duplicate() *ConnectionConfig { } func (this *ConnectionConfig) String() string { - return fmt.Sprintf("%s, user=%s", this.Key.DisplayString(), this.User) + return fmt.Sprintf("%s, user=%s, usingTLS=%t", this.Key.DisplayString(), this.User, this.tlsConfig != nil) } func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } +func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { + skipVerify := caCertificatePath == "" + var rootCertPool *x509.CertPool + if !skipVerify { + rootCertPool = x509.NewCertPool() + pem, err := ioutil.ReadFile(caCertificatePath) + if err != nil { + return err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return errors.New("could not add ca certificate to cert pool") + } + } + + this.tlsConfig = &tls.Config{ + RootCAs: rootCertPool, + InsecureSkipVerify: skipVerify, + } + + if err := mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig); err != nil { + return err + } + return nil +} + +func (this *ConnectionConfig) TLSConfig() *tls.Config { + return this.tlsConfig +} + func (this *ConnectionConfig) GetDBUri(databaseName string) string { hostname := this.Key.Hostname var ip = net.ParseIP(hostname) @@ -57,5 +94,9 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { hostname = fmt.Sprintf("[%s]", hostname) } interpolateParams := true - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams) + tlsOption := "false" + if this.tlsConfig != nil { + tlsOption = this.Key.StringCode() + } + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 19badf3..3a6cf29 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -6,6 +6,7 @@ package mysql import ( + "crypto/tls" "testing" "github.com/outbrain/golib/log" @@ -31,6 +32,10 @@ func TestDuplicateCredentials(t *testing.T) { c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" + c.tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + ServerName: "feathers", + } dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310}) test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost") @@ -39,6 +44,7 @@ func TestDuplicateCredentials(t *testing.T) { test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3310) test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") + test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig) } func TestDuplicate(t *testing.T) { @@ -63,5 +69,16 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") +} + +func TestGetDBUriWithTLSSetup(t *testing.T) { + c := NewConnectionConfig() + c.Key = InstanceKey{Hostname: "myhost", Port: 3306} + c.User = "gromit" + c.Password = "penguin" + c.tlsConfig = &tls.Config{} + + uri := c.GetDBUri("test") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=myhost:3306") } From 4f9367e6909dc93d824cda9f536921e449cf689e Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 16:59:42 -0600 Subject: [PATCH 014/103] Fix casing for initialism --- go/base/context.go | 4 ++-- go/cmd/gh-ost/main.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 856a29c..90581e2 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -100,7 +100,7 @@ type MigrationContext struct { CliUser string CliPassword string UseTLS bool - TlsCACertificate string + TLSCACertificate string CliMasterUser string CliMasterPassword string @@ -699,7 +699,7 @@ func (this *MigrationContext) ApplyCredentials() { func (this *MigrationContext) SetupTLS() error { if this.UseTLS { - return this.InspectorConnectionConfig.UseTLS(this.TlsCACertificate) + return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate) } return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 8bd8c78..db044f3 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -56,7 +56,7 @@ func main() { askPass := flag.Bool("ask-pass", false, "prompt for MySQL password") flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL") - flag.StringVar(&migrationContext.TlsCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") + flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") @@ -198,7 +198,7 @@ func main() { if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" { log.Fatalf("--master-password requires --assume-master-host") } - if migrationContext.TlsCACertificate != "" && !migrationContext.UseTLS { + if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } if *replicationLagQuery != "" { From dc599bb0368f0ccd15a605348a6615cfe8f03faf Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 17:20:11 -0600 Subject: [PATCH 015/103] Remove unnecessary branching --- go/mysql/connection.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/mysql/connection.go b/go/mysql/connection.go index bfb445a..7fa8268 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -76,10 +76,7 @@ func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { InsecureSkipVerify: skipVerify, } - if err := mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig); err != nil { - return err - } - return nil + return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) } func (this *ConnectionConfig) TLSConfig() *tls.Config { From c440112d440112e819d17f8d6997f3b41882edac Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 17:23:19 -0600 Subject: [PATCH 016/103] Explain default setting for TLS param in DSN --- go/mysql/connection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 7fa8268..742ca7e 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -91,6 +91,8 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { hostname = fmt.Sprintf("[%s]", hostname) } interpolateParams := true + // go-mysql-driver defaults to false if tls param is not provided; explicitly setting here to + // simplify construction of the DSN below. tlsOption := "false" if this.tlsConfig != nil { tlsOption = this.Key.StringCode() From f67ec15f2b972fd6b6b5b1892fe9838a3f0c9e34 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Fri, 1 Feb 2019 13:16:54 -0600 Subject: [PATCH 017/103] Handle returned error --- go/cmd/gh-ost/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index db044f3..9d7d60f 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -245,7 +245,9 @@ func main() { migrationContext.SetThrottleHTTP(*throttleHTTP) migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() - migrationContext.SetupTLS() + if err := migrationContext.SetupTLS(); err != nil { + log.Fatale(err) + } if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { log.Errore(err) } From 5319157789e317d296e298c0eb86240d3ef3cd56 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Fri, 1 Feb 2019 13:20:17 -0600 Subject: [PATCH 018/103] Expand usage statement to indicate setting applies to multiple hosts --- go/cmd/gh-ost/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 9d7d60f..d8add4f 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -55,8 +55,8 @@ func main() { flag.StringVar(&migrationContext.ConfigFile, "conf", "", "Config file") askPass := flag.Bool("ask-pass", false, "prompt for MySQL password") - flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL") - flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") + flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") + flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") From 79df0d1c5de9eec208da73f2657b6ff666af6239 Mon Sep 17 00:00:00 2001 From: Matt Belisle Date: Mon, 4 Feb 2019 14:46:08 -0600 Subject: [PATCH 019/103] Adding --ssl-insecure flag --- go/base/context.go | 19 ++++++++++--------- go/cmd/gh-ost/main.go | 4 ++++ go/mysql/connection.go | 28 ++++++++++++++++++---------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 90581e2..9f9de9b 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -94,15 +94,16 @@ type MigrationContext struct { AliyunRDS bool GoogleCloudPlatform bool - config ContextConfig - configMutex *sync.Mutex - ConfigFile string - CliUser string - CliPassword string - UseTLS bool - TLSCACertificate string - CliMasterUser string - CliMasterPassword string + config ContextConfig + configMutex *sync.Mutex + ConfigFile string + CliUser string + CliPassword string + UseTLS bool + TLSInsecureSkipVerify bool + TLSCACertificate string + CliMasterUser string + CliMasterPassword string HeartbeatIntervalMilliseconds int64 defaultNumRetries int64 diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index d8add4f..96bd2dc 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -57,6 +57,7 @@ func main() { flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") + flag.StringVar(&migrationContext.TLSInsecureSkipVerify, "ssl-insecure", false, "Do not verify that the TLS connection is secure. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") @@ -201,6 +202,9 @@ func main() { if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } + if migrationContext.TLSInsecureSkipVerify && !migrationContext.UseTLS { + log.Fatalf("--ssl-insecure requires --ssl") + } if *replicationLagQuery != "" { log.Warningf("--replication-lag-query is deprecated") } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 742ca7e..f0f73ce 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -58,22 +58,30 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { } func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { - skipVerify := caCertificatePath == "" var rootCertPool *x509.CertPool - if !skipVerify { - rootCertPool = x509.NewCertPool() - pem, err := ioutil.ReadFile(caCertificatePath) - if err != nil { - return err - } - if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { - return errors.New("could not add ca certificate to cert pool") + var err error + + if !this.TLSInsecureSkipVerify { + if caCertificatePath == "" { + rootCertPool, err = x509.SystemCertPool() + if err != nil { + return err + } + } else { + rootCertPool = x509.NewCertPool() + pem, err := ioutil.ReadFile(caCertificatePath) + if err != nil { + return err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return errors.New("could not add ca certificate to cert pool") + } } } this.tlsConfig = &tls.Config{ RootCAs: rootCertPool, - InsecureSkipVerify: skipVerify, + InsecureSkipVerify: this.TLSInsecureSkipVerify, } return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) From 5b0dfb009c5e0dfb815d125da2faec1f132a4c75 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Mon, 4 Feb 2019 16:21:25 -0600 Subject: [PATCH 020/103] Wireup allowing insecure ssl --- go/base/context.go | 22 +++++++++++----------- go/cmd/gh-ost/main.go | 6 +++--- go/mysql/connection.go | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 9f9de9b..e9991d8 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -94,16 +94,16 @@ type MigrationContext struct { AliyunRDS bool GoogleCloudPlatform bool - config ContextConfig - configMutex *sync.Mutex - ConfigFile string - CliUser string - CliPassword string - UseTLS bool - TLSInsecureSkipVerify bool - TLSCACertificate string - CliMasterUser string - CliMasterPassword string + config ContextConfig + configMutex *sync.Mutex + ConfigFile string + CliUser string + CliPassword string + UseTLS bool + TLSAllowInsecure bool + TLSCACertificate string + CliMasterUser string + CliMasterPassword string HeartbeatIntervalMilliseconds int64 defaultNumRetries int64 @@ -700,7 +700,7 @@ func (this *MigrationContext) ApplyCredentials() { func (this *MigrationContext) SetupTLS() error { if this.UseTLS { - return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate) + return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate, this.TLSAllowInsecure) } return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 96bd2dc..b02d6b7 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -57,7 +57,7 @@ func main() { flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") - flag.StringVar(&migrationContext.TLSInsecureSkipVerify, "ssl-insecure", false, "Do not verify that the TLS connection is secure. Requires --ssl") + flag.BoolVar(&migrationContext.TLSAllowInsecure, "ssl-allow-insecure", false, "Skips verification of MySQL hosts' certificate chain and host name. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") @@ -202,8 +202,8 @@ func main() { if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } - if migrationContext.TLSInsecureSkipVerify && !migrationContext.UseTLS { - log.Fatalf("--ssl-insecure requires --ssl") + if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS { + log.Fatalf("--ssl-allow-insecure requires --ssl") } if *replicationLagQuery != "" { log.Warningf("--replication-lag-query is deprecated") diff --git a/go/mysql/connection.go b/go/mysql/connection.go index f0f73ce..d6c7215 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -57,11 +57,11 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { +func (this *ConnectionConfig) UseTLS(caCertificatePath string, allowInsecure bool) error { var rootCertPool *x509.CertPool var err error - if !this.TLSInsecureSkipVerify { + if !allowInsecure { if caCertificatePath == "" { rootCertPool, err = x509.SystemCertPool() if err != nil { @@ -81,7 +81,7 @@ func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { this.tlsConfig = &tls.Config{ RootCAs: rootCertPool, - InsecureSkipVerify: this.TLSInsecureSkipVerify, + InsecureSkipVerify: allowInsecure, } return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) From 1543098891296e496e16f63b671d6d0ea80342e8 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Tue, 5 Feb 2019 08:02:47 -0600 Subject: [PATCH 021/103] Document ssl-related command line flags. --- doc/command-line-flags.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 804ba58..c20a5a8 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -177,6 +177,18 @@ By default `gh-ost` verifies no foreign keys exist on the migrated table. On ser See [`approve-renamed-columns`](#approve-renamed-columns) +### ssl + +By default `gh-ost` does not use ssl/tls connections to the database servers when performing migrations. This flag instructs `gh-ost` to use encrypted connections. If enabled, `gh-ost` will use the system's ca certificate pool for server certificate verification. If a different certificate is needed for server verification, see `--ssl-ca`. If you wish to skip server verification, but still use encrypted connections, use with `--ssl-allow-insecure`. + +### ssl-allow-insecure + +Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but without verifying the validity of the certificate provided by the server during the connection. Requires `--ssl`. + +### ssl-ca + +`--ssl-ca=/path/to/ca-cert.pem`: ca certificate file (in PEM format) to use for server certificate verification. If specified, the default system ca cert pool will not be used for verification, only the ca cert provided here. Requires `--ssl`. + ### test-on-replica Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) From b82fc452212e6757b1ab97fb1762d142424c1fd4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 12 Feb 2019 08:01:47 +0200 Subject: [PATCH 022/103] Adding maintainer email in DEB package --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 264620b..df9ac7d 100755 --- a/build.sh +++ b/build.sh @@ -42,8 +42,8 @@ function build { builddir=$(setuptree) cp $buildpath/$target $builddir/gh-ost/usr/bin cd $buildpath - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . fi } From a8fae9818daa42681c7806f843067e2d5d360273 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 14 Feb 2019 12:07:25 +0200 Subject: [PATCH 023/103] v1.0.48 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index bde91a2..56d0dad 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.47 +1.0.48 From 7d5749b84a30b1e41ec99bbb55795db1f0352425 Mon Sep 17 00:00:00 2001 From: babinomec Date: Thu, 14 Feb 2019 15:58:49 +0100 Subject: [PATCH 024/103] add --skip-strict-mode option --- doc/command-line-flags.md | 4 ++++ go/base/context.go | 1 + go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 24 ++++++++++++------------ 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index c20a5a8..02e7c64 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -173,6 +173,10 @@ See also: [`concurrent-migrations`](cheatsheet.md#concurrent-migrations) on the By default `gh-ost` verifies no foreign keys exist on the migrated table. On servers with large number of tables this check can take a long time. If you're absolutely certain no foreign keys exist (table does not reference other table nor is referenced by other tables) and wish to save the check time, provide with `--skip-foreign-key-checks`. +### skip-strict-mode + +By default `gh-ost` enforces STRICT_ALL_TABLES sql_mode as a safety measure. In some cases this changes the behaviour of other modes (namely ERROR_FOR_DIVISION_BY_ZERO, NO_ZERO_DATE, and NO_ZERO_IN_DATE) which may lead to errors during migration. Use `--skip-strict-mode` to explicitly tell `gh-ost` not to enforce this. **Danger** This may have some unexpected disastrous side effects. + ### skip-renamed-columns See [`approve-renamed-columns`](#approve-renamed-columns) diff --git a/go/base/context.go b/go/base/context.go index b2a6c15..09bc9c0 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -86,6 +86,7 @@ type MigrationContext struct { SwitchToRowBinlogFormat bool AssumeRBR bool SkipForeignKeyChecks bool + SkipStrictMode bool NullableUniqueKeyAllowed bool ApproveRenamedColumns bool SkipRenamedColumns bool diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a467f07..55e65d8 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -72,6 +72,7 @@ func main() { flag.BoolVar(&migrationContext.IsTungsten, "tungsten", false, "explicitly let gh-ost know that you are running on a tungsten-replication based topology (you are likely to also provide --assume-master-host)") flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys") flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that") + flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode") flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") diff --git a/go/logic/applier.go b/go/logic/applier.go index b3de0e1..1544359 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -482,10 +482,10 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected return nil, err } defer tx.Rollback() - sessionQuery := fmt.Sprintf(`SET - SESSION time_zone = '%s', - sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') - `, this.migrationContext.ApplierTimeZone) + sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone) + if !this.migrationContext.SkipStrictMode { + sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + } if _, err := tx.Exec(sessionQuery); err != nil { return nil, err } @@ -1005,10 +1005,10 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { tx.Rollback() return err } - sessionQuery := `SET - SESSION time_zone = '+00:00', - sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') - ` + sessionQuery := fmt.Sprintf("SET SESSION time_zone = '+00:00'") + if !this.migrationContext.SkipStrictMode { + sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + } if _, err := tx.Exec(sessionQuery); err != nil { return rollback(err) } @@ -1050,10 +1050,10 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) return err } - sessionQuery := `SET - SESSION time_zone = '+00:00', - sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') - ` + sessionQuery := "SET SESSION time_zone = '+00:00'" + if !this.migrationContext.SkipStrictMode { + sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + } if _, err := tx.Exec(sessionQuery); err != nil { return rollback(err) } From b4da7cfbfb0a08a94e13df82403789713da4ed71 Mon Sep 17 00:00:00 2001 From: Rafe Kettler Date: Fri, 22 Feb 2019 10:33:19 -0800 Subject: [PATCH 025/103] Improve SSL support Related issue: https://github.com/github/gh-ost/issues/521 - Add --ssl-cert and --ssl-key options to specify SSL public/private key files - Allow combining --ssl-allow-insecure with other --ssl* flags. `mysql.RegisterTLSConfig` allows combining the corresponding parameters in the `tls.Config` it receives, so gh-ost should allow this. I found being able to pass --ssl-allow-insecure along with --ssl-ca, --ssl-cert, and --ssl-key useful in testing. - Use the same TLS config everywhere. Since the CLI only supports a single set of --ssl* configuration parameters, this should be fine -- `mysql.RegisterTLSConfig` documentation indicates the TLS config given will not be modified, so it can safely be used in many goroutines provided we also do not modify it. The previous implementation did not work when the TLS config was duplicated, which happens when gh-ost walks up the replication chain trying to find the master. This is because, when the config is duplicated, we must call `RegisterTLSConfig` again with the new config. This config is exactly the same, so it's easiest to side-step the issue by registering the TLS config once and using it everywhere. --- doc/command-line-flags.md | 8 +++++++ go/base/context.go | 4 +++- go/cmd/gh-ost/main.go | 8 +++++++ go/mysql/connection.go | 47 +++++++++++++++++++++++-------------- go/mysql/connection_test.go | 2 +- 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index c20a5a8..2d9bfd1 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -189,6 +189,14 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but `--ssl-ca=/path/to/ca-cert.pem`: ca certificate file (in PEM format) to use for server certificate verification. If specified, the default system ca cert pool will not be used for verification, only the ca cert provided here. Requires `--ssl`. +### ssl-cert + +`--ssl-cert=/path/to/ssl-cert.crt`: SSL public key certificate file (in PEM format). + +### ssl-key + +`--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format). + ### test-on-replica Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) diff --git a/go/base/context.go b/go/base/context.go index b2a6c15..8a8f721 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -102,6 +102,8 @@ type MigrationContext struct { UseTLS bool TLSAllowInsecure bool TLSCACertificate string + TLSCertificate string + TLSKey string CliMasterUser string CliMasterPassword string @@ -702,7 +704,7 @@ func (this *MigrationContext) ApplyCredentials() { func (this *MigrationContext) SetupTLS() error { if this.UseTLS { - return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate, this.TLSAllowInsecure) + return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate, this.TLSCertificate, this.TLSKey, this.TLSAllowInsecure) } return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a467f07..5c30ffd 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -57,6 +57,8 @@ func main() { flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") + flag.StringVar(&migrationContext.TLSCertificate, "ssl-cert", "", "Certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") + flag.StringVar(&migrationContext.TLSKey, "ssl-key", "", "Key in PEM format for TLS connections to MySQL hosts. Requires --ssl") flag.BoolVar(&migrationContext.TLSAllowInsecure, "ssl-allow-insecure", false, "Skips verification of MySQL hosts' certificate chain and host name. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") @@ -204,6 +206,12 @@ func main() { if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } + if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS { + log.Fatalf("--ssl-cert requires --ssl") + } + if migrationContext.TLSKey != "" && !migrationContext.UseTLS { + log.Fatalf("--ssl-key requires --ssl") + } if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS { log.Fatalf("--ssl-allow-insecure requires --ssl") } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index d6c7215..654998c 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -16,6 +16,10 @@ import ( "github.com/go-sql-driver/mysql" ) +const ( + TLS_CONFIG_KEY = "ghost" +) + // ConnectionConfig is the minimal configuration required to connect to a MySQL server type ConnectionConfig struct { Key InstanceKey @@ -57,34 +61,41 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) UseTLS(caCertificatePath string, allowInsecure bool) error { +func (this *ConnectionConfig) UseTLS(caCertificatePath, clientCertificate, clientKey string, allowInsecure bool) error { var rootCertPool *x509.CertPool + var certs []tls.Certificate var err error - if !allowInsecure { - if caCertificatePath == "" { - rootCertPool, err = x509.SystemCertPool() - if err != nil { - return err - } - } else { - rootCertPool = x509.NewCertPool() - pem, err := ioutil.ReadFile(caCertificatePath) - if err != nil { - return err - } - if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { - return errors.New("could not add ca certificate to cert pool") - } + if caCertificatePath == "" { + rootCertPool, err = x509.SystemCertPool() + if err != nil { + return err } + } else { + rootCertPool = x509.NewCertPool() + pem, err := ioutil.ReadFile(caCertificatePath) + if err != nil { + return err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return errors.New("could not add ca certificate to cert pool") + } + } + if clientCertificate != "" || clientKey != "" { + cert, err := tls.LoadX509KeyPair(clientCertificate, clientKey) + if err != nil { + return err + } + certs = []tls.Certificate{cert} } this.tlsConfig = &tls.Config{ + Certificates: certs, RootCAs: rootCertPool, InsecureSkipVerify: allowInsecure, } - return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) + return mysql.RegisterTLSConfig(TLS_CONFIG_KEY, this.tlsConfig) } func (this *ConnectionConfig) TLSConfig() *tls.Config { @@ -103,7 +114,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { // simplify construction of the DSN below. tlsOption := "false" if this.tlsConfig != nil { - tlsOption = this.Key.StringCode() + tlsOption = TLS_CONFIG_KEY } return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 3a6cf29..6ce85a7 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=myhost:3306") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") } From 7ead4c4a56dda163fbf50fce6bb22ac6d96b690f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 25 Feb 2019 14:02:57 +0200 Subject: [PATCH 026/103] named throttle, no-throttle --- go/logic/server.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/logic/server.go b/go/logic/server.go index 831fdfa..774c4ab 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -292,12 +292,22 @@ help # This message } case "throttle", "pause", "suspend": { + if arg != "" && arg != this.migrationContext.OriginalTableName { + // User explicitly provided table name. This is a courtesy protection mechanism + err := fmt.Errorf("User commanded 'throttle' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) + return NoPrintStatusRule, err + } atomic.StoreInt64(&this.migrationContext.ThrottleCommandedByUser, 1) fmt.Fprintf(writer, throttleHint) return ForcePrintStatusAndHintRule, nil } case "no-throttle", "unthrottle", "resume", "continue": { + if arg != "" && arg != this.migrationContext.OriginalTableName { + // User explicitly provided table name. This is a courtesy protection mechanism + err := fmt.Errorf("User commanded 'no-throttle' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) + return NoPrintStatusRule, err + } atomic.StoreInt64(&this.migrationContext.ThrottleCommandedByUser, 0) return ForcePrintStatusAndHintRule, nil } From 4d7370529b833de4452d034bd284465526d99b9d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 24 Mar 2019 11:32:37 +0200 Subject: [PATCH 027/103] Always use NO_AUTO_VALUE_ON_ZERO --- localtests/autoinc-zero-value/create.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 localtests/autoinc-zero-value/create.sql diff --git a/localtests/autoinc-zero-value/create.sql b/localtests/autoinc-zero-value/create.sql new file mode 100644 index 0000000..ba08bd4 --- /dev/null +++ b/localtests/autoinc-zero-value/create.sql @@ -0,0 +1,9 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + primary key(id) +) auto_increment=1; + +set session sql_mode='NO_AUTO_VALUE_ON_ZERO'; +insert into gh_ost_test values (0, 23); From f0ed7aeb0b09ba9e11ac887802d246214210a4ff Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 24 Mar 2019 11:32:42 +0200 Subject: [PATCH 028/103] Always use NO_AUTO_VALUE_ON_ZERO --- go/logic/applier.go | 68 ++++++--------------------------------------- 1 file changed, 9 insertions(+), 59 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 1544359..5fb795b 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -483,9 +483,12 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } defer tx.Rollback() sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone) + sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO` if !this.migrationContext.SkipStrictMode { - sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum) } + sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum) + if _, err := tx.Exec(sessionQuery); err != nil { return nil, err } @@ -977,63 +980,6 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (result return append(results, newDmlBuildResultError(fmt.Errorf("Unknown dml event type: %+v", dmlEvent.DML))) } -// ApplyDMLEventQuery writes an entry to the ghost table, in response to an intercepted -// original-table binlog event -func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { - for _, buildResult := range this.buildDMLEventQuery(dmlEvent) { - if buildResult.err != nil { - return buildResult.err - } - // TODO The below is in preparation for transactional writes on the ghost tables. - // Such writes would be, for example: - // - prepended with sql_mode setup - // - prepended with time zone setup - // - prepended with SET SQL_LOG_BIN=0 - // - prepended with SET FK_CHECKS=0 - // etc. - // - // a known problem: https://github.com/golang/go/issues/9373 -- bitint unsigned values, not supported in database/sql - // is solved by silently converting unsigned bigints to string values. - // - - err := func() error { - tx, err := this.db.Begin() - if err != nil { - return err - } - rollback := func(err error) error { - tx.Rollback() - return err - } - sessionQuery := fmt.Sprintf("SET SESSION time_zone = '+00:00'") - if !this.migrationContext.SkipStrictMode { - sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" - } - if _, err := tx.Exec(sessionQuery); err != nil { - return rollback(err) - } - if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil { - return rollback(err) - } - if err := tx.Commit(); err != nil { - return err - } - return nil - }() - - if err != nil { - err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), buildResult.query, buildResult.args) - return log.Errore(err) - } - // no error - atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, 1) - if this.migrationContext.CountTableRows { - atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, buildResult.rowsDelta) - } - } - return nil -} - // ApplyDMLEventQueries applies multiple DML queries onto the _ghost_ table func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) error { @@ -1051,9 +997,13 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) } sessionQuery := "SET SESSION time_zone = '+00:00'" + + sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO` if !this.migrationContext.SkipStrictMode { - sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum) } + sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum) + if _, err := tx.Exec(sessionQuery); err != nil { return rollback(err) } From 773cddc88acaa47b52f99157a3faf746b3708f78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 29 Mar 2019 05:05:40 -0700 Subject: [PATCH 029/103] Clarify it's foreign key /constraints/ that don't work --- doc/requirements-and-limitations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 628451d..8376e18 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -22,7 +22,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th ### Limitations -- Foreign keys not supported. They may be supported in the future, to some extent. +- Foreign key constraints are not supported. They may be supported in the future, to some extent. - Triggers are not supported. They may be supported in the future. From 256025853c3cb386453ed223f865c9affc249b93 Mon Sep 17 00:00:00 2001 From: Jon Chambers Date: Fri, 12 Apr 2019 12:07:00 -0400 Subject: [PATCH 030/103] Fix a trivial it's/its typo. --- doc/requirements-and-limitations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 8376e18..096e2c9 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -38,7 +38,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - It is not allowed to migrate a table where another table exists with same name and different upper/lower case. - For example, you may not migrate `MyTable` if another table called `MYtable` exists in the same schema. -- Amazon RDS works, but has it's own [limitations](rds.md). +- Amazon RDS works, but has its own [limitations](rds.md). - Google Cloud SQL works, `--gcp` flag required. - Aliyun RDS works, `--aliyun-rds` flag required. From b8cdd5d133b97ac308dc33323b797a6d57bdc785 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 16 May 2019 13:29:40 +0300 Subject: [PATCH 031/103] Update go-sql-driver/mysql 2019-05-16 --- .../mysql/.github/ISSUE_TEMPLATE.md | 21 + .../mysql/.github/PULL_REQUEST_TEMPLATE.md | 9 + .../github.com/go-sql-driver/mysql/.gitignore | 1 + .../go-sql-driver/mysql/.travis.yml | 126 +- .../go-sql-driver/mysql/.travis/docker.cnf | 5 + .../go-sql-driver/mysql/.travis/gofmt.sh | 7 + .../go-sql-driver/mysql/.travis/wait_mysql.sh | 8 + vendor/github.com/go-sql-driver/mysql/AUTHORS | 57 +- .../go-sql-driver/mysql/CHANGELOG.md | 89 +- .../go-sql-driver/mysql/CONTRIBUTING.md | 17 - .../github.com/go-sql-driver/mysql/README.md | 197 +- .../go-sql-driver/mysql/appengine.go | 10 +- vendor/github.com/go-sql-driver/mysql/auth.go | 422 +++ .../go-sql-driver/mysql/auth_test.go | 1330 ++++++++ .../go-sql-driver/mysql/benchmark_test.go | 373 ++ .../github.com/go-sql-driver/mysql/buffer.go | 122 +- .../go-sql-driver/mysql/collations.go | 379 ++- .../go-sql-driver/mysql/conncheck.go | 53 + .../go-sql-driver/mysql/conncheck_dummy.go | 17 + .../go-sql-driver/mysql/conncheck_test.go | 38 + .../go-sql-driver/mysql/connection.go | 414 ++- .../go-sql-driver/mysql/connection_test.go | 175 + .../go-sql-driver/mysql/connector.go | 143 + .../github.com/go-sql-driver/mysql/const.go | 26 +- .../github.com/go-sql-driver/mysql/driver.go | 136 +- .../go-sql-driver/mysql/driver_go110.go | 37 + .../go-sql-driver/mysql/driver_go110_test.go | 137 + .../go-sql-driver/mysql/driver_test.go | 2996 +++++++++++++++++ vendor/github.com/go-sql-driver/mysql/dsn.go | 636 ++++ .../go-sql-driver/mysql/dsn_test.go | 415 +++ .../github.com/go-sql-driver/mysql/errors.go | 104 +- .../go-sql-driver/mysql/errors_test.go | 42 + .../github.com/go-sql-driver/mysql/fields.go | 194 ++ .../github.com/go-sql-driver/mysql/infile.go | 26 +- .../github.com/go-sql-driver/mysql/packets.go | 614 ++-- .../go-sql-driver/mysql/packets_test.go | 336 ++ vendor/github.com/go-sql-driver/mysql/rows.go | 191 +- .../go-sql-driver/mysql/statement.go | 136 +- .../go-sql-driver/mysql/statement_test.go | 126 + .../go-sql-driver/mysql/transaction.go | 4 +- .../github.com/go-sql-driver/mysql/utils.go | 730 ++-- .../go-sql-driver/mysql/utils_test.go | 334 ++ .../mysql/.github/ISSUE_TEMPLATE.md | 21 + .../mysql/.github/PULL_REQUEST_TEMPLATE.md | 9 + .../github.com/go-sql-driver/mysql/.gitignore | 9 + .../go-sql-driver/mysql/.travis.yml | 128 + .../go-sql-driver/mysql/.travis/docker.cnf | 5 + .../go-sql-driver/mysql/.travis/gofmt.sh | 7 + .../go-sql-driver/mysql/.travis/wait_mysql.sh | 8 + .../github.com/go-sql-driver/mysql/AUTHORS | 11 + .../go-sql-driver/mysql/CHANGELOG.md | 167 + .../go-sql-driver/mysql/CONTRIBUTING.md | 23 + .../github.com/go-sql-driver/mysql/README.md | 495 +++ .../go-sql-driver/mysql/appengine.go | 8 +- .../github.com/go-sql-driver/mysql/auth.go | 44 +- .../go-sql-driver/mysql/auth_test.go | 1330 ++++++++ .../go-sql-driver/mysql/benchmark_test.go | 373 ++ .../github.com/go-sql-driver/mysql/buffer.go | 97 +- .../go-sql-driver/mysql/collations.go | 376 ++- .../go-sql-driver/mysql/conncheck.go | 53 + .../go-sql-driver/mysql/conncheck_dummy.go | 17 + .../go-sql-driver/mysql/conncheck_test.go | 38 + .../go-sql-driver/mysql/connection.go | 45 +- .../go-sql-driver/mysql/connection_test.go | 175 + .../go-sql-driver/mysql/connector.go | 143 + .../github.com/go-sql-driver/mysql/driver.go | 136 +- .../go-sql-driver/mysql/driver_go110.go | 37 + .../go-sql-driver/mysql/driver_go110_test.go | 137 + .../go-sql-driver/mysql/driver_test.go | 2996 +++++++++++++++++ .../github.com/go-sql-driver/mysql/dsn.go | 73 +- .../go-sql-driver/mysql/dsn_test.go | 415 +++ .../go-sql-driver/mysql/errors_test.go | 42 + .../github.com/go-sql-driver/mysql/packets.go | 132 +- .../go-sql-driver/mysql/packets_test.go | 336 ++ .../github.com/go-sql-driver/mysql/rows.go | 7 + .../go-sql-driver/mysql/statement.go | 11 +- .../go-sql-driver/mysql/statement_test.go | 126 + .../github.com/go-sql-driver/mysql/utils.go | 6 +- .../go-sql-driver/mysql/utils_test.go | 334 ++ 79 files changed, 17791 insertions(+), 1812 deletions(-) create mode 100644 vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md create mode 100644 vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf create mode 100755 vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh create mode 100755 vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh create mode 100644 vendor/github.com/go-sql-driver/mysql/auth.go create mode 100644 vendor/github.com/go-sql-driver/mysql/auth_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/benchmark_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/conncheck.go create mode 100644 vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go create mode 100644 vendor/github.com/go-sql-driver/mysql/conncheck_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/connection_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/connector.go create mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110.go create mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/driver_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/dsn.go create mode 100644 vendor/github.com/go-sql-driver/mysql/dsn_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/errors_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/fields.go create mode 100644 vendor/github.com/go-sql-driver/mysql/packets_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/statement_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/utils_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf create mode 100755 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh create mode 100755 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go diff --git a/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md b/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..d9771f1 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,21 @@ +### Issue description +Tell us what should happen and what happens instead + +### Example code +```go +If possible, please enter some example code here to reproduce the issue. +``` + +### Error log +``` +If you have an error log, please paste it here. +``` + +### Configuration +*Driver version (or git SHA):* + +*Go version:* run `go version` in your console + +*Server version:* E.g. MySQL 5.6, MariaDB 10.0.20 + +*Server OS:* E.g. Debian 8.1 (Jessie), Windows 10 diff --git a/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md b/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6f5c7eb --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +### Description +Please explain the changes you made here. + +### Checklist +- [ ] Code compiles correctly +- [ ] Created tests which fail without the change (if possible) +- [ ] All tests passing +- [ ] Extended the README / documentation, if necessary +- [ ] Added myself / the copyright holder to the AUTHORS file diff --git a/vendor/github.com/go-sql-driver/mysql/.gitignore b/vendor/github.com/go-sql-driver/mysql/.gitignore index ba8e0cb..2de28da 100644 --- a/vendor/github.com/go-sql-driver/mysql/.gitignore +++ b/vendor/github.com/go-sql-driver/mysql/.gitignore @@ -6,3 +6,4 @@ Icon? ehthumbs.db Thumbs.db +.idea diff --git a/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/go-sql-driver/mysql/.travis.yml index 2f4e3c2..eae311b 100644 --- a/vendor/github.com/go-sql-driver/mysql/.travis.yml +++ b/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -1,10 +1,128 @@ sudo: false language: go go: - - 1.2 - - 1.3 - - 1.4 - - tip + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - master + +before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB" | sudo tee -a /etc/mysql/my.cnf + - sudo service mysql restart + - .travis/wait_mysql.sh - mysql -e 'create database gotest;' + +matrix: + include: + - env: DB=MYSQL8 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:8.0 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:8.0 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MYSQL57 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:5.7 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:5.7 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA55 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:5.5 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:5.5 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA10_1 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:10.1 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:10.1 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - os: osx + osx_image: xcode10.1 + addons: + homebrew: + packages: + - mysql + go: 1.12.x + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + - mysql.server start + - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + - mysql -uroot -e 'create database gotest;' + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3306 + - export MYSQL_TEST_CONCURRENT=1 + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - go vet ./... + - .travis/gofmt.sh +after_script: + - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf b/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf new file mode 100644 index 0000000..e57754e --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf @@ -0,0 +1,5 @@ +[client] +user = gotest +password = secret +host = 127.0.0.1 +port = 3307 diff --git a/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh b/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh new file mode 100755 index 0000000..9bf0d16 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ev + +# Only check for go1.10+ since the gofmt style changed +if [[ $(go version) =~ go1\.([0-9]+) ]] && ((${BASH_REMATCH[1]} >= 10)); then + test -z "$(gofmt -d -s . | tee /dev/stderr)" +fi diff --git a/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh b/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh new file mode 100755 index 0000000..e87993e --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh @@ -0,0 +1,8 @@ +#!/bin/sh +while : +do + if mysql -e 'select version()' 2>&1 | grep 'version()\|ERROR 2059 (HY000):'; then + break + fi + sleep 3 +done diff --git a/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/go-sql-driver/mysql/AUTHORS index 6dd0167..bfe74c4 100644 --- a/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -12,35 +12,90 @@ # Individual Persons Aaron Hopkins +Achille Roussel +Alexey Palazhchenko +Andrew Reid Arne Hormann +Asta Xie +Bulat Gaifullin Carlos Nieto Chris Moos +Craig Wilson +Daniel Montoya +Daniel Nichter +Daniël van Eeden +Dave Protasowski DisposaBoy +Egor Smolyakov +Erwan Martin +Evan Shaw Frederick Mayle Gustavo Kristic +Hajime Nakagami Hanno Braun Henri Yandell Hirotaka Yamamoto +Huyiguang +ICHINOSE Shogo +Ilia Cimpoes INADA Naoki +Jacek Szwec James Harr +Jeff Hodges +Jeffrey Charles +Jerome Meyer Jian Zhen Joshua Prunier +Julien Lefevre Julien Schmidt +Justin Li +Justin Nuß Kamil Dziedzic +Kevin Malachowski +Kieron Woodhouse +Lennart Rudolph Leonardo YongUk Kim +Linh Tran Tuan +Lion Yang +Luca Looz Lucas Liu Luke Scott +Maciej Zimnoch Michael Woolnough Nicola Peduzzi +Olivier Mengué +oscarzhao +Paul Bonser +Peter Schultz +Rebecca Chin +Reed Allman +Richard Wilkes +Robert Russell Runrioter Wung +Shuode Li +Simon J Mudd Soroush Pour Stan Putrya +Stanley Gunawan +Steven Hartland +Thomas Wodarek +Tim Ruffles +Tom Jenkinson +Xiangyu Hu Xiaobing Jiang Xiuming Chen -Julien Lefevre +Zhenye Xie # Organizations Barracuda Networks, Inc. +Counting Ltd. +Facebook Inc. +GitHub Inc. Google Inc. +InfoSum Ltd. +Keybase Inc. +Multiplay Ltd. +Percona LLC +Pivotal Inc. Stripe Inc. diff --git a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md index 161ad0f..2d87d74 100644 --- a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md +++ b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -1,21 +1,96 @@ -## HEAD +## Version 1.4 (2018-06-03) + +Changes: + + - Documentation fixes (#530, #535, #567) + - Refactoring (#575, #579, #580, #581, #603, #615, #704) + - Cache column names (#444) + - Sort the DSN parameters in DSNs generated from a config (#637) + - Allow native password authentication by default (#644) + - Use the default port if it is missing in the DSN (#668) + - Removed the `strict` mode (#676) + - Do not query `max_allowed_packet` by default (#680) + - Dropped support Go 1.6 and lower (#696) + - Updated `ConvertValue()` to match the database/sql/driver implementation (#760) + - Document the usage of `0000-00-00T00:00:00` as the time.Time zero value (#783) + - Improved the compatibility of the authentication system (#807) + +New Features: + + - Multi-Results support (#537) + - `rejectReadOnly` DSN option (#604) + - `context.Context` support (#608, #612, #627, #761) + - Transaction isolation level support (#619, #744) + - Read-Only transactions support (#618, #634) + - `NewConfig` function which initializes a config with default values (#679) + - Implemented the `ColumnType` interfaces (#667, #724) + - Support for custom string types in `ConvertValue` (#623) + - Implemented `NamedValueChecker`, improving support for uint64 with high bit set (#690, #709, #710) + - `caching_sha2_password` authentication plugin support (#794, #800, #801, #802) + - Implemented `driver.SessionResetter` (#779) + - `sha256_password` authentication plugin support (#808) + +Bugfixes: + + - Use the DSN hostname as TLS default ServerName if `tls=true` (#564, #718) + - Fixed LOAD LOCAL DATA INFILE for empty files (#590) + - Removed columns definition cache since it sometimes cached invalid data (#592) + - Don't mutate registered TLS configs (#600) + - Make RegisterTLSConfig concurrency-safe (#613) + - Handle missing auth data in the handshake packet correctly (#646) + - Do not retry queries when data was written to avoid data corruption (#302, #736) + - Cache the connection pointer for error handling before invalidating it (#678) + - Fixed imports for appengine/cloudsql (#700) + - Fix sending STMT_LONG_DATA for 0 byte data (#734) + - Set correct capacity for []bytes read from length-encoded strings (#766) + - Make RegisterDial concurrency-safe (#773) + + +## Version 1.3 (2016-12-01) Changes: - Go 1.1 is no longer supported - - Use decimals field from MySQL to format time types (#249) + - Use decimals fields in MySQL to format time types (#249) - Buffer optimizations (#269) - TLS ServerName defaults to the host (#283) + - Refactoring (#400, #410, #437) + - Adjusted documentation for second generation CloudSQL (#485) + - Documented DSN system var quoting rules (#502) + - Made statement.Close() calls idempotent to avoid errors in Go 1.6+ (#512) + +New Features: + + - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) + - Support for returning table alias on Columns() (#289, #359, #382) + - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490) + - Support for uint64 parameters with high bit set (#332, #345) + - Cleartext authentication plugin support (#327) + - Exported ParseDSN function and the Config struct (#403, #419, #429) + - Read / Write timeouts (#401) + - Support for JSON field type (#414) + - Support for multi-statements and multi-results (#411, #431) + - DSN parameter to set the driver-side max_allowed_packet value manually (#489) + - Native password authentication plugin support (#494, #524) Bugfixes: - - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) - Fixed handling of queries without columns and rows (#255) - Fixed a panic when SetKeepAlive() failed (#298) - -New Features: - - Support for returning table alias on Columns() (#289) - - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318) + - Handle ERR packets while reading rows (#321) + - Fixed reading NULL length-encoded integers in MySQL 5.6+ (#349) + - Fixed absolute paths support in LOAD LOCAL DATA INFILE (#356) + - Actually zero out bytes in handshake response (#378) + - Fixed race condition in registering LOAD DATA INFILE handler (#383) + - Fixed tests with MySQL 5.7.9+ (#380) + - QueryUnescape TLS config names (#397) + - Fixed "broken pipe" error by writing to closed socket (#390) + - Fixed LOAD LOCAL DATA INFILE buffering (#424) + - Fixed parsing of floats into float64 when placeholders are used (#434) + - Fixed DSN tests with Go 1.7+ (#459) + - Handle ERR packets while waiting for EOF (#473) + - Invalidate connection on error while discarding additional results (#513) + - Allow terminating packets of length 0 (#516) ## Version 1.2 (2014-06-03) diff --git a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md index f87c198..8fe16bc 100644 --- a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md +++ b/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md @@ -4,28 +4,11 @@ Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed). -Please provide the following minimum information: -* Your Go-MySQL-Driver version (or git SHA) -* Your Go version (run `go version` in your console) -* A detailed issue description -* Error Log if present -* If possible, a short example - - ## Contributing Code By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file. Don't forget to add yourself to the AUTHORS file. -### Pull Requests Checklist - -Please check the following points before submitting your pull request: -- [x] Code compiles correctly -- [x] Created tests, if possible -- [x] All tests pass -- [x] Extended the README / documentation, if necessary -- [x] Added yourself to the AUTHORS file - ### Code Review Everyone is invited to review and comment on pull requests. diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md index 706b7ef..c6adf1d 100644 --- a/vendor/github.com/go-sql-driver/mysql/README.md +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -1,13 +1,9 @@ # Go-MySQL-Driver -A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) package +A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package ![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin") -**Latest stable Release:** [Version 1.2 (June 03, 2014)](https://github.com/go-sql-driver/mysql/releases) - -[![Build Status](https://travis-ci.org/go-sql-driver/mysql.png?branch=master)](https://travis-ci.org/go-sql-driver/mysql) - --------------------------------------- * [Features](#features) * [Requirements](#requirements) @@ -19,6 +15,9 @@ A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) packa * [Address](#address) * [Parameters](#parameters) * [Examples](#examples) + * [Connection pool and timeouts](#connection-pool-and-timeouts) + * [context.Context Support](#contextcontext-support) + * [ColumnType Support](#columntype-support) * [LOAD DATA LOCAL INFILE support](#load-data-local-infile-support) * [time.Time support](#timetime-support) * [Unicode support](#unicode-support) @@ -30,31 +29,31 @@ A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) packa ## Features * Lightweight and [fast](https://github.com/go-sql-driver/sql-benchmark "golang MySQL-Driver performance") * Native Go implementation. No C-bindings, just pure Go - * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](http://godoc.org/github.com/go-sql-driver/mysql#DialFunc) + * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](https://godoc.org/github.com/go-sql-driver/mysql#DialFunc) * Automatic handling of broken connections * Automatic Connection Pooling *(by database/sql package)* * Supports queries larger than 16MB - * Full [`sql.RawBytes`](http://golang.org/pkg/database/sql/#RawBytes) support. + * Full [`sql.RawBytes`](https://golang.org/pkg/database/sql/#RawBytes) support. * Intelligent `LONG DATA` handling in prepared statements * Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support * Optional `time.Time` parsing * Optional placeholder interpolation ## Requirements - * Go 1.2 or higher + * Go 1.9 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- ## Installation -Simple install the package to your [$GOPATH](http://code.google.com/p/go-wiki/wiki/GOPATH "GOPATH") with the [go tool](http://golang.org/cmd/go/ "go command") from shell: +Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell: ```bash -$ go get github.com/go-sql-driver/mysql +$ go get -u github.com/go-sql-driver/mysql ``` -Make sure [Git is installed](http://git-scm.com/downloads) on your machine and in your system's `PATH`. +Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`. ## Usage -_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](http://golang.org/pkg/database/sql) API then. +_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](https://golang.org/pkg/database/sql/) API then. Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: ```go @@ -93,17 +92,20 @@ This has the same effect as an empty DSN string: ``` +Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. + #### Password Passwords can consist of any character. Escaping is **not** necessary. #### Protocol -See [net.Dial](http://golang.org/pkg/net/#Dial) for more information which networks are available. +See [net.Dial](https://golang.org/pkg/net/#Dial) for more information which networks are available. In general you should use an Unix domain socket if available and TCP otherwise for best performance. #### Address -For TCP and UDP networks, addresses have the form `host:port`. +For TCP and UDP networks, addresses have the form `host[:port]`. +If `port` is omitted, the default port will be used. If `host` is a literal IPv6 address, it must be enclosed in square brackets. -The functions [net.JoinHostPort](http://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](http://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. +The functions [net.JoinHostPort](https://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](https://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. For Unix domain sockets the address is the absolute path to the MySQL-Server-socket, e.g. `/var/run/mysqld/mysqld.sock` or `/tmp/mysql.sock`. @@ -133,6 +135,15 @@ Default: false `allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. +##### `allowNativePasswords` + +``` +Type: bool +Valid Values: true, false +Default: true +``` +`allowNativePasswords=false` disallows the usage of MySQL native password method. + ##### `allowOldPasswords` ``` @@ -160,13 +171,18 @@ Unless you need the fallback behavior, please use `collation` instead. ``` Type: string Valid Values: -Default: utf8_general_ci +Default: utf8mb4_general_ci ``` Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. A list of valid charsets for a server is retrievable with `SHOW COLLATION`. +The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You should use an older collation (e.g. `utf8_general_ci`) for older MySQL. + +Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). + + ##### `clientFoundRows` ``` @@ -213,12 +229,31 @@ Valid Values: Default: UTC ``` -Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](http://golang.org/pkg/time/#LoadLocation) for details. +Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](https://golang.org/pkg/time/#LoadLocation) for details. Note that this sets the location for time.Time values but does not change MySQL's [time_zone setting](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html). For that see the [time_zone system variable](#system-variables), which can also be set as a DSN parameter. -Please keep in mind, that param values must be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. +Please keep in mind, that param values must be [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. +##### `maxAllowedPacket` +``` +Type: decimal number +Default: 4194304 +``` + +Max packet size allowed in bytes. The default value is 4 MiB and should be adjusted to match the server settings. `maxAllowedPacket=0` can be used to automatically fetch the `max_allowed_packet` variable from server *on every connection*. + +##### `multiStatements` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. + +When `multiStatements` is used, `?` parameters must only be used in the first statement. ##### `parseTime` @@ -229,9 +264,19 @@ Default: false ``` `parseTime=true` changes the output type of `DATE` and `DATETIME` values to `time.Time` instead of `[]byte` / `string` +The date or datetime like `0000-00-00 00:00:00` is converted into zero value of `time.Time`. -##### `strict` +##### `readTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O read timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + +##### `rejectReadOnly` ``` Type: bool @@ -239,41 +284,89 @@ Valid Values: true, false Default: false ``` -`strict=true` enables the strict mode in which MySQL warnings are treated as errors. -By default MySQL also treats notes as warnings. Use [`sql_notes=false`](http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sql_notes) to ignore notes. See the [examples](#examples) for an DSN example. +`rejectReadOnly=true` causes the driver to reject read-only connections. This +is for a possible race condition during an automatic failover, where the mysql +client gets connected to a read-only replica after the failover. + +Note that this should be a fairly rare case, as an automatic failover normally +happens when the primary is down, and the race condition shouldn't happen +unless it comes back up online as soon as the failover is kicked off. On the +other hand, when this happens, a MySQL application can get stuck on a +read-only connection until restarted. It is however fairly easy to reproduce, +for example, using a manual failover on AWS Aurora's MySQL-compatible cluster. + +If you are not relying on read-only transactions to reject writes that aren't +supposed to happen, setting this on some MySQL providers (such as AWS Aurora) +is safer for failovers. + +Note that ERROR 1290 can be returned for a `read-only` server and this option will +cause a retry for that error. However the same error number is used for some +other cases. You should ensure your application will never cause an ERROR 1290 +except for `read-only` mode when enabling this option. + + +##### `serverPubKey` + +``` +Type: string +Valid Values: +Default: none +``` + +Server public keys can be registered with [`mysql.RegisterServerPubKey`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterServerPubKey), which can then be used by the assigned name in the DSN. +Public keys are used to transmit encrypted data, e.g. for authentication. +If the server's public key is known, it should be set manually to avoid expensive and potentially insecure transmissions of the public key from the server to the client each time it is required. ##### `timeout` ``` -Type: decimal number +Type: duration Default: OS default ``` -*Driver* side connection timeout. The value must be a string of decimal numbers, each with optional fraction and a unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. To set a server side timeout, use the parameter [`wait_timeout`](http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_wait_timeout). +Timeout for establishing connections, aka dial timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. ##### `tls` ``` Type: bool / string -Valid Values: true, false, skip-verify, +Valid Values: true, false, skip-verify, preferred, Default: false ``` -`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use a custom value registered with [`mysql.RegisterTLSConfig`](http://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side) or use `preferred` to use TLS only when advertised by the server. This is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Neither `skip-verify` nor `preferred` add any reliable security. You can use a custom TLS config after registering it with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). + + +##### `writeTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O write timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. ##### System Variables -All other parameters are interpreted as system variables: - * `autocommit`: `"SET autocommit="` - * [`time_zone`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `"SET time_zone="` - * [`tx_isolation`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `"SET tx_isolation="` - * `param`: `"SET ="` +Any other parameters are interpreted as system variables: + * `=`: `SET =` + * `=`: `SET =` + * `=%27%27`: `SET =''` + +Rules: +* The values for string variables must be quoted with `'`. +* The values must also be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed! + (which implies values of string variables must be wrapped with `%27`). + +Examples: + * `autocommit=1`: `SET autocommit=1` + * [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'` + * [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'` -*The values must be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed!* #### Examples ``` @@ -288,9 +381,9 @@ root:pw@unix(/tmp/mysql.sock)/myDatabase?loc=Local user:password@tcp(localhost:5555)/dbname?tls=skip-verify&autocommit=true ``` -Use the [strict mode](#strict) but ignore notes: +Treat warnings as errors by setting the system variable [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html): ``` -user:password@/dbname?strict=true&sql_notes=false +user:password@/dbname?sql_mode=TRADITIONAL ``` TCP via IPv6: @@ -303,11 +396,16 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine: +Google Cloud SQL on App Engine (First Generation MySQL Server): ``` user@cloudsql(project-id:instance-name)/dbname ``` +Google Cloud SQL on App Engine (Second Generation MySQL Server): +``` +user@cloudsql(project-id:regionname:instance-name)/dbname +``` + TCP using default port (3306) on localhost: ``` user:password@tcp/dbname?charset=utf8mb4,utf8&sys_var=esc%40ped @@ -323,6 +421,18 @@ No Database preselected: user:password@/ ``` + +### Connection pool and timeouts +The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively. + +## `ColumnType` Support +This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. + +## `context.Context` Support +Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts. +See [context support in the database/sql package](https://golang.org/doc/go1.8#database_sql) for more details. + + ### `LOAD DATA LOCAL INFILE` support For this feature you need direct access to the package. Therefore you must change the import path (no `_`): ```go @@ -333,17 +443,17 @@ Files must be whitelisted by registering them with `mysql.RegisterLocalFile(file To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. -See the [godoc of Go-MySQL-Driver](http://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. +See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. ### `time.Time` support -The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your programm. +The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program. -However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](http://golang.org/pkg/time/#Location) with the `loc` DSN parameter. +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical equivalent in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. **Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). -Alternatively you can use the [`NullTime`](http://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. +Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. ### Unicode support @@ -355,7 +465,6 @@ Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAM See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. - ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. @@ -374,13 +483,13 @@ Mozilla summarizes the license scope as follows: That means: - * You can **use** the **unchanged** source code both in private and commercially - * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0) - * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged** + * You can **use** the **unchanged** source code both in private and commercially. + * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0). + * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged**. -Please read the [MPL 2.0 FAQ](http://www.mozilla.org/MPL/2.0/FAQ.html) if you have further questions regarding the license. +Please read the [MPL 2.0 FAQ](https://www.mozilla.org/en-US/MPL/2.0/FAQ/) if you have further questions regarding the license. -You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) +You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE). ![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow") diff --git a/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/go-sql-driver/mysql/appengine.go index 565614e..914e662 100644 --- a/vendor/github.com/go-sql-driver/mysql/appengine.go +++ b/vendor/github.com/go-sql-driver/mysql/appengine.go @@ -11,9 +11,15 @@ package mysql import ( - "appengine/cloudsql" + "context" + "net" + + "google.golang.org/appengine/cloudsql" ) func init() { - RegisterDial("cloudsql", cloudsql.Dial) + RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { + // XXX: the cloudsql driver still does not export a Context-aware dialer. + return cloudsql.Dial(instance) + }) } diff --git a/vendor/github.com/go-sql-driver/mysql/auth.go b/vendor/github.com/go-sql-driver/mysql/auth.go new file mode 100644 index 0000000..fec7040 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/auth.go @@ -0,0 +1,422 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/x509" + "encoding/pem" + "sync" +) + +// server pub keys registry +var ( + serverPubKeyLock sync.RWMutex + serverPubKeyRegistry map[string]*rsa.PublicKey +) + +// RegisterServerPubKey registers a server RSA public key which can be used to +// send data in a secure manner to the server without receiving the public key +// in a potentially insecure way from the server first. +// Registered keys can afterwards be used adding serverPubKey= to the DSN. +// +// Note: The provided rsa.PublicKey instance is exclusively owned by the driver +// after registering it and may not be modified. +// +// data, err := ioutil.ReadFile("mykey.pem") +// if err != nil { +// log.Fatal(err) +// } +// +// block, _ := pem.Decode(data) +// if block == nil || block.Type != "PUBLIC KEY" { +// log.Fatal("failed to decode PEM block containing public key") +// } +// +// pub, err := x509.ParsePKIXPublicKey(block.Bytes) +// if err != nil { +// log.Fatal(err) +// } +// +// if rsaPubKey, ok := pub.(*rsa.PublicKey); ok { +// mysql.RegisterServerPubKey("mykey", rsaPubKey) +// } else { +// log.Fatal("not a RSA public key") +// } +// +func RegisterServerPubKey(name string, pubKey *rsa.PublicKey) { + serverPubKeyLock.Lock() + if serverPubKeyRegistry == nil { + serverPubKeyRegistry = make(map[string]*rsa.PublicKey) + } + + serverPubKeyRegistry[name] = pubKey + serverPubKeyLock.Unlock() +} + +// DeregisterServerPubKey removes the public key registered with the given name. +func DeregisterServerPubKey(name string) { + serverPubKeyLock.Lock() + if serverPubKeyRegistry != nil { + delete(serverPubKeyRegistry, name) + } + serverPubKeyLock.Unlock() +} + +func getServerPubKey(name string) (pubKey *rsa.PublicKey) { + serverPubKeyLock.RLock() + if v, ok := serverPubKeyRegistry[name]; ok { + pubKey = v + } + serverPubKeyLock.RUnlock() + return +} + +// Hash password using pre 4.1 (old password) method +// https://github.com/atcurtis/mariadb/blob/master/mysys/my_rnd.c +type myRnd struct { + seed1, seed2 uint32 +} + +const myRndMaxVal = 0x3FFFFFFF + +// Pseudo random number generator +func newMyRnd(seed1, seed2 uint32) *myRnd { + return &myRnd{ + seed1: seed1 % myRndMaxVal, + seed2: seed2 % myRndMaxVal, + } +} + +// Tested to be equivalent to MariaDB's floating point variant +// http://play.golang.org/p/QHvhd4qved +// http://play.golang.org/p/RG0q4ElWDx +func (r *myRnd) NextByte() byte { + r.seed1 = (r.seed1*3 + r.seed2) % myRndMaxVal + r.seed2 = (r.seed1 + r.seed2 + 33) % myRndMaxVal + + return byte(uint64(r.seed1) * 31 / myRndMaxVal) +} + +// Generate binary hash from byte string using insecure pre 4.1 method +func pwHash(password []byte) (result [2]uint32) { + var add uint32 = 7 + var tmp uint32 + + result[0] = 1345345333 + result[1] = 0x12345671 + + for _, c := range password { + // skip spaces and tabs in password + if c == ' ' || c == '\t' { + continue + } + + tmp = uint32(c) + result[0] ^= (((result[0] & 63) + add) * tmp) + (result[0] << 8) + result[1] += (result[1] << 8) ^ result[0] + add += tmp + } + + // Remove sign bit (1<<31)-1) + result[0] &= 0x7FFFFFFF + result[1] &= 0x7FFFFFFF + + return +} + +// Hash password using insecure pre 4.1 method +func scrambleOldPassword(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + scramble = scramble[:8] + + hashPw := pwHash([]byte(password)) + hashSc := pwHash(scramble) + + r := newMyRnd(hashPw[0]^hashSc[0], hashPw[1]^hashSc[1]) + + var out [8]byte + for i := range out { + out[i] = r.NextByte() + 64 + } + + mask := r.NextByte() + for i := range out { + out[i] ^= mask + } + + return out[:] +} + +// Hash password using 4.1+ method (SHA1) +func scramblePassword(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + // stage1Hash = SHA1(password) + crypt := sha1.New() + crypt.Write([]byte(password)) + stage1 := crypt.Sum(nil) + + // scrambleHash = SHA1(scramble + SHA1(stage1Hash)) + // inner Hash + crypt.Reset() + crypt.Write(stage1) + hash := crypt.Sum(nil) + + // outer Hash + crypt.Reset() + crypt.Write(scramble) + crypt.Write(hash) + scramble = crypt.Sum(nil) + + // token = scrambleHash XOR stage1Hash + for i := range scramble { + scramble[i] ^= stage1[i] + } + return scramble +} + +// Hash password using MySQL 8+ method (SHA256) +func scrambleSHA256Password(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + // XOR(SHA256(password), SHA256(SHA256(SHA256(password)), scramble)) + + crypt := sha256.New() + crypt.Write([]byte(password)) + message1 := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1) + message1Hash := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1Hash) + crypt.Write(scramble) + message2 := crypt.Sum(nil) + + for i := range message1 { + message1[i] ^= message2[i] + } + + return message1 +} + +func encryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, error) { + plain := make([]byte, len(password)+1) + copy(plain, password) + for i := range plain { + j := i % len(seed) + plain[i] ^= seed[j] + } + sha1 := sha1.New() + return rsa.EncryptOAEP(sha1, rand.Reader, pub, plain, nil) +} + +func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) error { + enc, err := encryptPassword(mc.cfg.Passwd, seed, pub) + if err != nil { + return err + } + return mc.writeAuthSwitchPacket(enc) +} + +func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { + switch plugin { + case "caching_sha2_password": + authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) + return authResp, nil + + case "mysql_old_password": + if !mc.cfg.AllowOldPasswords { + return nil, ErrOldPassword + } + // Note: there are edge cases where this should work but doesn't; + // this is currently "wontfix": + // https://github.com/go-sql-driver/mysql/issues/184 + authResp := append(scrambleOldPassword(authData[:8], mc.cfg.Passwd), 0) + return authResp, nil + + case "mysql_clear_password": + if !mc.cfg.AllowCleartextPasswords { + return nil, ErrCleartextPassword + } + // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html + // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html + return append([]byte(mc.cfg.Passwd), 0), nil + + case "mysql_native_password": + if !mc.cfg.AllowNativePasswords { + return nil, ErrNativePassword + } + // https://dev.mysql.com/doc/internals/en/secure-password-authentication.html + // Native password authentication only need and will need 20-byte challenge. + authResp := scramblePassword(authData[:20], mc.cfg.Passwd) + return authResp, nil + + case "sha256_password": + if len(mc.cfg.Passwd) == 0 { + return []byte{0}, nil + } + if mc.cfg.tls != nil || mc.cfg.Net == "unix" { + // write cleartext auth packet + return append([]byte(mc.cfg.Passwd), 0), nil + } + + pubKey := mc.cfg.pubKey + if pubKey == nil { + // request public key from server + return []byte{1}, nil + } + + // encrypted password + enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) + return enc, err + + default: + errLog.Print("unknown auth plugin:", plugin) + return nil, ErrUnknownPlugin + } +} + +func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { + // Read Result Packet + authData, newPlugin, err := mc.readAuthResult() + if err != nil { + return err + } + + // handle auth plugin switch, if requested + if newPlugin != "" { + // If CLIENT_PLUGIN_AUTH capability is not supported, no new cipher is + // sent and we have to keep using the cipher sent in the init packet. + if authData == nil { + authData = oldAuthData + } else { + // copy data from read buffer to owned slice + copy(oldAuthData, authData) + } + + plugin = newPlugin + + authResp, err := mc.auth(authData, plugin) + if err != nil { + return err + } + if err = mc.writeAuthSwitchPacket(authResp); err != nil { + return err + } + + // Read Result Packet + authData, newPlugin, err = mc.readAuthResult() + if err != nil { + return err + } + + // Do not allow to change the auth plugin more than once + if newPlugin != "" { + return ErrMalformPkt + } + } + + switch plugin { + + // https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/ + case "caching_sha2_password": + switch len(authData) { + case 0: + return nil // auth successful + case 1: + switch authData[0] { + case cachingSha2PasswordFastAuthSuccess: + if err = mc.readResultOK(); err == nil { + return nil // auth successful + } + + case cachingSha2PasswordPerformFullAuthentication: + if mc.cfg.tls != nil || mc.cfg.Net == "unix" { + // write cleartext auth packet + err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0)) + if err != nil { + return err + } + } else { + pubKey := mc.cfg.pubKey + if pubKey == nil { + // request public key from server + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + return err + } + data[4] = cachingSha2PasswordRequestPublicKey + mc.writePacket(data) + + // parse public key + if data, err = mc.readPacket(); err != nil { + return err + } + + block, _ := pem.Decode(data[1:]) + pkix, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + pubKey = pkix.(*rsa.PublicKey) + } + + // send encrypted password + err = mc.sendEncryptedPassword(oldAuthData, pubKey) + if err != nil { + return err + } + } + return mc.readResultOK() + + default: + return ErrMalformPkt + } + default: + return ErrMalformPkt + } + + case "sha256_password": + switch len(authData) { + case 0: + return nil // auth successful + default: + block, _ := pem.Decode(authData) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + + // send encrypted password + err = mc.sendEncryptedPassword(oldAuthData, pub.(*rsa.PublicKey)) + if err != nil { + return err + } + return mc.readResultOK() + } + + default: + return nil // auth successful + } + + return err +} diff --git a/vendor/github.com/go-sql-driver/mysql/auth_test.go b/vendor/github.com/go-sql-driver/mysql/auth_test.go new file mode 100644 index 0000000..1920ef3 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/auth_test.go @@ -0,0 +1,1330 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "testing" +) + +var testPubKey = []byte("-----BEGIN PUBLIC KEY-----\n" + + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAol0Z8G8U+25Btxk/g/fm\n" + + "UAW/wEKjQCTjkibDE4B+qkuWeiumg6miIRhtilU6m9BFmLQSy1ltYQuu4k17A4tQ\n" + + "rIPpOQYZges/qsDFkZh3wyK5jL5WEFVdOasf6wsfszExnPmcZS4axxoYJfiuilrN\n" + + "hnwinBAqfi3S0sw5MpSI4Zl1AbOrHG4zDI62Gti2PKiMGyYDZTS9xPrBLbN95Kby\n" + + "FFclQLEzA9RJcS1nHFsWtRgHjGPhhjCQxEm9NQ1nePFhCfBfApyfH1VM2VCOQum6\n" + + "Ci9bMuHWjTjckC84mzF99kOxOWVU7mwS6gnJqBzpuz8t3zq8/iQ2y7QrmZV+jTJP\n" + + "WQIDAQAB\n" + + "-----END PUBLIC KEY-----\n") + +var testPubKeyRSA *rsa.PublicKey + +func init() { + block, _ := pem.Decode(testPubKey) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + panic(err) + } + testPubKeyRSA = pub.(*rsa.PublicKey) +} + +func TestScrambleOldPass(t *testing.T) { + scramble := []byte{9, 8, 7, 6, 5, 4, 3, 2} + vectors := []struct { + pass string + out string + }{ + {" pass", "47575c5a435b4251"}, + {"pass ", "47575c5a435b4251"}, + {"123\t456", "575c47505b5b5559"}, + {"C0mpl!ca ted#PASS123", "5d5d554849584a45"}, + } + for _, tuple := range vectors { + ours := scrambleOldPassword(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed old password %q", tuple.pass) + } + } +} + +func TestScrambleSHA256Pass(t *testing.T) { + scramble := []byte{10, 47, 74, 111, 75, 73, 34, 48, 88, 76, 114, 74, 37, 13, 3, 80, 82, 2, 23, 21} + vectors := []struct { + pass string + out string + }{ + {"secret", "f490e76f66d9d86665ce54d98c78d0acfe2fb0b08b423da807144873d30b312c"}, + {"secret2", "abc3934a012cf342e876071c8ee202de51785b430258a7a0138bc79c4d800bc6"}, + } + for _, tuple := range vectors { + ours := scrambleSHA256Password(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed SHA256 password %q", tuple.pass) + } + } +} + +func TestAuthFastCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{102, 32, 5, 35, 143, 161, 140, 241, 171, 232, 56, + 139, 43, 14, 107, 196, 249, 170, 147, 60, 220, 204, 120, 178, 214, 15, + 184, 150, 26, 61, 57, 235} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 3, // Fast Auth Success + 7, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // pub key response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{1, 0, 0, 3, 2, 0, 1, 0, 5}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCleartextPasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthFastCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowNativePasswords = false + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthFastNativePassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{53, 177, 140, 159, 251, 189, 127, 53, 109, 252, + 172, 50, 211, 192, 240, 164, 26, 48, 207, 45} + if writtenAuthRespLen != 20 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{1} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + // hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + + // unset TLS config to prevent the actual establishment of a TLS wrapper + mc.cfg.tls = nil + + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 6, 1}, testPubKey...), + + // OK + {7, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 4 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Pub Key Request + 1, 0, 0, 5, 2, + + // 3. Packet: Encrypted Password + 0, 1, 0, 7, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {2, 0, 0, 4, 1, 4}, // Perform Full Authentication + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Cleartext password + 7, 0, 0, 5, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + conn.maxReads = 1 + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthSwitchCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = false + + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + conn.maxReads = 1 + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthSwitchNativePassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{20, 0, 0, 3, 202, 41, 195, 164, 34, 226, 49, 103, + 21, 211, 167, 199, 227, 116, 8, 48, 57, 71, 149, 146} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchOldPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +// Same to TestAuthSwitchOldPasswordNotAllowed, but use OldAuthSwitch request. +func TestOldAuthSwitchNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +func TestAuthSwitchOldPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPassword, but use OldAuthSwitch request. +func TestOldAuthSwitch(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} +func TestAuthSwitchOldPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPasswordEmpty, but use OldAuthSwitch request. +func TestOldAuthSwitchPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // OldAuthSwitch request. + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Empty Password + 1, 0, 0, 3, 0, + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Pub Key Request + 1, 0, 0, 3, 1, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Encrypted Password + 0, 1, 0, 3, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Cleartext Password + 7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/benchmark_test.go b/vendor/github.com/go-sql-driver/mysql/benchmark_test.go new file mode 100644 index 0000000..3e25a3b --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/benchmark_test.go @@ -0,0 +1,373 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "fmt" + "math" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +type TB testing.B + +func (tb *TB) check(err error) { + if err != nil { + tb.Fatal(err) + } +} + +func (tb *TB) checkDB(db *sql.DB, err error) *sql.DB { + tb.check(err) + return db +} + +func (tb *TB) checkRows(rows *sql.Rows, err error) *sql.Rows { + tb.check(err) + return rows +} + +func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt { + tb.check(err) + return stmt +} + +func initDB(b *testing.B, queries ...string) *sql.DB { + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + for _, query := range queries { + if _, err := db.Exec(query); err != nil { + b.Fatalf("error on %q: %v", query, err) + } + } + return db +} + +const concurrencyLevel = 10 + +func BenchmarkQuery(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + var got string + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Errorf("query = %q; want one", got) + wg.Done() + return + } + } + }() + } +} + +func BenchmarkExec(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := tb.checkDB(sql.Open("mysql", dsn)) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("DO 1")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + if _, err := stmt.Exec(); err != nil { + b.Fatal(err.Error()) + } + } + }() + } +} + +// data, but no db writes +var roundtripSample []byte + +func initRoundtripBenchmarks() ([]byte, int, int) { + if roundtripSample == nil { + roundtripSample = []byte(strings.Repeat("0123456789abcdef", 1024*1024)) + } + return roundtripSample, 16, len(roundtripSample) +} + +func BenchmarkRoundtripTxt(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + sampleString := string(sample) + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + b.StartTimer() + var result string + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sampleString[0:length] + rows := tb.checkRows(db.Query(`SELECT "` + test + `"`)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if result != test { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkRoundtripBin(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + stmt := tb.checkStmt(db.Prepare("SELECT ?")) + defer stmt.Close() + b.StartTimer() + var result sql.RawBytes + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sample[0:length] + rows := tb.checkRows(stmt.Query(test)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if !bytes.Equal(result, test) { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkInterpolation(b *testing.B) { + mc := &mysqlConn{ + cfg: &Config{ + InterpolateParams: true, + Loc: time.UTC, + }, + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + buf: newBuffer(nil), + } + + args := []driver.Value{ + int64(42424242), + float64(math.Pi), + false, + time.Unix(1423411542, 807015000), + []byte("bytes containing special chars ' \" \a \x00"), + "string containing special chars ' \" \a \x00", + } + q := "SELECT ?, ?, ?, ?, ?, ?" + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := mc.interpolateParams(q, args) + if err != nil { + b.Fatal(err) + } + } +} + +func benchmarkQueryContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + var got string + for pb.Next() { + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Fatalf("query = %q; want one", got) + } + } + }) +} + +func BenchmarkQueryContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +func benchmarkExecContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "DO 1")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := stmt.ExecContext(ctx); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkExecContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +// BenchmarkQueryRawBytes benchmarks fetching 100 blobs using sql.RawBytes. +// "size=" means size of each blobs. +func BenchmarkQueryRawBytes(b *testing.B) { + var sizes []int = []int{100, 1000, 2000, 4000, 8000, 12000, 16000, 32000, 64000, 256000} + db := initDB(b, + "DROP TABLE IF EXISTS bench_rawbytes", + "CREATE TABLE bench_rawbytes (id INT PRIMARY KEY, val LONGBLOB)", + ) + defer db.Close() + + blob := make([]byte, sizes[len(sizes)-1]) + for i := range blob { + blob[i] = 42 + } + for i := 0; i < 100; i++ { + _, err := db.Exec("INSERT INTO bench_rawbytes VALUES (?, ?)", i, blob) + if err != nil { + b.Fatal(err) + } + } + + for _, s := range sizes { + b.Run(fmt.Sprintf("size=%v", s), func(b *testing.B) { + db.SetMaxIdleConns(0) + db.SetMaxIdleConns(1) + b.ReportAllocs() + b.ResetTimer() + + for j := 0; j < b.N; j++ { + rows, err := db.Query("SELECT LEFT(val, ?) as v FROM bench_rawbytes", s) + if err != nil { + b.Fatal(err) + } + nrows := 0 + for rows.Next() { + var buf sql.RawBytes + err := rows.Scan(&buf) + if err != nil { + b.Fatal(err) + } + if len(buf) != s { + b.Fatalf("size mismatch: expected %v, got %v", s, len(buf)) + } + nrows++ + } + rows.Close() + if nrows != 100 { + b.Fatalf("numbers of rows mismatch: expected %v, got %v", 100, nrows) + } + } + }) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/go-sql-driver/mysql/buffer.go index 509ce89..0774c5c 100644 --- a/vendor/github.com/go-sql-driver/mysql/buffer.go +++ b/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -8,53 +8,86 @@ package mysql -import "io" +import ( + "io" + "net" + "time" +) const defaultBufSize = 4096 +const maxCachedBufSize = 256 * 1024 // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. +// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { - buf []byte - rd io.Reader - idx int - length int + buf []byte // buf is a byte buffer who's length and capacity are equal. + nc net.Conn + idx int + length int + timeout time.Duration + dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer + flipcnt uint // flipccnt is the current buffer counter for double-buffering } -func newBuffer(rd io.Reader) buffer { - var b [defaultBufSize]byte +// newBuffer allocates and returns a new buffer. +func newBuffer(nc net.Conn) buffer { + fg := make([]byte, defaultBufSize) return buffer{ - buf: b[:], - rd: rd, + buf: fg, + nc: nc, + dbuf: [2][]byte{fg, nil}, } } +// flip replaces the active buffer with the background buffer +// this is a delayed flip that simply increases the buffer counter; +// the actual flip will be performed the next time we call `buffer.fill` +func (b *buffer) flip() { + b.flipcnt += 1 +} + // fill reads into the buffer until at least _need_ bytes are in it func (b *buffer) fill(need int) error { n := b.length + // fill data into its double-buffering target: if we've called + // flip on this buffer, we'll be copying to the background buffer, + // and then filling it with network data; otherwise we'll just move + // the contents of the current buffer to the front before filling it + dest := b.dbuf[b.flipcnt&1] - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) - } - - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { + // grow buffer if necessary to fit the whole packet. + if need > len(dest) { // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf + dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) + + // if the allocated buffer is not too large, move it to backing storage + // to prevent extra allocations on applications that perform large reads + if len(dest) <= maxCachedBufSize { + b.dbuf[b.flipcnt&1] = dest + } } + // if we're filling the fg buffer, move the existing data to the start of it. + // if we're filling the bg buffer, copy over the data + if n > 0 { + copy(dest[:n], b.buf[b.idx:]) + } + + b.buf = dest b.idx = 0 for { - nn, err := b.rd.Read(b.buf[n:]) + if b.timeout > 0 { + if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil { + return err + } + } + + nn, err := b.nc.Read(b.buf[n:]) n += nn switch err { @@ -94,43 +127,56 @@ func (b *buffer) readNext(need int) ([]byte, error) { return b.buf[offset:b.idx], nil } -// returns a buffer with the requested size. +// takeBuffer returns a buffer with the requested size. // If possible, a slice from the existing buffer is returned. // Otherwise a bigger buffer is made. // Only one buffer (total) can be used at a time. -func (b *buffer) takeBuffer(length int) []byte { +func (b *buffer) takeBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } // test (cheap) general case first - if length <= defaultBufSize || length <= cap(b.buf) { - return b.buf[:length] + if length <= cap(b.buf) { + return b.buf[:length], nil } if length < maxPacketSize { b.buf = make([]byte, length) - return b.buf + return b.buf, nil } - return make([]byte, length) + + // buffer is larger than we want to store. + return make([]byte, length), nil } -// shortcut which can be used if the requested buffer is guaranteed to be -// smaller than defaultBufSize +// takeSmallBuffer is shortcut which can be used if length is +// known to be smaller than defaultBufSize. // Only one buffer (total) can be used at a time. -func (b *buffer) takeSmallBuffer(length int) []byte { - if b.length == 0 { - return b.buf[:length] +func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { + if b.length > 0 { + return nil, ErrBusyBuffer } - return nil + return b.buf[:length], nil } // takeCompleteBuffer returns the complete existing buffer. // This can be used if the necessary buffer size is unknown. +// cap and len of the returned buffer will be equal. // Only one buffer (total) can be used at a time. -func (b *buffer) takeCompleteBuffer() []byte { - if b.length == 0 { - return b.buf +func (b *buffer) takeCompleteBuffer() ([]byte, error) { + if b.length > 0 { + return nil, ErrBusyBuffer + } + return b.buf, nil +} + +// store stores buf, an updated buffer, if its suitable to do so. +func (b *buffer) store(buf []byte) error { + if b.length > 0 { + return ErrBusyBuffer + } else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) { + b.buf = buf[:cap(buf)] } return nil } diff --git a/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/go-sql-driver/mysql/collations.go index 6c1d613..8d2b556 100644 --- a/vendor/github.com/go-sql-driver/mysql/collations.go +++ b/vendor/github.com/go-sql-driver/mysql/collations.go @@ -8,182 +8,190 @@ package mysql -const defaultCollation byte = 33 // utf8_general_ci +const defaultCollation = "utf8mb4_general_ci" +const binaryCollation = "binary" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. +// +// ucs2, utf16, and utf32 can't be used for connection charset. +// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset +// They are commented out to reduce this map. var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + //"ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + //"utf16_general_ci": 54, + //"utf16_bin": 55, + //"utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + //"utf32_general_ci": 60, + //"utf32_bin": 61, + //"utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "utf8_tolower_ci": 76, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + //"ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + //"utf16_unicode_ci": 101, + //"utf16_icelandic_ci": 102, + //"utf16_latvian_ci": 103, + //"utf16_romanian_ci": 104, + //"utf16_slovenian_ci": 105, + //"utf16_polish_ci": 106, + //"utf16_estonian_ci": 107, + //"utf16_spanish_ci": 108, + //"utf16_swedish_ci": 109, + //"utf16_turkish_ci": 110, + //"utf16_czech_ci": 111, + //"utf16_danish_ci": 112, + //"utf16_lithuanian_ci": 113, + //"utf16_slovak_ci": 114, + //"utf16_spanish2_ci": 115, + //"utf16_roman_ci": 116, + //"utf16_persian_ci": 117, + //"utf16_esperanto_ci": 118, + //"utf16_hungarian_ci": 119, + //"utf16_sinhala_ci": 120, + //"utf16_german2_ci": 121, + //"utf16_croatian_ci": 122, + //"utf16_unicode_520_ci": 123, + //"utf16_vietnamese_ci": 124, + //"ucs2_unicode_ci": 128, + //"ucs2_icelandic_ci": 129, + //"ucs2_latvian_ci": 130, + //"ucs2_romanian_ci": 131, + //"ucs2_slovenian_ci": 132, + //"ucs2_polish_ci": 133, + //"ucs2_estonian_ci": 134, + //"ucs2_spanish_ci": 135, + //"ucs2_swedish_ci": 136, + //"ucs2_turkish_ci": 137, + //"ucs2_czech_ci": 138, + //"ucs2_danish_ci": 139, + //"ucs2_lithuanian_ci": 140, + //"ucs2_slovak_ci": 141, + //"ucs2_spanish2_ci": 142, + //"ucs2_roman_ci": 143, + //"ucs2_persian_ci": 144, + //"ucs2_esperanto_ci": 145, + //"ucs2_hungarian_ci": 146, + //"ucs2_sinhala_ci": 147, + //"ucs2_german2_ci": 148, + //"ucs2_croatian_ci": 149, + //"ucs2_unicode_520_ci": 150, + //"ucs2_vietnamese_ci": 151, + //"ucs2_general_mysql500_ci": 159, + //"utf32_unicode_ci": 160, + //"utf32_icelandic_ci": 161, + //"utf32_latvian_ci": 162, + //"utf32_romanian_ci": 163, + //"utf32_slovenian_ci": 164, + //"utf32_polish_ci": 165, + //"utf32_estonian_ci": 166, + //"utf32_spanish_ci": 167, + //"utf32_swedish_ci": 168, + //"utf32_turkish_ci": 169, + //"utf32_czech_ci": 170, + //"utf32_danish_ci": 171, + //"utf32_lithuanian_ci": 172, + //"utf32_slovak_ci": 173, + //"utf32_spanish2_ci": 174, + //"utf32_roman_ci": 175, + //"utf32_persian_ci": 176, + //"utf32_esperanto_ci": 177, + //"utf32_hungarian_ci": 178, + //"utf32_sinhala_ci": 179, + //"utf32_german2_ci": 180, + //"utf32_croatian_ci": 181, + //"utf32_unicode_520_ci": 182, + //"utf32_vietnamese_ci": 183, "utf8_unicode_ci": 192, "utf8_icelandic_ci": 193, "utf8_latvian_ci": 194, @@ -233,18 +241,25 @@ var collations = map[string]byte{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, + "gb18030_unicode_520_ci": 250, + "utf8mb4_0900_ai_ci": 255, } // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. -var unsafeCollations = map[byte]bool{ - 1: true, // big5_chinese_ci - 13: true, // sjis_japanese_ci - 28: true, // gbk_chinese_ci - 84: true, // big5_bin - 86: true, // gb2312_bin - 87: true, // gbk_bin - 88: true, // sjis_bin - 95: true, // cp932_japanese_ci - 96: true, // cp932_bin +var unsafeCollations = map[string]bool{ + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, + "gb18030_chinese_ci": true, + "gb18030_bin": true, + "gb18030_unicode_520_ci": true, } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/go-sql-driver/mysql/conncheck.go new file mode 100644 index 0000000..cc47aa5 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -0,0 +1,53 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows,!appengine + +package mysql + +import ( + "errors" + "io" + "net" + "syscall" +) + +var errUnexpectedRead = errors.New("unexpected read from socket") + +func connCheck(c net.Conn) error { + var ( + n int + err error + buff [1]byte + ) + + sconn, ok := c.(syscall.Conn) + if !ok { + return nil + } + rc, err := sconn.SyscallConn() + if err != nil { + return err + } + rerr := rc.Read(func(fd uintptr) bool { + n, err = syscall.Read(int(fd), buff[:]) + return true + }) + switch { + case rerr != nil: + return rerr + case n == 0 && err == nil: + return io.EOF + case n > 0: + return errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + return nil + default: + return err + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go new file mode 100644 index 0000000..fd01f64 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -0,0 +1,17 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows appengine + +package mysql + +import "net" + +func connCheck(c net.Conn) error { + return nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go new file mode 100644 index 0000000..b7234b0 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go @@ -0,0 +1,38 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10,!windows + +package mysql + +import ( + "testing" + "time" +) + +func TestStaleConnectionChecks(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("SET @@SESSION.wait_timeout = 2") + + if err := dbt.db.Ping(); err != nil { + dbt.Fatal(err) + } + + // wait for MySQL to close our connection + time.Sleep(3 * time.Second) + + tx, err := dbt.db.Begin() + if err != nil { + dbt.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + dbt.Fatal(err) + } + }) +} diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index 72ed09d..565a548 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -9,9 +9,10 @@ package mysql import ( - "crypto/tls" + "context" + "database/sql" "database/sql/driver" - "errors" + "io" "net" "strconv" "strings" @@ -21,40 +22,31 @@ import ( type mysqlConn struct { buf buffer netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. affectedRows uint64 insertId uint64 - cfg *config - maxPacketAllowed int + cfg *Config + maxAllowedPacket int maxWriteSize int + writeTimeout time.Duration flags clientFlag status statusFlag sequence uint8 parseTime bool - strict bool -} + reset bool // set when the Go SQL package calls ResetSession -type config struct { - user string - passwd string - net string - addr string - dbname string - params map[string]string - loc *time.Location - tls *tls.Config - timeout time.Duration - collation uint8 - allowAllFiles bool - allowOldPasswords bool - allowCleartextPasswords bool - clientFoundRows bool - columnsWithAlias bool - interpolateParams bool + // for context support (Go 1.8+) + watching bool + watcher chan<- context.Context + closech chan struct{} + finished chan<- struct{} + canceled atomicError // set non-nil if conn is canceled + closed atomicBool // set when conn is closed, before closech is closed } // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { - for param, val := range mc.cfg.params { + for param, val := range mc.cfg.Params { switch param { // Charset case "charset": @@ -70,27 +62,6 @@ func (mc *mysqlConn) handleParams() (err error) { return } - // time.Time parsing - case "parseTime": - var isBool bool - mc.parseTime, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Strict mode - case "strict": - var isBool bool - mc.strict, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Compression - case "compress": - err = errors.New("Compression not implemented yet") - return - // System Vars default: err = mc.exec("SET " + param + "=" + val + "") @@ -103,46 +74,87 @@ func (mc *mysqlConn) handleParams() (err error) { return } +func (mc *mysqlConn) markBadConn(err error) error { + if mc == nil { + return err + } + if err != errBadConnNoWrite { + return err + } + return driver.ErrBadConn +} + func (mc *mysqlConn) Begin() (driver.Tx, error) { - if mc.netConn == nil { + return mc.begin(false) +} + +func (mc *mysqlConn) begin(readOnly bool) (driver.Tx, error) { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } - err := mc.exec("START TRANSACTION") + var q string + if readOnly { + q = "START TRANSACTION READ ONLY" + } else { + q = "START TRANSACTION" + } + err := mc.exec(q) if err == nil { return &mysqlTx{mc}, err } - - return nil, err + return nil, mc.markBadConn(err) } func (mc *mysqlConn) Close() (err error) { // Makes Close idempotent - if mc.netConn != nil { + if !mc.closed.IsSet() { err = mc.writeCommandPacket(comQuit) - if err == nil { - err = mc.netConn.Close() - } else { - mc.netConn.Close() - } - mc.netConn = nil } - mc.cfg = nil - mc.buf.rd = nil + mc.cleanup() return } -func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { +// Closes the network connection and unsets internal variables. Do not call this +// function after successfully authentication, call Close instead. This function +// is called before auth or on auth failure because MySQL will have already +// closed the network connection. +func (mc *mysqlConn) cleanup() { + if !mc.closed.TrySet(true) { + return + } + + // Makes cleanup idempotent + close(mc.closech) if mc.netConn == nil { + return + } + if err := mc.netConn.Close(); err != nil { + errLog.Print(err) + } +} + +func (mc *mysqlConn) error() error { + if mc.closed.IsSet() { + if err := mc.canceled.Value(); err != nil { + return err + } + return ErrInvalidConn + } + return nil +} + +func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, err + return nil, mc.markBadConn(err) } stmt := &mysqlStmt{ @@ -167,11 +179,16 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { } func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) { - buf := mc.buf.takeCompleteBuffer() - if buf == nil { + // Number of ? should be same to len(args) + if strings.Count(query, "?") != len(args) { + return "", driver.ErrSkip + } + + buf, err := mc.buf.takeCompleteBuffer() + if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return "", driver.ErrBadConn + errLog.Print(err) + return "", ErrInvalidConn } buf = buf[:0] argPos := 0 @@ -196,6 +213,9 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin switch v := arg.(type) { case int64: buf = strconv.AppendInt(buf, v, 10) + case uint64: + // Handle uint64 explicitly because our custom ConvertValue emits unsigned values + buf = strconv.AppendUint(buf, v, 10) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: @@ -208,7 +228,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { - v := v.In(mc.cfg.loc) + v := v.In(mc.cfg.Loc) v = v.Add(time.Nanosecond * 500) // To round under microsecond year := v.Year() year100 := year / 100 @@ -273,7 +293,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - if len(buf)+4 > mc.maxPacketAllowed { + if len(buf)+4 > mc.maxAllowedPacket { return "", driver.ErrSkip } } @@ -284,12 +304,12 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin } func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) { - if mc.netConn == nil { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement @@ -298,7 +318,6 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err return nil, err } query = prepared - args = nil } mc.affectedRows = 0 mc.insertId = 0 @@ -310,37 +329,48 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err insertId: int64(mc.insertId), }, err } - return nil, err + return nil, mc.markBadConn(err) } // Internal function to execute commands func (mc *mysqlConn) exec(query string) error { // Send command - err := mc.writeCommandPacketStr(comQuery, query) - if err != nil { - return err + if err := mc.writeCommandPacketStr(comQuery, query); err != nil { + return mc.markBadConn(err) } // Read Result resLen, err := mc.readResultSetHeaderPacket() - if err == nil && resLen > 0 { - if err = mc.readUntilEOF(); err != nil { + if err != nil { + return err + } + + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { return err } - err = mc.readUntilEOF() + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } } - return err + return mc.discardResults() } func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) { - if mc.netConn == nil { + return mc.query(query, args) +} + +func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try client-side prepare to reduce roundtrip @@ -349,7 +379,6 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro return nil, err } query = prepared - args = nil } // Send command err := mc.writeCommandPacketStr(comQuery, query) @@ -362,15 +391,22 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro rows.mc = mc if resLen == 0 { - // no columns, no more data - return emptyRows{}, nil + rows.rs.done = true + + switch err := rows.NextResultSet(); err { + case nil, io.EOF: + return rows, nil + default: + return nil, err + } } + // Columns - rows.columns, err = mc.readColumns(resLen) + rows.rs.columns, err = mc.readColumns(resLen) return rows, err } } - return nil, err + return nil, mc.markBadConn(err) } // Gets the value of the given MySQL System Variable @@ -386,6 +422,7 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { if err == nil { rows := new(textRows) rows.mc = mc + rows.rs.columns = []mysqlField{{fieldType: fieldTypeVarChar}} if resLen > 0 { // Columns @@ -401,3 +438,212 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { } return nil, err } + +// finish is called when the query has canceled. +func (mc *mysqlConn) cancel(err error) { + mc.canceled.Set(err) + mc.cleanup() +} + +// finish is called when the query has succeeded. +func (mc *mysqlConn) finish() { + if !mc.watching || mc.finished == nil { + return + } + select { + case mc.finished <- struct{}{}: + mc.watching = false + case <-mc.closech: + } +} + +// Ping implements driver.Pinger interface +func (mc *mysqlConn) Ping(ctx context.Context) (err error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return driver.ErrBadConn + } + + if err = mc.watchCancel(ctx); err != nil { + return + } + defer mc.finish() + + if err = mc.writeCommandPacket(comPing); err != nil { + return mc.markBadConn(err) + } + + return mc.readResultOK() +} + +// BeginTx implements driver.ConnBeginTx interface +func (mc *mysqlConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { + level, err := mapIsolationLevel(opts.Isolation) + if err != nil { + return nil, err + } + err = mc.exec("SET TRANSACTION ISOLATION LEVEL " + level) + if err != nil { + return nil, err + } + } + + return mc.begin(opts.ReadOnly) +} + +func (mc *mysqlConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := mc.query(query, dargs) + if err != nil { + mc.finish() + return nil, err + } + rows.finish = mc.finish + return rows, err +} + +func (mc *mysqlConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + return mc.Exec(query, dargs) +} + +func (mc *mysqlConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + stmt, err := mc.Prepare(query) + mc.finish() + if err != nil { + return nil, err + } + + select { + default: + case <-ctx.Done(): + stmt.Close() + return nil, ctx.Err() + } + return stmt, nil +} + +func (stmt *mysqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := stmt.query(dargs) + if err != nil { + stmt.mc.finish() + return nil, err + } + rows.finish = stmt.mc.finish + return rows, err +} + +func (stmt *mysqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + defer stmt.mc.finish() + + return stmt.Exec(dargs) +} + +func (mc *mysqlConn) watchCancel(ctx context.Context) error { + if mc.watching { + // Reach here if canceled, + // so the connection is already invalid + mc.cleanup() + return nil + } + // When ctx is already cancelled, don't watch it. + if err := ctx.Err(); err != nil { + return err + } + // When ctx is not cancellable, don't watch it. + if ctx.Done() == nil { + return nil + } + // When watcher is not alive, can't watch it. + if mc.watcher == nil { + return nil + } + + mc.watching = true + mc.watcher <- ctx + return nil +} + +func (mc *mysqlConn) startWatcher() { + watcher := make(chan context.Context, 1) + mc.watcher = watcher + finished := make(chan struct{}) + mc.finished = finished + go func() { + for { + var ctx context.Context + select { + case ctx = <-watcher: + case <-mc.closech: + return + } + + select { + case <-ctx.Done(): + mc.cancel(ctx.Err()) + case <-finished: + case <-mc.closech: + return + } + } + }() +} + +func (mc *mysqlConn) CheckNamedValue(nv *driver.NamedValue) (err error) { + nv.Value, err = converter{}.ConvertValue(nv.Value) + return +} + +// ResetSession implements driver.SessionResetter. +// (From Go 1.10) +func (mc *mysqlConn) ResetSession(ctx context.Context) error { + if mc.closed.IsSet() { + return driver.ErrBadConn + } + mc.reset = true + return nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/connection_test.go b/vendor/github.com/go-sql-driver/mysql/connection_test.go new file mode 100644 index 0000000..19c17ff --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connection_test.go @@ -0,0 +1,175 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "errors" + "net" + "testing" +) + +func TestInterpolateParams(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42), "gopher"}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT 42+'gopher'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + +func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42)}) + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +// We don't support placeholder in string literal for now. +// https://github.com/go-sql-driver/mysql/pull/490 +func TestInterpolateParamsPlaceholderInString(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT 'abc?xyz',?", []driver.Value{int64(42)}) + // When InterpolateParams support string literal, this should return `"SELECT 'abc?xyz', 42` + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +func TestInterpolateParamsUint64(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?", []driver.Value{uint64(42)}) + if err != nil { + t.Errorf("Expected err=nil, got err=%#v, q=%#v", err, q) + } + if q != "SELECT 42" { + t.Errorf("Expected uint64 interpolation to work, got q=%#v", q) + } +} + +func TestCheckNamedValue(t *testing.T) { + value := driver.NamedValue{Value: ^uint64(0)} + x := &mysqlConn{} + err := x.CheckNamedValue(&value) + + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if value.Value != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", value.Value, value.Value) + } +} + +// TestCleanCancel tests passed context is cancelled at start. +// No packet should be sent. Connection should keep current status. +func TestCleanCancel(t *testing.T) { + mc := &mysqlConn{ + closech: make(chan struct{}), + } + mc.startWatcher() + defer mc.cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + for i := 0; i < 3; i++ { // Repeat same behavior + err := mc.Ping(ctx) + if err != context.Canceled { + t.Errorf("expected context.Canceled, got %#v", err) + } + + if mc.closed.IsSet() { + t.Error("expected mc is not closed, closed actually") + } + + if mc.watching { + t.Error("expected watching is false, but true") + } + } +} + +func TestPingMarkBadConnection(t *testing.T) { + nc := badConnection{err: errors.New("boom")} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + } + + err := ms.Ping(context.Background()) + + if err != driver.ErrBadConn { + t.Errorf("expected driver.ErrBadConn, got %#v", err) + } +} + +func TestPingErrInvalidConn(t *testing.T) { + nc := badConnection{err: errors.New("failed to write"), n: 10} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + closech: make(chan struct{}), + } + + err := ms.Ping(context.Background()) + + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %#v", err) + } +} + +type badConnection struct { + n int + err error + net.Conn +} + +func (bc badConnection) Write(b []byte) (n int, err error) { + return bc.n, bc.err +} + +func (bc badConnection) Close() error { + return nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/go-sql-driver/mysql/connector.go new file mode 100644 index 0000000..5aaaba4 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connector.go @@ -0,0 +1,143 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "net" +) + +type connector struct { + cfg *Config // immutable private copy. +} + +// Connect implements driver.Connector interface. +// Connect returns a connection to the database. +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + cfg: c.cfg, + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + mc.netConn, err = dial(ctx, mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + } + + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Temporary() { + errLog.Print("net.Error from Dial()': ", nerr.Error()) + return nil, driver.ErrBadConn + } + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +// Driver implements driver.Connector interface. +// Driver returns &MySQLDriver{}. +func (c *connector) Driver() driver.Driver { + return &MySQLDriver{} +} diff --git a/vendor/github.com/go-sql-driver/mysql/const.go b/vendor/github.com/go-sql-driver/mysql/const.go index dddc129..b1e6b85 100644 --- a/vendor/github.com/go-sql-driver/mysql/const.go +++ b/vendor/github.com/go-sql-driver/mysql/const.go @@ -9,7 +9,9 @@ package mysql const ( - minProtocolVersion byte = 10 + defaultAuthPlugin = "mysql_native_password" + defaultMaxAllowedPacket = 4 << 20 // 4 MiB + minProtocolVersion = 10 maxPacketSize = 1<<24 - 1 timeFormat = "2006-01-02 15:04:05.999999" ) @@ -18,10 +20,11 @@ const ( // http://dev.mysql.com/doc/internals/en/client-server-protocol.html const ( - iOK byte = 0x00 - iLocalInFile byte = 0xfb - iEOF byte = 0xfe - iERR byte = 0xff + iOK byte = 0x00 + iAuthMoreData byte = 0x01 + iLocalInFile byte = 0xfb + iEOF byte = 0xfe + iERR byte = 0xff ) // https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags @@ -87,8 +90,10 @@ const ( ) // https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnType +type fieldType byte + const ( - fieldTypeDecimal byte = iota + fieldTypeDecimal fieldType = iota fieldTypeTiny fieldTypeShort fieldTypeLong @@ -107,7 +112,8 @@ const ( fieldTypeBit ) const ( - fieldTypeNewDecimal byte = iota + 0xf6 + fieldTypeJSON fieldType = iota + 0xf5 + fieldTypeNewDecimal fieldTypeEnum fieldTypeSet fieldTypeTinyBLOB @@ -160,3 +166,9 @@ const ( statusInTransReadonly statusSessionStateChanged ) + +const ( + cachingSha2PasswordRequestPublicKey = 2 + cachingSha2PasswordFastAuthSuccess = 3 + cachingSha2PasswordPerformFullAuthentication = 4 +) diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index d310624..1f9decf 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// Package mysql provides a MySQL driver for Go's database/sql package. // // The driver should be used via the database/sql package: // @@ -17,131 +17,67 @@ package mysql import ( + "context" "database/sql" "database/sql/driver" "net" + "sync" ) -// This struct is exported to make the driver directly accessible. +// MySQLDriver is exported to make the driver directly accessible. // In general the driver is used via the database/sql package. type MySQLDriver struct{} // DialFunc is a function which can be used to establish the network connection. // Custom dial functions must be registered with RegisterDial +// +// Deprecated: users should register a DialContextFunc instead type DialFunc func(addr string) (net.Conn, error) -var dials map[string]DialFunc +// DialContextFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDialContext +type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error) -// RegisterDial registers a custom dial function. It can then be used by the +var ( + dialsLock sync.RWMutex + dials map[string]DialContextFunc +) + +// RegisterDialContext registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { +// The current context for the connection and its address is passed to the dial function. +func RegisterDialContext(net string, dial DialContextFunc) { + dialsLock.Lock() + defer dialsLock.Unlock() if dials == nil { - dials = make(map[string]DialFunc) + dials = make(map[string]DialContextFunc) } dials[net] = dial } +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +// +// Deprecated: users should call RegisterDialContext instead +func RegisterDial(network string, dial DialFunc) { + RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) { + return dial(addr) + }) +} + // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how -// the DSN string is formated +// the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxPacketAllowed: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - } - mc.cfg, err = parseDSN(dsn) + cfg, err := ParseDSN(dsn) if err != nil { return nil, err } - - // Connect to Server - if dial, ok := dials[mc.cfg.net]; ok { - mc.netConn, err = dial(mc.cfg.addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.timeout} - mc.netConn, err = nd.Dial(mc.cfg.net, mc.cfg.addr) + c := &connector{ + cfg: cfg, } - if err != nil { - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - mc.buf = newBuffer(mc.netConn) - - // Reading Handshake Initialization Packet - cipher, err := mc.readInitPacket() - if err != nil { - mc.Close() - return nil, err - } - - // Send Client Authentication Packet - if err = mc.writeAuthPacket(cipher); err != nil { - mc.Close() - return nil, err - } - - // Read Result Packet - err = mc.readResultOK() - if err != nil { - // Retry with old authentication method, if allowed - if mc.cfg != nil && mc.cfg.allowOldPasswords && err == ErrOldPassword { - if err = mc.writeOldAuthPacket(cipher); err != nil { - mc.Close() - return nil, err - } - if err = mc.readResultOK(); err != nil { - mc.Close() - return nil, err - } - } else if mc.cfg != nil && mc.cfg.allowCleartextPasswords && err == ErrCleartextPassword { - if err = mc.writeClearAuthPacket(); err != nil { - mc.Close() - return nil, err - } - if err = mc.readResultOK(); err != nil { - mc.Close() - return nil, err - } - } else { - mc.Close() - return nil, err - } - - } - - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxPacketAllowed = stringToInt(maxap) - 1 - if mc.maxPacketAllowed < maxPacketSize { - mc.maxWriteSize = mc.maxPacketAllowed - } - - // Handle DSN Params - err = mc.handleParams() - if err != nil { - mc.Close() - return nil, err - } - - return mc, nil + return c.Connect(context.Background()) } func init() { diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110.go b/vendor/github.com/go-sql-driver/mysql/driver_go110.go new file mode 100644 index 0000000..eb5a8fe --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/driver_go110.go @@ -0,0 +1,37 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "database/sql/driver" +) + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go b/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go new file mode 100644 index 0000000..19a0e59 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go @@ -0,0 +1,137 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "net" + "testing" + "time" +) + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_test.go b/vendor/github.com/go-sql-driver/mysql/driver_test.go new file mode 100644 index 0000000..3dee1ba --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/driver_test.go @@ -0,0 +1,2996 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "crypto/tls" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "net" + "net/url" + "os" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Ensure that all the driver interfaces are implemented +var ( + _ driver.Rows = &binaryRows{} + _ driver.Rows = &textRows{} +) + +var ( + user string + pass string + prot string + addr string + dbname string + dsn string + netAddr string + available bool +) + +var ( + tDate = time.Date(2012, 6, 14, 0, 0, 0, 0, time.UTC) + sDate = "2012-06-14" + tDateTime = time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC) + sDateTime = "2011-11-20 21:27:37" + tDate0 = time.Time{} + sDate0 = "0000-00-00" + sDateTime0 = "0000-00-00 00:00:00" +) + +// See https://github.com/go-sql-driver/mysql/wiki/Testing +func init() { + // get environment variables + env := func(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue + } + user = env("MYSQL_TEST_USER", "root") + pass = env("MYSQL_TEST_PASS", "") + prot = env("MYSQL_TEST_PROT", "tcp") + addr = env("MYSQL_TEST_ADDR", "localhost:3306") + dbname = env("MYSQL_TEST_DBNAME", "gotest") + netAddr = fmt.Sprintf("%s(%s)", prot, addr) + dsn = fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, pass, netAddr, dbname) + c, err := net.Dial(prot, addr) + if err == nil { + available = true + c.Close() + } +} + +type DBTest struct { + *testing.T + db *sql.DB +} + +type netErrorMock struct { + temporary bool + timeout bool +} + +func (e netErrorMock) Temporary() bool { + return e.temporary +} + +func (e netErrorMock) Timeout() bool { + return e.timeout +} + +func (e netErrorMock) Error() string { + return fmt.Sprintf("mock net error. Temporary: %v, Timeout %v", e.temporary, e.timeout) +} + +func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn += "&multiStatements=true" + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + } +} + +func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + db.Exec("DROP TABLE IF EXISTS test") + + dsn2 := dsn + "&interpolateParams=true" + var db2 *sql.DB + if _, err := ParseDSN(dsn2); err != errInvalidDSNUnsafeCollation { + db2, err = sql.Open("mysql", dsn2) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db2.Close() + } + + dsn3 := dsn + "&multiStatements=true" + var db3 *sql.DB + if _, err := ParseDSN(dsn3); err != errInvalidDSNUnsafeCollation { + db3, err = sql.Open("mysql", dsn3) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db3.Close() + } + + dbt := &DBTest{t, db} + dbt2 := &DBTest{t, db2} + dbt3 := &DBTest{t, db3} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + if db2 != nil { + test(dbt2) + dbt2.db.Exec("DROP TABLE IF EXISTS test") + } + if db3 != nil { + test(dbt3) + dbt3.db.Exec("DROP TABLE IF EXISTS test") + } + } +} + +func (dbt *DBTest) fail(method, query string, err error) { + if len(query) > 300 { + query = "[query too large to print]" + } + dbt.Fatalf("error on %s %s: %s", method, query, err.Error()) +} + +func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) { + res, err := dbt.db.Exec(query, args...) + if err != nil { + dbt.fail("exec", query, err) + } + return res +} + +func (dbt *DBTest) mustQuery(query string, args ...interface{}) (rows *sql.Rows) { + rows, err := dbt.db.Query(query, args...) + if err != nil { + dbt.fail("query", query, err) + } + return rows +} + +func maybeSkip(t *testing.T, err error, skipErrno uint16) { + mySQLErr, ok := err.(*MySQLError) + if !ok { + return + } + + if mySQLErr.Number == skipErrno { + t.Skipf("skipping test for error: %v", err) + } +} + +func TestEmptyQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // just a comment, no query + rows := dbt.mustQuery("--") + defer rows.Close() + // will hang before #255 + if rows.Next() { + dbt.Errorf("next on rows must be false") + } + }) +} + +func TestCRUD(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + + // Test for unexpected data + var out bool + rows := dbt.mustQuery("SELECT * FROM test") + if rows.Next() { + dbt.Error("unexpected data in empty table") + } + rows.Close() + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + id, err := res.LastInsertId() + if err != nil { + dbt.Fatalf("res.LastInsertId() returned error: %s", err.Error()) + } + if id != 0 { + dbt.Fatalf("expected InsertId 0, got %d", id) + } + + // Read + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if true != out { + dbt.Errorf("true != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Update + res = dbt.mustExec("UPDATE test SET value = ? WHERE value = ?", false, true) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check Update + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if false != out { + dbt.Errorf("false != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Delete + res = dbt.mustExec("DELETE FROM test WHERE value = ?", false) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check for unexpected rows + res = dbt.mustExec("DELETE FROM test") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 0 { + dbt.Fatalf("expected 0 affected row, got %d", count) + } + }) +} + +func TestMultiQuery(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE `test` (`id` int(11) NOT NULL, `value` int(11) NOT NULL) ") + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1, 1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Update + res = dbt.mustExec("UPDATE test SET value = 3 WHERE id = 1; UPDATE test SET value = 4 WHERE id = 1; UPDATE test SET value = 5 WHERE id = 1;") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Read + var out int + rows := dbt.mustQuery("SELECT value FROM test WHERE id=1;") + if rows.Next() { + rows.Scan(&out) + if 5 != out { + dbt.Errorf("5 != %d", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + }) +} + +func TestInt(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [5]string{"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT"} + in := int64(42) + var out int64 + var rows *sql.Rows + + // SIGNED + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // UNSIGNED ZEROFILL + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + " ZEROFILL)") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s ZEROFILL: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s ZEROFILL: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat32(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + in := float32(42.23) + var out float32 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %g != %g", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (42.23)") + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64Placeholder(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (id int, value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (1, 42.23)") + rows = dbt.mustQuery("SELECT value FROM test WHERE id = ?", 1) + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestString(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [6]string{"CHAR(255)", "VARCHAR(255)", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"} + in := "κόσμε üöäßñóùéàâÿœ'îë Árvíztűrő いろはにほへとちりぬるを イロハニホヘト דג סקרן чащах น่าฟังเอย" + var out string + var rows *sql.Rows + + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ") CHARACTER SET utf8") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %s != %s", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // BLOB + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + + id := 2 + in = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. " + + "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." + dbt.mustExec("INSERT INTO test VALUES (?, ?)", id, in) + + err := dbt.db.QueryRow("SELECT value FROM test WHERE id = ?", id).Scan(&out) + if err != nil { + dbt.Fatalf("Error on BLOB-Query: %s", err.Error()) + } else if out != in { + dbt.Errorf("BLOB: %s != %s", in, out) + } + }) +} + +func TestRawBytes(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + v1 := []byte("aaa") + v2 := []byte("bbb") + rows := dbt.mustQuery("SELECT ?, ?", v1, v2) + defer rows.Close() + if rows.Next() { + var o1, o2 sql.RawBytes + if err := rows.Scan(&o1, &o2); err != nil { + dbt.Errorf("Got error: %v", err) + } + if !bytes.Equal(v1, o1) { + dbt.Errorf("expected %v, got %v", v1, o1) + } + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + // https://github.com/go-sql-driver/mysql/issues/765 + // Appending to RawBytes shouldn't overwrite next RawBytes. + o1 = append(o1, "xyzzy"...) + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + } else { + dbt.Errorf("no data") + } + }) +} + +type testValuer struct { + value string +} + +func (tv testValuer) Value() (driver.Value, error) { + return tv.value, nil +} + +func TestValuer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuer{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE test (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + }) +} + +type testValuerWithValidation struct { + value string +} + +func (tv testValuerWithValidation) Value() (driver.Value, error) { + if len(tv.value) == 0 { + return nil, fmt.Errorf("Invalid string valuer. Value must not be empty") + } + + return tv.value, nil +} + +func TestValuerWithValidation(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuerWithValidation{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE testValuer (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO testValuer VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM testValuer") + defer rows.Close() + + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", testValuerWithValidation{""}); err == nil { + dbt.Errorf("Failed to check valuer error") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", nil); err != nil { + dbt.Errorf("Failed to check nil") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", map[string]bool{}); err == nil { + dbt.Errorf("Failed to check not valuer") + } + + dbt.mustExec("DROP TABLE IF EXISTS testValuer") + }) +} + +type timeTests struct { + dbtype string + tlayout string + tests []timeTest +} + +type timeTest struct { + s string // leading "!": do not use t as value in queries + t time.Time +} + +type timeMode byte + +func (t timeMode) String() string { + switch t { + case binaryString: + return "binary:string" + case binaryTime: + return "binary:time.Time" + case textString: + return "text:string" + } + panic("unsupported timeMode") +} + +func (t timeMode) Binary() bool { + switch t { + case binaryString, binaryTime: + return true + } + return false +} + +const ( + binaryString timeMode = iota + binaryTime + textString +) + +func (t timeTest) genQuery(dbtype string, mode timeMode) string { + var inner string + if mode.Binary() { + inner = "?" + } else { + inner = `"%s"` + } + return `SELECT cast(` + inner + ` as ` + dbtype + `)` +} + +func (t timeTest) run(dbt *DBTest, dbtype, tlayout string, mode timeMode) { + var rows *sql.Rows + query := t.genQuery(dbtype, mode) + switch mode { + case binaryString: + rows = dbt.mustQuery(query, t.s) + case binaryTime: + rows = dbt.mustQuery(query, t.t) + case textString: + query = fmt.Sprintf(query, t.s) + rows = dbt.mustQuery(query) + default: + panic("unsupported mode") + } + defer rows.Close() + var err error + if !rows.Next() { + err = rows.Err() + if err == nil { + err = fmt.Errorf("no data") + } + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + var dst interface{} + err = rows.Scan(&dst) + if err != nil { + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + switch val := dst.(type) { + case []uint8: + str := string(val) + if str == t.s { + return + } + if mode.Binary() && dbtype == "DATETIME" && len(str) == 26 && str[:19] == t.s { + // a fix mainly for TravisCI: + // accept full microsecond resolution in result for DATETIME columns + // where the binary protocol was used + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, str, + ) + case time.Time: + if val == t.t { + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, val.Format(tlayout), + ) + default: + fmt.Printf("%#v\n", []interface{}{dbtype, tlayout, mode, t.s, t.t}) + dbt.Errorf("%s [%s]: unhandled type %T (is '%v')", + dbtype, mode, + val, val, + ) + } +} + +func TestDateTime(t *testing.T) { + afterTime := func(t time.Time, d string) time.Time { + dur, err := time.ParseDuration(d) + if err != nil { + panic(err) + } + return t.Add(dur) + } + // NOTE: MySQL rounds DATETIME(x) up - but that's not included in the tests + format := "2006-01-02 15:04:05.999999" + t0 := time.Time{} + tstr0 := "0000-00-00 00:00:00.000000" + testcases := []timeTests{ + {"DATE", format[:10], []timeTest{ + {t: time.Date(2011, 11, 20, 0, 0, 0, 0, time.UTC)}, + {t: t0, s: tstr0[:10]}, + }}, + {"DATETIME", format[:19], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(0)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(1)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 100000000, time.UTC)}, + {t: t0, s: tstr0[:21]}, + }}, + {"DATETIME(6)", format, []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 123456000, time.UTC)}, + {t: t0, s: tstr0}, + }}, + {"TIME", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(0)", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(1)", format[11:21], []timeTest{ + {t: afterTime(t0, "12345600ms")}, + {s: "!-12:34:56.7"}, + {s: "!-838:59:58.9"}, + {s: "!838:59:58.9"}, + {t: t0, s: tstr0[11:21]}, + }}, + {"TIME(6)", format[11:], []timeTest{ + {t: afterTime(t0, "1234567890123000ns")}, + {s: "!-12:34:56.789012"}, + {s: "!-838:59:58.999999"}, + {s: "!838:59:58.999999"}, + {t: t0, s: tstr0[11:]}, + }}, + } + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + microsecsSupported := false + zeroDateSupported := false + var rows *sql.Rows + var err error + rows, err = dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`) + if err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + rows, err = dbt.db.Query(`SELECT cast("0000-00-00" as DATE) = "0000-00-00"`) + if err == nil { + rows.Scan(&zeroDateSupported) + rows.Close() + } + for _, setups := range testcases { + if t := setups.dbtype; !microsecsSupported && t[len(t)-1:] == ")" { + // skip fractional second tests if unsupported by server + continue + } + for _, setup := range setups.tests { + allowBinTime := true + if setup.s == "" { + // fill time string wherever Go can reliable produce it + setup.s = setup.t.Format(setups.tlayout) + } else if setup.s[0] == '!' { + // skip tests using setup.t as source in queries + allowBinTime = false + // fix setup.s - remove the "!" + setup.s = setup.s[1:] + } + if !zeroDateSupported && setup.s == tstr0[:len(setup.s)] { + // skip disallowed 0000-00-00 date + continue + } + setup.run(dbt, setups.dbtype, setups.tlayout, textString) + setup.run(dbt, setups.dbtype, setups.tlayout, binaryString) + if allowBinTime { + setup.run(dbt, setups.dbtype, setups.tlayout, binaryTime) + } + } + } + }) + } +} + +func TestTimestampMicros(t *testing.T) { + format := "2006-01-02 15:04:05.999999" + f0 := format[:19] + f1 := format[:21] + f6 := format[:26] + runTests(t, dsn, func(dbt *DBTest) { + // check if microseconds are supported. + // Do not use timestamp(x) for that check - before 5.5.6, x would mean display width + // and not precision. + // Se last paragraph at http://dev.mysql.com/doc/refman/5.6/en/fractional-seconds.html + microsecsSupported := false + if rows, err := dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`); err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + if !microsecsSupported { + // skip test + return + } + _, err := dbt.db.Exec(` + CREATE TABLE test ( + value0 TIMESTAMP NOT NULL DEFAULT '` + f0 + `', + value1 TIMESTAMP(1) NOT NULL DEFAULT '` + f1 + `', + value6 TIMESTAMP(6) NOT NULL DEFAULT '` + f6 + `' + )`, + ) + if err != nil { + dbt.Error(err) + } + defer dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("INSERT INTO test SET value0=?, value1=?, value6=?", f0, f1, f6) + var res0, res1, res6 string + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Errorf("test contained no selectable values") + } + err = rows.Scan(&res0, &res1, &res6) + if err != nil { + dbt.Error(err) + } + if res0 != f0 { + dbt.Errorf("expected %q, got %q", f0, res0) + } + if res1 != f1 { + dbt.Errorf("expected %q, got %q", f1, res1) + } + if res6 != f6 { + dbt.Errorf("expected %q, got %q", f6, res6) + } + }) +} + +func TestNULL(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + nullStmt, err := dbt.db.Prepare("SELECT NULL") + if err != nil { + dbt.Fatal(err) + } + defer nullStmt.Close() + + nonNullStmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + defer nonNullStmt.Close() + + // NullBool + var nb sql.NullBool + // Invalid + if err = nullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if nb.Valid { + dbt.Error("valid NullBool which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if !nb.Valid { + dbt.Error("invalid NullBool which should be valid") + } else if nb.Bool != true { + dbt.Errorf("Unexpected NullBool value: %t (should be true)", nb.Bool) + } + + // NullFloat64 + var nf sql.NullFloat64 + // Invalid + if err = nullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if nf.Valid { + dbt.Error("valid NullFloat64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if !nf.Valid { + dbt.Error("invalid NullFloat64 which should be valid") + } else if nf.Float64 != float64(1) { + dbt.Errorf("unexpected NullFloat64 value: %f (should be 1.0)", nf.Float64) + } + + // NullInt64 + var ni sql.NullInt64 + // Invalid + if err = nullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if ni.Valid { + dbt.Error("valid NullInt64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if !ni.Valid { + dbt.Error("invalid NullInt64 which should be valid") + } else if ni.Int64 != int64(1) { + dbt.Errorf("unexpected NullInt64 value: %d (should be 1)", ni.Int64) + } + + // NullString + var ns sql.NullString + // Invalid + if err = nullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if ns.Valid { + dbt.Error("valid NullString which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if !ns.Valid { + dbt.Error("invalid NullString which should be valid") + } else if ns.String != `1` { + dbt.Error("unexpected NullString value:" + ns.String + " (should be `1`)") + } + + // nil-bytes + var b []byte + // Read nil + if err = nullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil []byte which should be nil") + } + // Read non-nil + if err = nonNullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil []byte which should be non-nil") + } + // Insert nil + b = nil + success := false + if err = dbt.db.QueryRow("SELECT ? IS NULL", b).Scan(&success); err != nil { + dbt.Fatal(err) + } + if !success { + dbt.Error("inserting []byte(nil) as NULL failed") + } + // Check input==output with input==nil + b = nil + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil echo from nil input") + } + // Check input==output with input!=nil + b = []byte("") + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil echo from non-nil input") + } + + // Insert NULL + dbt.mustExec("CREATE TABLE test (dummmy1 int, value int, dummy2 int)") + + dbt.mustExec("INSERT INTO test VALUES (?, ?, ?)", 1, nil, 2) + + var out interface{} + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if out != nil { + dbt.Errorf("%v != nil", out) + } + } else { + dbt.Error("no data") + } + }) +} + +func TestUint64(t *testing.T) { + const ( + u0 = uint64(0) + uall = ^u0 + uhigh = uall >> 1 + utop = ^uhigh + s0 = int64(0) + sall = ^s0 + shigh = int64(uhigh) + stop = ^shigh + ) + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare(`SELECT ?, ?, ? ,?, ?, ?, ?, ?`) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + row := stmt.QueryRow( + u0, uhigh, utop, uall, + s0, shigh, stop, sall, + ) + + var ua, ub, uc, ud uint64 + var sa, sb, sc, sd int64 + + err = row.Scan(&ua, &ub, &uc, &ud, &sa, &sb, &sc, &sd) + if err != nil { + dbt.Fatal(err) + } + switch { + case ua != u0, + ub != uhigh, + uc != utop, + ud != uall, + sa != s0, + sb != shigh, + sc != stop, + sd != sall: + dbt.Fatal("unexpected result value") + } + }) +} + +func TestLongData(t *testing.T) { + runTests(t, dsn+"&maxAllowedPacket=0", func(dbt *DBTest) { + var maxAllowedPacketSize int + err := dbt.db.QueryRow("select @@max_allowed_packet").Scan(&maxAllowedPacketSize) + if err != nil { + dbt.Fatal(err) + } + maxAllowedPacketSize-- + + // don't get too ambitious + if maxAllowedPacketSize > 1<<25 { + maxAllowedPacketSize = 1 << 25 + } + + dbt.mustExec("CREATE TABLE test (value LONGBLOB)") + + in := strings.Repeat(`a`, maxAllowedPacketSize+1) + var out string + var rows *sql.Rows + + // Long text data + const nonDataQueryLen = 28 // length query w/o value + inS := in[:maxAllowedPacketSize-nonDataQueryLen] + dbt.mustExec("INSERT INTO test VALUES('" + inS + "')") + rows = dbt.mustQuery("SELECT value FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if inS != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(inS), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + dbt.Fatalf("LONGBLOB: no data") + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Long binary data + dbt.mustExec("INSERT INTO test VALUES(?)", in) + rows = dbt.mustQuery("SELECT value FROM test WHERE 1=?", 1) + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(in), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + if err = rows.Err(); err != nil { + dbt.Fatalf("LONGBLOB: no data (err: %s)", err.Error()) + } else { + dbt.Fatal("LONGBLOB: no data (err: )") + } + } + }) +} + +func TestLoadData(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + verifyLoadDataResult := func() { + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + dbt.Fatal(err.Error()) + } + + i := 0 + values := [4]string{ + "a string", + "a string containing a \t", + "a string containing a \n", + "a string containing both \t\n", + } + + var id int + var value string + + for rows.Next() { + i++ + err = rows.Scan(&id, &value) + if err != nil { + dbt.Fatal(err.Error()) + } + if i != id { + dbt.Fatalf("%d != %d", i, id) + } + if values[i-1] != value { + dbt.Fatalf("%q != %q", values[i-1], value) + } + } + err = rows.Err() + if err != nil { + dbt.Fatal(err.Error()) + } + + if i != 4 { + dbt.Fatalf("rows count mismatch. Got %d, want 4", i) + } + } + + dbt.db.Exec("DROP TABLE IF EXISTS test") + dbt.mustExec("CREATE TABLE test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") + + // Local File + file, err := ioutil.TempFile("", "gotest") + defer os.Remove(file.Name()) + if err != nil { + dbt.Fatal(err) + } + RegisterLocalFile(file.Name()) + + // Try first with empty file + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + var count int + err = dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&count) + if err != nil { + dbt.Fatal(err.Error()) + } + if count != 0 { + dbt.Fatalf("unexpected row count: got %d, want 0", count) + } + + // Then fille File with data and try to load it + file.WriteString("1\ta string\n2\ta string containing a \\t\n3\ta string containing a \\n\n4\ta string containing both \\t\\n\n") + file.Close() + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + verifyLoadDataResult() + + // Try with non-existing file + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent file didn't fail") + } else if err.Error() != "local file 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Reader + RegisterReaderHandler("test", func() io.Reader { + file, err = os.Open(file.Name()) + if err != nil { + dbt.Fatal(err) + } + return file + }) + dbt.mustExec("LOAD DATA LOCAL INFILE 'Reader::test' INTO TABLE test") + verifyLoadDataResult() + // negative test + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'Reader::doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent Reader didn't fail") + } else if err.Error() != "Reader 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + }) +} + +func TestFoundRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + }) + runTests(t, dsn+"&clientFoundRows=true", func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 matched rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 3 { + dbt.Fatalf("Expected 3 matched rows, got %d", count) + } + }) +} + +func TestTLS(t *testing.T) { + tlsTestReq := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + if err == ErrNoTLS { + dbt.Skip("server does not support TLS") + } else { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + rows := dbt.mustQuery("SHOW STATUS LIKE 'Ssl_cipher'") + defer rows.Close() + + var variable, value *sql.RawBytes + for rows.Next() { + if err := rows.Scan(&variable, &value); err != nil { + dbt.Fatal(err.Error()) + } + + if (*value == nil) || (len(*value) == 0) { + dbt.Fatalf("no Cipher") + } else { + dbt.Logf("Cipher: %s", *value) + } + } + } + tlsTestOpt := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + runTests(t, dsn+"&tls=preferred", tlsTestOpt) + runTests(t, dsn+"&tls=skip-verify", tlsTestReq) + + // Verify that registering / using a custom cfg works + RegisterTLSConfig("custom-skip-verify", &tls.Config{ + InsecureSkipVerify: true, + }) + runTests(t, dsn+"&tls=custom-skip-verify", tlsTestReq) +} + +func TestReuseClosedConnection(t *testing.T) { + // this test does not use sql.database, it uses the driver directly + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + md := &MySQLDriver{} + conn, err := md.Open(dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + stmt, err := conn.Prepare("DO 1") + if err != nil { + t.Fatalf("error preparing statement: %s", err.Error()) + } + _, err = stmt.Exec(nil) + if err != nil { + t.Fatalf("error executing statement: %s", err.Error()) + } + err = conn.Close() + if err != nil { + t.Fatalf("error closing connection: %s", err.Error()) + } + + defer func() { + if err := recover(); err != nil { + t.Errorf("panic after reusing a closed connection: %v", err) + } + }() + _, err = stmt.Exec(nil) + if err != nil && err != driver.ErrBadConn { + t.Errorf("unexpected error '%s', expected '%s'", + err.Error(), driver.ErrBadConn.Error()) + } +} + +func TestCharset(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mustSetCharset := func(charsetParam, expected string) { + runTests(t, dsn+"&"+charsetParam, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT @@character_set_connection") + defer rows.Close() + + if !rows.Next() { + dbt.Fatalf("error getting connection charset: %s", rows.Err()) + } + + var got string + rows.Scan(&got) + + if got != expected { + dbt.Fatalf("expected connection charset %s but got %s", expected, got) + } + }) + } + + // non utf8 test + mustSetCharset("charset=ascii", "ascii") + + // when the first charset is invalid, use the second + mustSetCharset("charset=none,utf8", "utf8") + + // when the first charset is valid, use it + mustSetCharset("charset=ascii,utf8", "ascii") + mustSetCharset("charset=utf8,ascii", "utf8") +} + +func TestFailingCharset(t *testing.T) { + runTests(t, dsn+"&charset=none", func(dbt *DBTest) { + // run query to really establish connection... + _, err := dbt.db.Exec("SELECT 1") + if err == nil { + dbt.db.Close() + t.Fatalf("connection must not succeed without a valid charset") + } + }) +} + +func TestCollation(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + defaultCollation := "utf8mb4_general_ci" + testCollations := []string{ + "", // do not set + defaultCollation, // driver default + "latin1_general_ci", + "binary", + "utf8_unicode_ci", + "cp1257_bin", + } + + for _, collation := range testCollations { + var expected, tdsn string + if collation != "" { + tdsn = dsn + "&collation=" + collation + expected = collation + } else { + tdsn = dsn + expected = defaultCollation + } + + runTests(t, tdsn, func(dbt *DBTest) { + var got string + if err := dbt.db.QueryRow("SELECT @@collation_connection").Scan(&got); err != nil { + dbt.Fatal(err) + } + + if got != expected { + dbt.Fatalf("expected connection collation %s but got %s", expected, got) + } + }) + } +} + +func TestColumnsWithAlias(t *testing.T) { + runTests(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1 AS A") + defer rows.Close() + cols, _ := rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A" { + t.Fatalf("expected column name \"A\", got \"%s\"", cols[0]) + } + + rows = dbt.mustQuery("SELECT * FROM (SELECT 1 AS one) AS A") + defer rows.Close() + cols, _ = rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A.one" { + t.Fatalf("expected column name \"A.one\", got \"%s\"", cols[0]) + } + }) +} + +func TestRawBytesResultExceedsBuffer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // defaultBufSize from buffer.go + expected := strings.Repeat("abc", defaultBufSize) + + rows := dbt.mustQuery("SELECT '" + expected + "'") + defer rows.Close() + if !rows.Next() { + dbt.Error("expected result, got none") + } + var result sql.RawBytes + rows.Scan(&result) + if expected != string(result) { + dbt.Error("result did not match expected value") + } + }) +} + +func TestTimezoneConversion(t *testing.T) { + zones := []string{"UTC", "US/Central", "US/Pacific", "Local"} + + // Regression test for timezone handling + tzTest := func(dbt *DBTest) { + // Create table + dbt.mustExec("CREATE TABLE test (ts TIMESTAMP)") + + // Insert local time into database (should be converted) + usCentral, _ := time.LoadLocation("US/Central") + reftime := time.Date(2014, 05, 30, 18, 03, 17, 0, time.UTC).In(usCentral) + dbt.mustExec("INSERT INTO test VALUE (?)", reftime) + + // Retrieve time from DB + rows := dbt.mustQuery("SELECT ts FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Fatal("did not get any rows out") + } + + var dbTime time.Time + err := rows.Scan(&dbTime) + if err != nil { + dbt.Fatal("Err", err) + } + + // Check that dates match + if reftime.Unix() != dbTime.Unix() { + dbt.Errorf("times do not match.\n") + dbt.Errorf(" Now(%v)=%v\n", usCentral, reftime) + dbt.Errorf(" Now(UTC)=%v\n", dbTime) + } + } + + for _, tz := range zones { + runTests(t, dsn+"&parseTime=true&loc="+url.QueryEscape(tz), tzTest) + } +} + +// Special cases + +func TestRowsClose(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows, err := dbt.db.Query("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + err = rows.Close() + if err != nil { + dbt.Fatal(err) + } + + if rows.Next() { + dbt.Fatal("unexpected row after rows.Close()") + } + + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + }) +} + +// dangling statements +// http://code.google.com/p/go/issues/detail?id=3865 +func TestCloseStmtBeforeRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + rows, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows.Close() + + err = stmt.Close() + if err != nil { + dbt.Fatal(err) + } + + if !rows.Next() { + dbt.Fatal("getting row failed") + } else { + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + + var out bool + err = rows.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + }) +} + +// It is valid to have multiple Rows for the same Stmt +// http://code.google.com/p/go/issues/detail?id=3734 +func TestStmtMultiRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1 UNION SELECT 0") + if err != nil { + dbt.Fatal(err) + } + + rows1, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows1.Close() + + rows2, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows2.Close() + + var out bool + + // 1 + if !rows1.Next() { + dbt.Fatal("first rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + if !rows2.Next() { + dbt.Fatal("first rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + // 2 + if !rows1.Next() { + dbt.Fatal("second rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows1.Next() { + dbt.Fatal("unexpected row on rows1") + } + err = rows1.Close() + if err != nil { + dbt.Fatal(err) + } + } + + if !rows2.Next() { + dbt.Fatal("second rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows2.Next() { + dbt.Fatal("unexpected row on rows2") + } + err = rows2.Close() + if err != nil { + dbt.Fatal(err) + } + } + }) +} + +// Regression test for +// * more than 32 NULL parameters (issue 209) +// * more parameters than fit into the buffer (issue 201) +// * parameters * 64 > max_allowed_packet (issue 734) +func TestPreparedManyCols(t *testing.T) { + numParams := 65535 + runTests(t, dsn, func(dbt *DBTest) { + query := "SELECT ?" + strings.Repeat(",?", numParams-1) + stmt, err := dbt.db.Prepare(query) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + + // create more parameters than fit into the buffer + // which will take nil-values + params := make([]interface{}, numParams) + rows, err := stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + + // Create 0byte string which we can't send via STMT_LONG_DATA. + for i := 0; i < numParams; i++ { + params[i] = "" + } + rows, err = stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + }) +} + +func TestConcurrent(t *testing.T) { + if enabled, _ := readBool(os.Getenv("MYSQL_TEST_CONCURRENT")); !enabled { + t.Skip("MYSQL_TEST_CONCURRENT env var not set") + } + + runTests(t, dsn, func(dbt *DBTest) { + var max int + err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + dbt.Logf("testing up to %d concurrent connections \r\n", max) + + var remaining, succeeded int32 = int32(max), 0 + + var wg sync.WaitGroup + wg.Add(max) + + var fatalError string + var once sync.Once + fatalf := func(s string, vals ...interface{}) { + once.Do(func() { + fatalError = fmt.Sprintf(s, vals...) + }) + } + + for i := 0; i < max; i++ { + go func(id int) { + defer wg.Done() + + tx, err := dbt.db.Begin() + atomic.AddInt32(&remaining, -1) + + if err != nil { + if err.Error() != "Error 1040: Too many connections" { + fatalf("error on conn %d: %s", id, err.Error()) + } + return + } + + // keep the connection busy until all connections are open + for remaining > 0 { + if _, err = tx.Exec("DO 1"); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + } + + if err = tx.Commit(); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + + // everything went fine with this connection + atomic.AddInt32(&succeeded, 1) + }(i) + } + + // wait until all conections are open + wg.Wait() + + if fatalError != "" { + dbt.Fatal(fatalError) + } + + dbt.Logf("reached %d concurrent connections\r\n", succeeded) + }) +} + +func testDialError(t *testing.T, dialErr error, expectErr error) { + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + return nil, dialErr + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + _, err = db.Exec("DO 1") + if err != expectErr { + t.Fatalf("was expecting %s. Got: %s", dialErr, err) + } +} + +func TestDialUnknownError(t *testing.T) { + testErr := fmt.Errorf("test") + testDialError(t, testErr, testErr) +} + +func TestDialNonRetryableNetErr(t *testing.T) { + testErr := netErrorMock{} + testDialError(t, testErr, testErr) +} + +func TestDialTemporaryNetErr(t *testing.T) { + testErr := netErrorMock{temporary: true} + testDialError(t, testErr, driver.ErrBadConn) +} + +// Tests custom dial functions +func TestCustomDial(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + // our custom dial function which justs wraps net.Dial here + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + if _, err = db.Exec("DO 1"); err != nil { + t.Fatalf("connection failed: %s", err.Error()) + } +} + +func TestSQLInjection(t *testing.T) { + createTest := func(arg string) func(dbt *DBTest) { + return func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (?)", 1) + + var v int + // NULL can't be equal to anything, the idea here is to inject query so it returns row + // This test verifies that escapeQuotes and escapeBackslash are working properly + err := dbt.db.QueryRow("SELECT v FROM test WHERE NULL = ?", arg).Scan(&v) + if err == sql.ErrNoRows { + return // success, sql injection failed + } else if err == nil { + dbt.Errorf("sql injection successful with arg: %s", arg) + } else { + dbt.Errorf("error running query with arg: %s; err: %s", arg, err.Error()) + } + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, createTest("1 OR 1=1")) + runTests(t, testdsn, createTest("' OR '1'='1")) + } +} + +// Test if inserted data is correctly retrieved after being escaped +func TestInsertRetrieveEscapedData(t *testing.T) { + testData := func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v VARCHAR(255))") + + // All sequences that are escaped by escapeQuotes and escapeBackslash + v := "foo \x00\n\r\x1a\"'\\" + dbt.mustExec("INSERT INTO test VALUES (?)", v) + + var out string + err := dbt.db.QueryRow("SELECT v FROM test").Scan(&out) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + if out != v { + dbt.Errorf("%q != %q", out, v) + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, testData) + } +} + +func TestUnixSocketAuthFail(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Save the current logger so we can restore it. + oldLogger := errLog + + // Set a new logger so we can capture its output. + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + newLogger := log.New(buffer, "prefix: ", 0) + SetLogger(newLogger) + + // Restore the logger. + defer SetLogger(oldLogger) + + // Make a new DSN that uses the MySQL socket file and a bad password, which + // we can make by simply appending any character to the real password. + badPass := pass + "x" + socket := "" + if prot == "unix" { + socket = addr + } else { + // Get socket file from MySQL. + err := dbt.db.QueryRow("SELECT @@socket").Scan(&socket) + if err != nil { + t.Fatalf("error on SELECT @@socket: %s", err.Error()) + } + } + t.Logf("socket: %s", socket) + badDSN := fmt.Sprintf("%s:%s@unix(%s)/%s?timeout=30s", user, badPass, socket, dbname) + db, err := sql.Open("mysql", badDSN) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + // Connect to MySQL for real. This will cause an auth failure. + err = db.Ping() + if err == nil { + t.Error("expected Ping() to return an error") + } + + // The driver should not log anything. + if actual := buffer.String(); actual != "" { + t.Errorf("expected no output, got %q", actual) + } + }) +} + +// See Issue #422 +func TestInterruptBySignal(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + DROP PROCEDURE IF EXISTS test_signal; + CREATE PROCEDURE test_signal(ret INT) + BEGIN + SELECT ret; + SIGNAL SQLSTATE + '45001' + SET + MESSAGE_TEXT = "an error", + MYSQL_ERRNO = 45001; + END + `) + defer dbt.mustExec("DROP PROCEDURE test_signal") + + var val int + + // text protocol + rows, err := dbt.db.Query("CALL test_signal(42)") + if err != nil { + dbt.Fatalf("error on text query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + + // binary protocol + rows, err = dbt.db.Query("CALL test_signal(?)", 42) + if err != nil { + dbt.Fatalf("error on binary query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + }) +} + +func TestColumnsReusesSlice(t *testing.T) { + rows := mysqlRows{ + rs: resultSet{ + columns: []mysqlField{ + { + tableName: "test", + name: "A", + }, + { + tableName: "test", + name: "B", + }, + }, + }, + } + + allocs := testing.AllocsPerRun(1, func() { + cols := rows.Columns() + + if len(cols) != 2 { + t.Fatalf("expected 2 columns, got %d", len(cols)) + } + }) + + if allocs != 0 { + t.Fatalf("expected 0 allocations, got %d", int(allocs)) + } + + if rows.rs.columnNames == nil { + t.Fatalf("expected columnNames to be set, got nil") + } +} + +func TestRejectReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read-only. We didn't set the `rejectReadOnly` + // option, so any writes after this should fail. + _, err := dbt.db.Exec("SET SESSION TRANSACTION READ ONLY") + // Error 1193: Unknown system variable 'TRANSACTION' => skip test, + // MySQL server version is too old + maybeSkip(t, err, 1193) + if _, err := dbt.db.Exec("DROP TABLE test"); err == nil { + t.Fatalf("writing to DB in read-only session without " + + "rejectReadOnly did not error") + } + // Set the session back to read-write so runTests() can properly clean + // up the table `test`. + dbt.mustExec("SET SESSION TRANSACTION READ WRITE") + }) + + // Enable the `rejectReadOnly` option. + runTests(t, dsn+"&rejectReadOnly=true", func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read only. Any writes after this should error on + // a driver.ErrBadConn, and cause `database/sql` to initiate a new + // connection. + dbt.mustExec("SET SESSION TRANSACTION READ ONLY") + // This would error, but `database/sql` should automatically retry on a + // new connection which is not read-only, and eventually succeed. + dbt.mustExec("DROP TABLE test") + }) +} + +func TestPing(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.fail("Ping", "Ping", err) + } + }) +} + +// See Issue #799 +func TestEmptyPassword(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn := fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, "", netAddr, dbname) + db, err := sql.Open("mysql", dsn) + if err == nil { + defer db.Close() + err = db.Ping() + } + + if pass == "" { + if err != nil { + t.Fatal(err.Error()) + } + } else { + if err == nil { + t.Fatal("expected authentication error") + } + if !strings.HasPrefix(err.Error(), "Error 1045") { + t.Fatal(err.Error()) + } + } +} + +// static interface implementation checks of mysqlConn +var ( + _ driver.ConnBeginTx = &mysqlConn{} + _ driver.ConnPrepareContext = &mysqlConn{} + _ driver.ExecerContext = &mysqlConn{} + _ driver.Pinger = &mysqlConn{} + _ driver.QueryerContext = &mysqlConn{} +) + +// static interface implementation checks of mysqlStmt +var ( + _ driver.StmtExecContext = &mysqlStmt{} + _ driver.StmtQueryContext = &mysqlStmt{} +) + +// Ensure that all the driver interfaces are implemented +var ( + // _ driver.RowsColumnTypeLength = &binaryRows{} + // _ driver.RowsColumnTypeLength = &textRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &binaryRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &textRows{} + _ driver.RowsColumnTypeNullable = &binaryRows{} + _ driver.RowsColumnTypeNullable = &textRows{} + _ driver.RowsColumnTypePrecisionScale = &binaryRows{} + _ driver.RowsColumnTypePrecisionScale = &textRows{} + _ driver.RowsColumnTypeScanType = &binaryRows{} + _ driver.RowsColumnTypeScanType = &textRows{} + _ driver.RowsNextResultSet = &binaryRows{} + _ driver.RowsNextResultSet = &textRows{} +) + +func TestMultiResultSet(t *testing.T) { + type result struct { + values [][]int + columns []string + } + + // checkRows is a helper test function to validate rows containing 3 result + // sets with specific values and columns. The basic query would look like this: + // + // SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + // SELECT 0 UNION SELECT 1; + // SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + // + // to distinguish test cases the first string argument is put in front of + // every error or fatal message. + checkRows := func(desc string, rows *sql.Rows, dbt *DBTest) { + expected := []result{ + { + values: [][]int{{1, 2}, {3, 4}}, + columns: []string{"col1", "col2"}, + }, + { + values: [][]int{{1, 2, 3}, {4, 5, 6}}, + columns: []string{"col1", "col2", "col3"}, + }, + } + + var res1 result + for rows.Next() { + var res [2]int + if err := rows.Scan(&res[0], &res[1]); err != nil { + dbt.Fatal(err) + } + res1.values = append(res1.values, res[:]) + } + + cols, err := rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res1.columns = cols + + if !reflect.DeepEqual(expected[0], res1) { + dbt.Error(desc, "want =", expected[0], "got =", res1) + } + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + // ignoring one result set + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + var res2 result + cols, err = rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res2.columns = cols + + for rows.Next() { + var res [3]int + if err := rows.Scan(&res[0], &res[1], &res[2]); err != nil { + dbt.Fatal(desc, err) + } + res2.values = append(res2.values, res[:]) + } + + if !reflect.DeepEqual(expected[1], res2) { + dbt.Error(desc, "want =", expected[1], "got =", res2) + } + + if rows.NextResultSet() { + dbt.Error(desc, "unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error(desc, err) + } + } + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery(`DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6;`) + defer rows.Close() + checkRows("query: ", rows, dbt) + }) + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + queries := []string{ + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + } + + defer dbt.mustExec("DROP PROCEDURE IF EXISTS test_mrss") + + for i, query := range queries { + dbt.mustExec(query) + + stmt, err := dbt.db.Prepare("CALL test_mrss()") + if err != nil { + dbt.Fatalf("%v (i=%d)", err, i) + } + defer stmt.Close() + + for j := 0; j < 2; j++ { + rows, err := stmt.Query() + if err != nil { + dbt.Fatalf("%v (i=%d) (j=%d)", err, i, j) + } + checkRows(fmt.Sprintf("prepared stmt query (i=%d) (j=%d): ", i, j), rows, dbt) + } + } + }) +} + +func TestMultiResultSetNoSelect(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("DO 1; DO 2;") + defer rows.Close() + + if rows.Next() { + dbt.Error("unexpected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +// tests if rows are set in a proper state if some results were ignored before +// calling rows.NextResultSet. +func TestSkipResults(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1, 2") + defer rows.Close() + + if !rows.Next() { + dbt.Error("expected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +func TestPingContext(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := dbt.db.PingContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (1)"); err == nil { + dbt.Error("expected error") + } else if err.Error() != "context canceled" { + dbt.Fatalf("unexpected error: %s", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (1)"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQueryRow(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (1), (2), (3)") + ctx, cancel := context.WithCancel(context.Background()) + + rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM test") + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + // the first row will be succeed. + var v int + if !rows.Next() { + dbt.Fatalf("unexpected end") + } + if err := rows.Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + + cancel() + // make sure the driver receives the cancel request. + time.Sleep(100 * time.Millisecond) + + if rows.Next() { + dbt.Errorf("expected end, but not") + } + if err := rows.Err(); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelPrepare(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := dbt.db.PrepareContext(ctx, "SELECT 1"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelStmtExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.ExecContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelStmtQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.QueryContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query has done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelBegin(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + tx, err := dbt.db.BeginTx(ctx, nil) + if err != nil { + dbt.Fatal(err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(100*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := tx.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Transaction is canceled, so expect an error. + switch err := tx.Commit(); err { + case sql.ErrTxDone: + // because the transaction has already been rollbacked. + // the database/sql package watches ctx + // and rollbacks when ctx is canceled. + case context.Canceled: + // the database/sql package rollbacks on another goroutine, + // so the transaction may not be rollbacked depending on goroutine scheduling. + default: + dbt.Errorf("expected sql.ErrTxDone or context.Canceled, got %v", err) + } + + // Context is canceled, so cannot begin a transaction. + if _, err := dbt.db.BeginTx(ctx, nil); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextBeginIsolationLevel(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx1, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelRepeatableRead, + }) + if err != nil { + dbt.Fatal(err) + } + + tx2, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelReadCommitted, + }) + if err != nil { + dbt.Fatal(err) + } + + _, err = tx1.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if err != nil { + dbt.Fatal(err) + } + + var v int + row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Because writer transaction wasn't commited yet, it should be available + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + err = tx1.Commit() + if err != nil { + dbt.Fatal(err) + } + + row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Data written by writer transaction is already commited, it should be selectable + if v != 1 { + dbt.Errorf("expected val to be 1, got %d", v) + } + tx2.Commit() + }) +} + +func TestContextBeginReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + ReadOnly: true, + }) + if _, ok := err.(*MySQLError); ok { + dbt.Skip("It seems that your MySQL does not support READ ONLY transactions") + return + } else if err != nil { + dbt.Fatal(err) + } + + // INSERT queries fail in a READ ONLY transaction. + _, err = tx.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if _, ok := err.(*MySQLError); !ok { + dbt.Errorf("expected MySQLError, got %v", err) + } + + // SELECT queries can be executed. + var v int + row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + if err := tx.Commit(); err != nil { + dbt.Fatal(err) + } + }) +} + +func TestRowsColumnTypes(t *testing.T) { + niNULL := sql.NullInt64{Int64: 0, Valid: false} + ni0 := sql.NullInt64{Int64: 0, Valid: true} + ni1 := sql.NullInt64{Int64: 1, Valid: true} + ni42 := sql.NullInt64{Int64: 42, Valid: true} + nfNULL := sql.NullFloat64{Float64: 0.0, Valid: false} + nf0 := sql.NullFloat64{Float64: 0.0, Valid: true} + nf1337 := sql.NullFloat64{Float64: 13.37, Valid: true} + nt0 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} + nt1 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} + nt2 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} + nt6 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} + nd1 := NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} + nd2 := NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} + ndNULL := NullTime{Time: time.Time{}, Valid: false} + rbNULL := sql.RawBytes(nil) + rb0 := sql.RawBytes("0") + rb42 := sql.RawBytes("42") + rbTest := sql.RawBytes("Test") + rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 + rbx0 := sql.RawBytes("\x00") + rbx42 := sql.RawBytes("\x42") + + var columns = []struct { + name string + fieldType string // type used when creating table schema + databaseTypeName string // actual type used by MySQL + scanType reflect.Type + nullable bool + precision int64 // 0 if not ok + scale int64 + valuesIn [3]string + valuesOut [3]interface{} + }{ + {"bit8null", "BIT(8)", "BIT", scanTypeRawBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]interface{}{rbx0, rbNULL, rbx42}}, + {"boolnull", "BOOL", "TINYINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "true", "0"}, [3]interface{}{niNULL, ni1, ni0}}, + {"bool", "BOOL NOT NULL", "TINYINT", scanTypeInt8, false, 0, 0, [3]string{"1", "0", "FALSE"}, [3]interface{}{int8(1), int8(0), int8(0)}}, + {"intnull", "INTEGER", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"smallint", "SMALLINT NOT NULL", "SMALLINT", scanTypeInt16, false, 0, 0, [3]string{"0", "-32768", "32767"}, [3]interface{}{int16(0), int16(-32768), int16(32767)}}, + {"smallintnull", "SMALLINT", "SMALLINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int3null", "INT(3)", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int7", "INT(7) NOT NULL", "INT", scanTypeInt32, false, 0, 0, [3]string{"0", "-1337", "42"}, [3]interface{}{int32(0), int32(-1337), int32(42)}}, + {"mediumintnull", "MEDIUMINT", "MEDIUMINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "42", "NULL"}, [3]interface{}{ni0, ni42, niNULL}}, + {"bigint", "BIGINT NOT NULL", "BIGINT", scanTypeInt64, false, 0, 0, [3]string{"0", "65535", "-42"}, [3]interface{}{int64(0), int64(65535), int64(-42)}}, + {"bigintnull", "BIGINT", "BIGINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "1", "42"}, [3]interface{}{niNULL, ni1, ni42}}, + {"tinyuint", "TINYINT UNSIGNED NOT NULL", "TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, + {"smalluint", "SMALLINT UNSIGNED NOT NULL", "SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, + {"biguint", "BIGINT UNSIGNED NOT NULL", "BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, + {"uint13", "INT(13) UNSIGNED NOT NULL", "INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, + {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float32(0), float32(42), float32(13.37)}}, + {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"double", "DOUBLE NOT NULL", "DOUBLE", scanTypeFloat64, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float64(0), float64(42), float64(13.37)}}, + {"doublenull", "DOUBLE", "DOUBLE", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), sql.RawBytes("13.370000"), sql.RawBytes("1234.123456")}}, + {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeRawBytes, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), rbNULL, sql.RawBytes("1234.123456")}}, + {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), sql.RawBytes("13.3700"), sql.RawBytes("1234.1235")}}, + {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, + {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, + {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, + {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, + {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, + {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, + {"date", "DATE", "DATE", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02'", "NULL", "'2006-03-04'"}, [3]interface{}{nd1, ndNULL, nd2}}, + {"year", "YEAR NOT NULL", "YEAR", scanTypeUint16, false, 0, 0, [3]string{"2006", "2000", "1994"}, [3]interface{}{uint16(2006), uint16(2000), uint16(1994)}}, + } + + schema := "" + values1 := "" + values2 := "" + values3 := "" + for _, column := range columns { + schema += fmt.Sprintf("`%s` %s, ", column.name, column.fieldType) + values1 += column.valuesIn[0] + ", " + values2 += column.valuesIn[1] + ", " + values3 += column.valuesIn[2] + ", " + } + schema = schema[:len(schema)-2] + values1 = values1[:len(values1)-2] + values2 = values2[:len(values2)-2] + values3 = values3[:len(values3)-2] + + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (" + schema + ")") + dbt.mustExec("INSERT INTO test VALUES (" + values1 + "), (" + values2 + "), (" + values3 + ")") + + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + t.Fatalf("Query: %v", err) + } + + tt, err := rows.ColumnTypes() + if err != nil { + t.Fatalf("ColumnTypes: %v", err) + } + + if len(tt) != len(columns) { + t.Fatalf("unexpected number of columns: expected %d, got %d", len(columns), len(tt)) + } + + types := make([]reflect.Type, len(tt)) + for i, tp := range tt { + column := columns[i] + + // Name + name := tp.Name() + if name != column.name { + t.Errorf("column name mismatch %s != %s", name, column.name) + continue + } + + // DatabaseTypeName + databaseTypeName := tp.DatabaseTypeName() + if databaseTypeName != column.databaseTypeName { + t.Errorf("databasetypename name mismatch for column %q: %s != %s", name, databaseTypeName, column.databaseTypeName) + continue + } + + // ScanType + scanType := tp.ScanType() + if scanType != column.scanType { + if scanType == nil { + t.Errorf("scantype is null for column %q", name) + } else { + t.Errorf("scantype mismatch for column %q: %s != %s", name, scanType.Name(), column.scanType.Name()) + } + continue + } + types[i] = scanType + + // Nullable + nullable, ok := tp.Nullable() + if !ok { + t.Errorf("nullable not ok %q", name) + continue + } + if nullable != column.nullable { + t.Errorf("nullable mismatch for column %q: %t != %t", name, nullable, column.nullable) + } + + // Length + // length, ok := tp.Length() + // if length != column.length { + // if !ok { + // t.Errorf("length not ok for column %q", name) + // } else { + // t.Errorf("length mismatch for column %q: %d != %d", name, length, column.length) + // } + // continue + // } + + // Precision and Scale + precision, scale, ok := tp.DecimalSize() + if precision != column.precision { + if !ok { + t.Errorf("precision not ok for column %q", name) + } else { + t.Errorf("precision mismatch for column %q: %d != %d", name, precision, column.precision) + } + continue + } + if scale != column.scale { + if !ok { + t.Errorf("scale not ok for column %q", name) + } else { + t.Errorf("scale mismatch for column %q: %d != %d", name, scale, column.scale) + } + continue + } + } + + values := make([]interface{}, len(tt)) + for i := range values { + values[i] = reflect.New(types[i]).Interface() + } + i := 0 + for rows.Next() { + err = rows.Scan(values...) + if err != nil { + t.Fatalf("failed to scan values in %v", err) + } + for j := range values { + value := reflect.ValueOf(values[j]).Elem().Interface() + if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { + if columns[j].scanType == scanTypeRawBytes { + t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) + } else { + t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) + } + } + } + i++ + } + if i != 3 { + t.Errorf("expected 3 rows, got %d", i) + } + + if err := rows.Close(); err != nil { + t.Errorf("error closing rows: %s", err) + } + }) + } +} + +func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (value VARCHAR(255))") + dbt.db.Exec("INSERT INTO test VALUES (?)", (*testValuer)(nil)) + // This test will panic on the INSERT if ConvertValue() does not check for typed nil before calling Value() + }) +} + +// TestRawBytesAreNotModified checks for a race condition that arises when a query context +// is canceled while a user is calling rows.Scan. This is a more stringent test than the one +// proposed in https://github.com/golang/go/issues/23519. Here we're explicitly using +// `sql.RawBytes` to check the contents of our internal buffers are not modified after an implicit +// call to `Rows.Close`, so Context cancellation should **not** invalidate the backing buffers. +func TestRawBytesAreNotModified(t *testing.T) { + const blob = "abcdefghijklmnop" + const contextRaceIterations = 20 + const blobSize = defaultBufSize * 3 / 4 // Second row overwrites first row. + const insertRows = 4 + + var sqlBlobs = [2]string{ + strings.Repeat(blob, blobSize/len(blob)), + strings.Repeat(strings.ToUpper(blob), blobSize/len(blob)), + } + + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + for i := 0; i < insertRows; i++ { + dbt.mustExec("INSERT INTO test VALUES (?, ?)", i+1, sqlBlobs[i&1]) + } + + for i := 0; i < contextRaceIterations; i++ { + func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rows, err := dbt.db.QueryContext(ctx, `SELECT id, value FROM test`) + if err != nil { + t.Fatal(err) + } + + var b int + var raw sql.RawBytes + for rows.Next() { + if err := rows.Scan(&b, &raw); err != nil { + t.Fatal(err) + } + + before := string(raw) + // Ensure cancelling the query does not corrupt the contents of `raw` + cancel() + time.Sleep(time.Microsecond * 100) + after := string(raw) + + if before != after { + t.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) + } + } + rows.Close() + }() + } + }) +} diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go new file mode 100644 index 0000000..1d9b4ab --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -0,0 +1,636 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rsa" + "crypto/tls" + "errors" + "fmt" + "math/big" + "net" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +var ( + errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") + errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") + errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") + errInvalidDSNUnsafeCollation = errors.New("invalid DSN: interpolateParams can not be used with unsafe collations") +) + +// Config is a configuration parsed from a DSN string. +// If a new Config is created instead of being parsed from a DSN string, +// the NewConfig function should be used, which sets default values. +type Config struct { + User string // Username + Passwd string // Password (requires User) + Net string // Network type + Addr string // Network address (requires Net) + DBName string // Database name + Params map[string]string // Connection parameters + Collation string // Connection collation + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + ServerPubKey string // Server public key name + pubKey *rsa.PublicKey // Server public key + TLSConfig string // TLS configuration name + tls *tls.Config // TLS configuration + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + + AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE + AllowCleartextPasswords bool // Allows the cleartext client side plugin + AllowNativePasswords bool // Allows the native password authentication method + AllowOldPasswords bool // Allows the old insecure password method + ClientFoundRows bool // Return number of matching rows instead of rows changed + ColumnsWithAlias bool // Prepend table alias to column names + InterpolateParams bool // Interpolate placeholders into query string + MultiStatements bool // Allow multiple statements in one query + ParseTime bool // Parse time values to time.Time + RejectReadOnly bool // Reject read-only connections +} + +// NewConfig creates a new Config and sets default values. +func NewConfig() *Config { + return &Config{ + Collation: defaultCollation, + Loc: time.UTC, + MaxAllowedPacket: defaultMaxAllowedPacket, + AllowNativePasswords: true, + } +} + +func (cfg *Config) Clone() *Config { + cp := *cfg + if cp.tls != nil { + cp.tls = cfg.tls.Clone() + } + if len(cp.Params) > 0 { + cp.Params = make(map[string]string, len(cfg.Params)) + for k, v := range cfg.Params { + cp.Params[k] = v + } + } + if cfg.pubKey != nil { + cp.pubKey = &rsa.PublicKey{ + N: new(big.Int).Set(cfg.pubKey.N), + E: cfg.pubKey.E, + } + } + return &cp +} + +func (cfg *Config) normalize() error { + if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { + return errInvalidDSNUnsafeCollation + } + + // Set default network if empty + if cfg.Net == "" { + cfg.Net = "tcp" + } + + // Set default address if empty + if cfg.Addr == "" { + switch cfg.Net { + case "tcp": + cfg.Addr = "127.0.0.1:3306" + case "unix": + cfg.Addr = "/tmp/mysql.sock" + default: + return errors.New("default addr for network '" + cfg.Net + "' unknown") + } + } else if cfg.Net == "tcp" { + cfg.Addr = ensureHavePort(cfg.Addr) + } + + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.tls = &tls.Config{} + case "skip-verify", "preferred": + cfg.tls = &tls.Config{InsecureSkipVerify: true} + default: + cfg.tls = getTLSConfigClone(cfg.TLSConfig) + if cfg.tls == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } + } + + if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + + if cfg.ServerPubKey != "" { + cfg.pubKey = getServerPubKey(cfg.ServerPubKey) + if cfg.pubKey == nil { + return errors.New("invalid value / unknown server pub key name: " + cfg.ServerPubKey) + } + } + + return nil +} + +// FormatDSN formats the given Config into a DSN string which can be passed to +// the driver. +func (cfg *Config) FormatDSN() string { + var buf bytes.Buffer + + // [username[:password]@] + if len(cfg.User) > 0 { + buf.WriteString(cfg.User) + if len(cfg.Passwd) > 0 { + buf.WriteByte(':') + buf.WriteString(cfg.Passwd) + } + buf.WriteByte('@') + } + + // [protocol[(address)]] + if len(cfg.Net) > 0 { + buf.WriteString(cfg.Net) + if len(cfg.Addr) > 0 { + buf.WriteByte('(') + buf.WriteString(cfg.Addr) + buf.WriteByte(')') + } + } + + // /dbname + buf.WriteByte('/') + buf.WriteString(cfg.DBName) + + // [?param1=value1&...¶mN=valueN] + hasParam := false + + if cfg.AllowAllFiles { + hasParam = true + buf.WriteString("?allowAllFiles=true") + } + + if cfg.AllowCleartextPasswords { + if hasParam { + buf.WriteString("&allowCleartextPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowCleartextPasswords=true") + } + } + + if !cfg.AllowNativePasswords { + if hasParam { + buf.WriteString("&allowNativePasswords=false") + } else { + hasParam = true + buf.WriteString("?allowNativePasswords=false") + } + } + + if cfg.AllowOldPasswords { + if hasParam { + buf.WriteString("&allowOldPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowOldPasswords=true") + } + } + + if cfg.ClientFoundRows { + if hasParam { + buf.WriteString("&clientFoundRows=true") + } else { + hasParam = true + buf.WriteString("?clientFoundRows=true") + } + } + + if col := cfg.Collation; col != defaultCollation && len(col) > 0 { + if hasParam { + buf.WriteString("&collation=") + } else { + hasParam = true + buf.WriteString("?collation=") + } + buf.WriteString(col) + } + + if cfg.ColumnsWithAlias { + if hasParam { + buf.WriteString("&columnsWithAlias=true") + } else { + hasParam = true + buf.WriteString("?columnsWithAlias=true") + } + } + + if cfg.InterpolateParams { + if hasParam { + buf.WriteString("&interpolateParams=true") + } else { + hasParam = true + buf.WriteString("?interpolateParams=true") + } + } + + if cfg.Loc != time.UTC && cfg.Loc != nil { + if hasParam { + buf.WriteString("&loc=") + } else { + hasParam = true + buf.WriteString("?loc=") + } + buf.WriteString(url.QueryEscape(cfg.Loc.String())) + } + + if cfg.MultiStatements { + if hasParam { + buf.WriteString("&multiStatements=true") + } else { + hasParam = true + buf.WriteString("?multiStatements=true") + } + } + + if cfg.ParseTime { + if hasParam { + buf.WriteString("&parseTime=true") + } else { + hasParam = true + buf.WriteString("?parseTime=true") + } + } + + if cfg.ReadTimeout > 0 { + if hasParam { + buf.WriteString("&readTimeout=") + } else { + hasParam = true + buf.WriteString("?readTimeout=") + } + buf.WriteString(cfg.ReadTimeout.String()) + } + + if cfg.RejectReadOnly { + if hasParam { + buf.WriteString("&rejectReadOnly=true") + } else { + hasParam = true + buf.WriteString("?rejectReadOnly=true") + } + } + + if len(cfg.ServerPubKey) > 0 { + if hasParam { + buf.WriteString("&serverPubKey=") + } else { + hasParam = true + buf.WriteString("?serverPubKey=") + } + buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + } + + if cfg.Timeout > 0 { + if hasParam { + buf.WriteString("&timeout=") + } else { + hasParam = true + buf.WriteString("?timeout=") + } + buf.WriteString(cfg.Timeout.String()) + } + + if len(cfg.TLSConfig) > 0 { + if hasParam { + buf.WriteString("&tls=") + } else { + hasParam = true + buf.WriteString("?tls=") + } + buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + } + + if cfg.WriteTimeout > 0 { + if hasParam { + buf.WriteString("&writeTimeout=") + } else { + hasParam = true + buf.WriteString("?writeTimeout=") + } + buf.WriteString(cfg.WriteTimeout.String()) + } + + if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { + if hasParam { + buf.WriteString("&maxAllowedPacket=") + } else { + hasParam = true + buf.WriteString("?maxAllowedPacket=") + } + buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) + + } + + // other params + if cfg.Params != nil { + var params []string + for param := range cfg.Params { + params = append(params, param) + } + sort.Strings(params) + for _, param := range params { + if hasParam { + buf.WriteByte('&') + } else { + hasParam = true + buf.WriteByte('?') + } + + buf.WriteString(param) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(cfg.Params[param])) + } + } + + return buf.String() +} + +// ParseDSN parses the DSN string to a Config +func ParseDSN(dsn string) (cfg *Config, err error) { + // New config with some default values + cfg = NewConfig() + + // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] + // Find the last '/' (since the password or the net addr might contain a '/') + foundSlash := false + for i := len(dsn) - 1; i >= 0; i-- { + if dsn[i] == '/' { + foundSlash = true + var j, k int + + // left part is empty if i <= 0 + if i > 0 { + // [username[:password]@][protocol[(address)]] + // Find the last '@' in dsn[:i] + for j = i; j >= 0; j-- { + if dsn[j] == '@' { + // username[:password] + // Find the first ':' in dsn[:j] + for k = 0; k < j; k++ { + if dsn[k] == ':' { + cfg.Passwd = dsn[k+1 : j] + break + } + } + cfg.User = dsn[:k] + + break + } + } + + // [protocol[(address)]] + // Find the first '(' in dsn[j+1:i] + for k = j + 1; k < i; k++ { + if dsn[k] == '(' { + // dsn[i-1] must be == ')' if an address is specified + if dsn[i-1] != ')' { + if strings.ContainsRune(dsn[k+1:i], ')') { + return nil, errInvalidDSNUnescaped + } + return nil, errInvalidDSNAddr + } + cfg.Addr = dsn[k+1 : i-1] + break + } + } + cfg.Net = dsn[j+1 : k] + } + + // dbname[?param1=value1&...¶mN=valueN] + // Find the first '?' in dsn[i+1:] + for j = i + 1; j < len(dsn); j++ { + if dsn[j] == '?' { + if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { + return + } + break + } + } + cfg.DBName = dsn[i+1 : j] + + break + } + } + + if !foundSlash && len(dsn) > 0 { + return nil, errInvalidDSNNoSlash + } + + if err = cfg.normalize(); err != nil { + return nil, err + } + return +} + +// parseDSNParams parses the DSN "query string" +// Values must be url.QueryEscape'ed +func parseDSNParams(cfg *Config, params string) (err error) { + for _, v := range strings.Split(params, "&") { + param := strings.SplitN(v, "=", 2) + if len(param) != 2 { + continue + } + + // cfg params + switch value := param[1]; param[0] { + // Disable INFILE whitelist / enable all files + case "allowAllFiles": + var isBool bool + cfg.AllowAllFiles, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use cleartext authentication mode (MySQL 5.5.10+) + case "allowCleartextPasswords": + var isBool bool + cfg.AllowCleartextPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use native password authentication + case "allowNativePasswords": + var isBool bool + cfg.AllowNativePasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use old authentication mode (pre MySQL 4.1) + case "allowOldPasswords": + var isBool bool + cfg.AllowOldPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Switch "rowsAffected" mode + case "clientFoundRows": + var isBool bool + cfg.ClientFoundRows, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Collation + case "collation": + cfg.Collation = value + break + + case "columnsWithAlias": + var isBool bool + cfg.ColumnsWithAlias, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Compression + case "compress": + return errors.New("compression not implemented yet") + + // Enable client side placeholder substitution + case "interpolateParams": + var isBool bool + cfg.InterpolateParams, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Time Location + case "loc": + if value, err = url.QueryUnescape(value); err != nil { + return + } + cfg.Loc, err = time.LoadLocation(value) + if err != nil { + return + } + + // multiple statements in one query + case "multiStatements": + var isBool bool + cfg.MultiStatements, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // time.Time parsing + case "parseTime": + var isBool bool + cfg.ParseTime, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // I/O read Timeout + case "readTimeout": + cfg.ReadTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // Reject read-only connections + case "rejectReadOnly": + var isBool bool + cfg.RejectReadOnly, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Server public key + case "serverPubKey": + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for server pub key name: %v", err) + } + cfg.ServerPubKey = name + + // Strict mode + case "strict": + panic("strict mode has been removed. See https://github.com/go-sql-driver/mysql/wiki/strict-mode") + + // Dial Timeout + case "timeout": + cfg.Timeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // TLS-Encryption + case "tls": + boolValue, isBool := readBool(value) + if isBool { + if boolValue { + cfg.TLSConfig = "true" + } else { + cfg.TLSConfig = "false" + } + } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { + cfg.TLSConfig = vl + } else { + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for TLS config name: %v", err) + } + cfg.TLSConfig = name + } + + // I/O write Timeout + case "writeTimeout": + cfg.WriteTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + case "maxAllowedPacket": + cfg.MaxAllowedPacket, err = strconv.Atoi(value) + if err != nil { + return + } + default: + // lazy init + if cfg.Params == nil { + cfg.Params = make(map[string]string) + } + + if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { + return + } + } + } + + return +} + +func ensureHavePort(addr string) string { + if _, _, err := net.SplitHostPort(addr); err != nil { + return net.JoinHostPort(addr, "3306") + } + return addr +} diff --git a/vendor/github.com/go-sql-driver/mysql/dsn_test.go b/vendor/github.com/go-sql-driver/mysql/dsn_test.go new file mode 100644 index 0000000..50dc293 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/dsn_test.go @@ -0,0 +1,415 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/tls" + "fmt" + "net/url" + "reflect" + "testing" + "time" +) + +var testDSNs = []struct { + in string + out *Config +}{{ + "username:password@protocol(address)/dbname?param=value", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, +}, { + "user@unix(/path/to/socket)/dbname?charset=utf8", + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, +}, { + "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, +}, { + "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, +}, { + "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "@/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:p@/ssword@/", + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "unix/?arg=%2Fsome%2Fpath.ext", + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(127.0.0.1)/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(de:ad:be:ef::ca:fe)/dbname", + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, +} + +func TestDSNParser(t *testing.T) { + for i, tst := range testDSNs { + cfg, err := ParseDSN(tst.in) + if err != nil { + t.Error(err.Error()) + } + + // pointer not static + cfg.tls = nil + + if !reflect.DeepEqual(cfg, tst.out) { + t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) + } + } +} + +func TestDSNParserInvalid(t *testing.T) { + var invalidDSNs = []string{ + "@net(addr/", // no closing brace + "@tcp(/", // no closing brace + "tcp(/", // no closing brace + "(/", // no closing brace + "net(addr)//", // unescaped + "User:pass@tcp(1.2.3.4:3306)", // no trailing slash + "net()/", // unknown default addr + //"/dbname?arg=/some/unescaped/path", + } + + for i, tst := range invalidDSNs { + if _, err := ParseDSN(tst); err == nil { + t.Errorf("invalid DSN #%d. (%s) didn't error!", i, tst) + } + } +} + +func TestDSNReformat(t *testing.T) { + for i, tst := range testDSNs { + dsn1 := tst.in + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Error(err.Error()) + continue + } + cfg1.tls = nil // pointer not static + res1 := fmt.Sprintf("%+v", cfg1) + + dsn2 := cfg1.FormatDSN() + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Error(err.Error()) + continue + } + cfg2.tls = nil // pointer not static + res2 := fmt.Sprintf("%+v", cfg2) + + if res1 != res2 { + t.Errorf("%d. %q does not match %q", i, res2, res1) + } + } +} + +func TestDSNServerPubKey(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + tst := baseDSN + "testKey" + cfg, err := ParseDSN(tst) + if err != nil { + t.Error(err.Error()) + } + + if cfg.ServerPubKey != "testKey" { + t.Errorf("unexpected cfg.ServerPubKey value: %v", cfg.ServerPubKey) + } + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } + + // Key is missing + tst = baseDSN + "invalid_name" + cfg, err = ParseDSN(tst) + if err == nil { + t.Errorf("invalid name in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } +} + +func TestDSNServerPubKeyQueryEscape(t *testing.T) { + const name = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + url.QueryEscape(name) + + RegisterServerPubKey(name, testPubKeyRSA) + defer DeregisterServerPubKey(name) + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } +} + +func TestDSNWithCustomTLS(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?tls=" + tlsCfg := tls.Config{} + + RegisterTLSConfig("utils_test", &tlsCfg) + defer DeregisterTLSConfig("utils_test") + + // Custom TLS is missing + tst := baseDSN + "invalid_tls" + cfg, err := ParseDSN(tst) + if err == nil { + t.Errorf("invalid custom TLS in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } + + tst = baseDSN + "utils_test" + + // Custom TLS with a server name + name := "foohost" + tlsCfg.ServerName = name + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, tst) + } + + // Custom TLS without a server name + name = "localhost" + tlsCfg.ServerName = "" + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct ServerName (%s) parsing DSN (%s).", name, tst) + } else if tlsCfg.ServerName != "" { + t.Errorf("tlsCfg was mutated ServerName (%s) should be empty parsing DSN (%s).", name, tst) + } +} + +func TestDSNTLSConfig(t *testing.T) { + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true" + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + dsn = "tcp(example.com)/?tls=true" + cfg, err = ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host without port)", expectedServerName, cfg.tls.ServerName) + } +} + +func TestDSNWithCustomTLSQueryEscape(t *testing.T) { + const configKey = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?tls=" + url.QueryEscape(configKey) + name := "foohost" + tlsCfg := tls.Config{ServerName: name} + + RegisterTLSConfig(configKey, &tlsCfg) + defer DeregisterTLSConfig(configKey) + + cfg, err := ParseDSN(dsn) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, dsn) + } +} + +func TestDSNUnsafeCollation(t *testing.T) { + _, err := ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=true") + if err != errInvalidDSNUnsafeCollation { + t.Errorf("expected %v, got %v", errInvalidDSNUnsafeCollation, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=false") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=ascii_bin&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=latin1_german1_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8mb4_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } +} + +func TestParamsAreSorted(t *testing.T) { + expected := "/dbname?interpolateParams=true&foobar=baz&quux=loo" + cfg := NewConfig() + cfg.DBName = "dbname" + cfg.InterpolateParams = true + cfg.Params = map[string]string{ + "quux": "loo", + "foobar": "baz", + } + actual := cfg.FormatDSN() + if actual != expected { + t.Errorf("generic Config.Params were not sorted: want %#v, got %#v", expected, actual) + } +} + +func TestCloneConfig(t *testing.T) { + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true&foobar=baz&serverPubKey=testKey" + cfg, err := ParseDSN(dsn) + if err != nil { + t.Fatal(err.Error()) + } + + cfg2 := cfg.Clone() + if cfg == cfg2 { + t.Errorf("Config.Clone did not create a separate config struct") + } + + if cfg2.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + cfg2.tls.ServerName = "example2.com" + if cfg.tls.ServerName == cfg2.tls.ServerName { + t.Errorf("changed cfg.tls.Server name should not propagate to original Config") + } + + if _, ok := cfg2.Params["foobar"]; !ok { + t.Errorf("cloned Config is missing custom params") + } + + delete(cfg2.Params, "foobar") + + if _, ok := cfg.Params["foobar"]; !ok { + t.Errorf("custom params in cloned Config should not propagate to original Config") + } + + if !reflect.DeepEqual(cfg.pubKey, cfg2.pubKey) { + t.Errorf("public key in Config should be identical") + } +} + +func TestNormalizeTLSConfig(t *testing.T) { + tt := []struct { + tlsConfig string + want *tls.Config + }{ + {"", nil}, + {"false", nil}, + {"true", &tls.Config{ServerName: "myserver"}}, + {"skip-verify", &tls.Config{InsecureSkipVerify: true}}, + {"preferred", &tls.Config{InsecureSkipVerify: true}}, + {"test_tls_config", &tls.Config{ServerName: "myServerName"}}, + } + + RegisterTLSConfig("test_tls_config", &tls.Config{ServerName: "myServerName"}) + defer func() { DeregisterTLSConfig("test_tls_config") }() + + for _, tc := range tt { + t.Run(tc.tlsConfig, func(t *testing.T) { + cfg := &Config{ + Addr: "myserver:3306", + TLSConfig: tc.tlsConfig, + } + + cfg.normalize() + + if cfg.tls == nil { + if tc.want != nil { + t.Fatal("wanted a tls config but got nil instead") + } + return + } + + if cfg.tls.ServerName != tc.want.ServerName { + t.Errorf("tls.ServerName doesn't match (want: '%s', got: '%s')", + tc.want.ServerName, cfg.tls.ServerName) + } + if cfg.tls.InsecureSkipVerify != tc.want.InsecureSkipVerify { + t.Errorf("tls.InsecureSkipVerify doesn't match (want: %T, got :%T)", + tc.want.InsecureSkipVerify, cfg.tls.InsecureSkipVerify) + } + }) + } +} + +func BenchmarkParseDSN(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + for _, tst := range testDSNs { + if _, err := ParseDSN(tst.in); err != nil { + b.Error(err.Error()) + } + } + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/errors.go b/vendor/github.com/go-sql-driver/mysql/errors.go index 44cf30d..760782f 100644 --- a/vendor/github.com/go-sql-driver/mysql/errors.go +++ b/vendor/github.com/go-sql-driver/mysql/errors.go @@ -9,30 +9,35 @@ package mysql import ( - "database/sql/driver" "errors" "fmt" - "io" "log" "os" ) // Various errors the driver might return. Can change between driver versions. var ( - ErrInvalidConn = errors.New("Invalid Connection") - ErrMalformPkt = errors.New("Malformed Packet") - ErrNoTLS = errors.New("TLS encryption requested but server does not support TLS") - ErrOldPassword = errors.New("This user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") - ErrCleartextPassword = errors.New("This user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN.") - ErrUnknownPlugin = errors.New("The authentication plugin is not supported.") - ErrOldProtocol = errors.New("MySQL-Server does not support required Protocol 41+") - ErrPktSync = errors.New("Commands out of sync. You can't run this command now") - ErrPktSyncMul = errors.New("Commands out of sync. Did you run multiple statements at once?") - ErrPktTooLarge = errors.New("Packet for query is too large. You can change this value on the server by adjusting the 'max_allowed_packet' variable.") - ErrBusyBuffer = errors.New("Busy buffer") + ErrInvalidConn = errors.New("invalid connection") + ErrMalformPkt = errors.New("malformed packet") + ErrNoTLS = errors.New("TLS requested but server does not support TLS") + ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") + ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") + ErrUnknownPlugin = errors.New("this authentication plugin is not supported") + ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") + ErrPktSync = errors.New("commands out of sync. You can't run this command now") + ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") + ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") + ErrBusyBuffer = errors.New("busy buffer") + + // errBadConnNoWrite is used for connection errors where nothing was sent to the database yet. + // If this happens first in a function starting a database interaction, it should be replaced by driver.ErrBadConn + // to trigger a resend. + // See https://github.com/go-sql-driver/mysql/pull/302 + errBadConnNoWrite = errors.New("bad connection") ) -var errLog Logger = log.New(os.Stderr, "[MySQL] ", log.Ldate|log.Ltime|log.Lshortfile) +var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) // Logger is used to log critical error messages. type Logger interface { @@ -58,74 +63,3 @@ type MySQLError struct { func (me *MySQLError) Error() string { return fmt.Sprintf("Error %d: %s", me.Number, me.Message) } - -// MySQLWarnings is an error type which represents a group of one or more MySQL -// warnings -type MySQLWarnings []MySQLWarning - -func (mws MySQLWarnings) Error() string { - var msg string - for i, warning := range mws { - if i > 0 { - msg += "\r\n" - } - msg += fmt.Sprintf( - "%s %s: %s", - warning.Level, - warning.Code, - warning.Message, - ) - } - return msg -} - -// MySQLWarning is an error type which represents a single MySQL warning. -// Warnings are returned in groups only. See MySQLWarnings -type MySQLWarning struct { - Level string - Code string - Message string -} - -func (mc *mysqlConn) getWarnings() (err error) { - rows, err := mc.Query("SHOW WARNINGS", nil) - if err != nil { - return - } - - var warnings = MySQLWarnings{} - var values = make([]driver.Value, 3) - - for { - err = rows.Next(values) - switch err { - case nil: - warning := MySQLWarning{} - - if raw, ok := values[0].([]byte); ok { - warning.Level = string(raw) - } else { - warning.Level = fmt.Sprintf("%s", values[0]) - } - if raw, ok := values[1].([]byte); ok { - warning.Code = string(raw) - } else { - warning.Code = fmt.Sprintf("%s", values[1]) - } - if raw, ok := values[2].([]byte); ok { - warning.Message = string(raw) - } else { - warning.Message = fmt.Sprintf("%s", values[0]) - } - - warnings = append(warnings, warning) - - case io.EOF: - return warnings - - default: - rows.Close() - return - } - } -} diff --git a/vendor/github.com/go-sql-driver/mysql/errors_test.go b/vendor/github.com/go-sql-driver/mysql/errors_test.go new file mode 100644 index 0000000..96f9126 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/errors_test.go @@ -0,0 +1,42 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "log" + "testing" +) + +func TestErrorsSetLogger(t *testing.T) { + previous := errLog + defer func() { + errLog = previous + }() + + // set up logger + const expected = "prefix: test\n" + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + logger := log.New(buffer, "prefix: ", 0) + + // print + SetLogger(logger) + errLog.Print("test") + + // check result + if actual := buffer.String(); actual != expected { + t.Errorf("expected %q, got %q", expected, actual) + } +} + +func TestErrorsStrictIgnoreNotes(t *testing.T) { + runTests(t, dsn+"&sql_notes=false", func(dbt *DBTest) { + dbt.mustExec("DROP TABLE IF EXISTS does_not_exist") + }) +} diff --git a/vendor/github.com/go-sql-driver/mysql/fields.go b/vendor/github.com/go-sql-driver/mysql/fields.go new file mode 100644 index 0000000..e1e2ece --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/fields.go @@ -0,0 +1,194 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql" + "reflect" +) + +func (mf *mysqlField) typeDatabaseName() string { + switch mf.fieldType { + case fieldTypeBit: + return "BIT" + case fieldTypeBLOB: + if mf.charSet != collations[binaryCollation] { + return "TEXT" + } + return "BLOB" + case fieldTypeDate: + return "DATE" + case fieldTypeDateTime: + return "DATETIME" + case fieldTypeDecimal: + return "DECIMAL" + case fieldTypeDouble: + return "DOUBLE" + case fieldTypeEnum: + return "ENUM" + case fieldTypeFloat: + return "FLOAT" + case fieldTypeGeometry: + return "GEOMETRY" + case fieldTypeInt24: + return "MEDIUMINT" + case fieldTypeJSON: + return "JSON" + case fieldTypeLong: + return "INT" + case fieldTypeLongBLOB: + if mf.charSet != collations[binaryCollation] { + return "LONGTEXT" + } + return "LONGBLOB" + case fieldTypeLongLong: + return "BIGINT" + case fieldTypeMediumBLOB: + if mf.charSet != collations[binaryCollation] { + return "MEDIUMTEXT" + } + return "MEDIUMBLOB" + case fieldTypeNewDate: + return "DATE" + case fieldTypeNewDecimal: + return "DECIMAL" + case fieldTypeNULL: + return "NULL" + case fieldTypeSet: + return "SET" + case fieldTypeShort: + return "SMALLINT" + case fieldTypeString: + if mf.charSet == collations[binaryCollation] { + return "BINARY" + } + return "CHAR" + case fieldTypeTime: + return "TIME" + case fieldTypeTimestamp: + return "TIMESTAMP" + case fieldTypeTiny: + return "TINYINT" + case fieldTypeTinyBLOB: + if mf.charSet != collations[binaryCollation] { + return "TINYTEXT" + } + return "TINYBLOB" + case fieldTypeVarChar: + if mf.charSet == collations[binaryCollation] { + return "VARBINARY" + } + return "VARCHAR" + case fieldTypeVarString: + if mf.charSet == collations[binaryCollation] { + return "VARBINARY" + } + return "VARCHAR" + case fieldTypeYear: + return "YEAR" + default: + return "" + } +} + +var ( + scanTypeFloat32 = reflect.TypeOf(float32(0)) + scanTypeFloat64 = reflect.TypeOf(float64(0)) + scanTypeInt8 = reflect.TypeOf(int8(0)) + scanTypeInt16 = reflect.TypeOf(int16(0)) + scanTypeInt32 = reflect.TypeOf(int32(0)) + scanTypeInt64 = reflect.TypeOf(int64(0)) + scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) + scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) + scanTypeNullTime = reflect.TypeOf(NullTime{}) + scanTypeUint8 = reflect.TypeOf(uint8(0)) + scanTypeUint16 = reflect.TypeOf(uint16(0)) + scanTypeUint32 = reflect.TypeOf(uint32(0)) + scanTypeUint64 = reflect.TypeOf(uint64(0)) + scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) + scanTypeUnknown = reflect.TypeOf(new(interface{})) +) + +type mysqlField struct { + tableName string + name string + length uint32 + flags fieldFlag + fieldType fieldType + decimals byte + charSet uint8 +} + +func (mf *mysqlField) scanType() reflect.Type { + switch mf.fieldType { + case fieldTypeTiny: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint8 + } + return scanTypeInt8 + } + return scanTypeNullInt + + case fieldTypeShort, fieldTypeYear: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint16 + } + return scanTypeInt16 + } + return scanTypeNullInt + + case fieldTypeInt24, fieldTypeLong: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint32 + } + return scanTypeInt32 + } + return scanTypeNullInt + + case fieldTypeLongLong: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint64 + } + return scanTypeInt64 + } + return scanTypeNullInt + + case fieldTypeFloat: + if mf.flags&flagNotNULL != 0 { + return scanTypeFloat32 + } + return scanTypeNullFloat + + case fieldTypeDouble: + if mf.flags&flagNotNULL != 0 { + return scanTypeFloat64 + } + return scanTypeNullFloat + + case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, + fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, + fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, + fieldTypeTime: + return scanTypeRawBytes + + case fieldTypeDate, fieldTypeNewDate, + fieldTypeTimestamp, fieldTypeDateTime: + // NullTime is always returned for more consistent behavior as it can + // handle both cases of parseTime regardless if the field is nullable. + return scanTypeNullTime + + default: + return scanTypeUnknown + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/infile.go b/vendor/github.com/go-sql-driver/mysql/infile.go index 84c53a9..273cb0b 100644 --- a/vendor/github.com/go-sql-driver/mysql/infile.go +++ b/vendor/github.com/go-sql-driver/mysql/infile.go @@ -96,6 +96,10 @@ func deferredClose(err *error, closer io.Closer) { func (mc *mysqlConn) handleInFileRequest(name string) (err error) { var rdr io.Reader var data []byte + packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP + if mc.maxWriteSize < packetSize { + packetSize = mc.maxWriteSize + } if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader // The server might return an an absolute path. See issue #355. @@ -108,8 +112,6 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { if inMap { rdr = handler() if rdr != nil { - data = make([]byte, 4+mc.maxWriteSize) - if cl, ok := rdr.(io.Closer); ok { defer deferredClose(&err, cl) } @@ -124,7 +126,7 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { fileRegisterLock.RLock() fr := fileRegister[name] fileRegisterLock.RUnlock() - if mc.cfg.allowAllFiles || fr { + if mc.cfg.AllowAllFiles || fr { var file *os.File var fi os.FileInfo @@ -134,22 +136,20 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // get file size if fi, err = file.Stat(); err == nil { rdr = file - if fileSize := int(fi.Size()); fileSize <= mc.maxWriteSize { - data = make([]byte, 4+fileSize) - } else if fileSize <= mc.maxPacketAllowed { - data = make([]byte, 4+mc.maxWriteSize) - } else { - err = fmt.Errorf("Local File '%s' too large: Size: %d, Max: %d", name, fileSize, mc.maxPacketAllowed) + if fileSize := int(fi.Size()); fileSize < packetSize { + packetSize = fileSize } } } } else { - err = fmt.Errorf("Local File '%s' is not registered. Use the DSN parameter 'allowAllFiles=true' to allow all files", name) + err = fmt.Errorf("local file '%s' is not registered", name) } } // send content packets - if err == nil { + // if packetSize == 0, the Reader contains no data + if err == nil && packetSize > 0 { + data := make([]byte, 4+packetSize) var n int for err == nil { n, err = rdr.Read(data[4:]) @@ -175,8 +175,8 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // read OK packet if err == nil { return mc.readResultOK() - } else { - mc.readPacket() } + + mc.readPacket() return err } diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index 76cb7c8..30b3352 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -13,6 +13,7 @@ import ( "crypto/tls" "database/sql/driver" "encoding/binary" + "errors" "fmt" "io" "math" @@ -24,55 +25,66 @@ import ( // Read packet to buffer 'data' func (mc *mysqlConn) readPacket() ([]byte, error) { - var payload []byte + var prevData []byte for { - // Read packet header + // read packet header data, err := mc.buf.readNext(4) if err != nil { + if cerr := mc.canceled.Value(); cerr != nil { + return nil, cerr + } errLog.Print(err) mc.Close() - return nil, driver.ErrBadConn + return nil, ErrInvalidConn } - // Packet Length [24 bit] + // packet length [24 bit] pktLen := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) - if pktLen < 1 { - errLog.Print(ErrMalformPkt) - mc.Close() - return nil, driver.ErrBadConn - } - - // Check Packet Sync [8 bit] + // check packet sync [8 bit] if data[3] != mc.sequence { if data[3] > mc.sequence { return nil, ErrPktSyncMul - } else { - return nil, ErrPktSync } + return nil, ErrPktSync } mc.sequence++ - // Read packet body [pktLen bytes] + // packets with length 0 terminate a previous packet which is a + // multiple of (2^24)-1 bytes long + if pktLen == 0 { + // there was no previous packet + if prevData == nil { + errLog.Print(ErrMalformPkt) + mc.Close() + return nil, ErrInvalidConn + } + + return prevData, nil + } + + // read packet body [pktLen bytes] data, err = mc.buf.readNext(pktLen) if err != nil { + if cerr := mc.canceled.Value(); cerr != nil { + return nil, cerr + } errLog.Print(err) mc.Close() - return nil, driver.ErrBadConn + return nil, ErrInvalidConn } - isLastPacket := (pktLen < maxPacketSize) + // return data if this was the last packet + if pktLen < maxPacketSize { + // zero allocations for non-split packets + if prevData == nil { + return data, nil + } - // Zero allocations for non-splitting packets - if isLastPacket && payload == nil { - return data, nil + return append(prevData, data...), nil } - payload = append(payload, data...) - - if isLastPacket { - return payload, nil - } + prevData = append(prevData, data...) } } @@ -80,10 +92,39 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { func (mc *mysqlConn) writePacket(data []byte) error { pktLen := len(data) - 4 - if pktLen > mc.maxPacketAllowed { + if pktLen > mc.maxAllowedPacket { return ErrPktTooLarge } + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.reset { + mc.reset = false + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + // If this connection has a ReadTimeout which we've been setting on + // reads, reset it to its default value before we attempt a non-blocking + // read, otherwise the scheduler will just time us out before we can read + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Time{}) + } + if err == nil { + err = connCheck(conn) + } + if err != nil { + errLog.Print("closing bad idle connection: ", err) + mc.Close() + return driver.ErrBadConn + } + } + for { var size int if pktLen >= maxPacketSize { @@ -100,6 +141,12 @@ func (mc *mysqlConn) writePacket(data []byte) error { data[3] = mc.sequence // Write packet + if mc.writeTimeout > 0 { + if err := mc.netConn.SetWriteDeadline(time.Now().Add(mc.writeTimeout)); err != nil { + return err + } + } + n, err := mc.netConn.Write(data[:4+size]) if err == nil && n == 4+size { mc.sequence++ @@ -113,34 +160,48 @@ func (mc *mysqlConn) writePacket(data []byte) error { // Handle error if err == nil { // n != len(data) + mc.cleanup() errLog.Print(ErrMalformPkt) } else { + if cerr := mc.canceled.Value(); cerr != nil { + return cerr + } + if n == 0 && pktLen == len(data)-4 { + // only for the first loop iteration when nothing was written yet + return errBadConnNoWrite + } + mc.cleanup() errLog.Print(err) } - return driver.ErrBadConn + return ErrInvalidConn } } /****************************************************************************** -* Initialisation Process * +* Initialization Process * ******************************************************************************/ // Handshake Initialization Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake -func (mc *mysqlConn) readInitPacket() ([]byte, error) { - data, err := mc.readPacket() +func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) { + data, err = mc.readPacket() if err != nil { - return nil, err + // for init we can rewrite this to ErrBadConn for sql.Driver to retry, since + // in connection initialization we don't risk retrying non-idempotent actions. + if err == ErrInvalidConn { + return nil, "", driver.ErrBadConn + } + return } if data[0] == iERR { - return nil, mc.handleErrorPacket(data) + return nil, "", mc.handleErrorPacket(data) } // protocol version [1 byte] if data[0] < minProtocolVersion { - return nil, fmt.Errorf( - "Unsupported MySQL Protocol Version %d. Protocol Version %d or higher is required", + return nil, "", fmt.Errorf( + "unsupported protocol version %d. Version %d or higher is required", data[0], minProtocolVersion, ) @@ -151,7 +212,7 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { pos := 1 + bytes.IndexByte(data[1:], 0x00) + 1 + 4 // first part of the password cipher [8 bytes] - cipher := data[pos : pos+8] + authData := data[pos : pos+8] // (filler) always 0x00 [1 byte] pos += 8 + 1 @@ -159,10 +220,14 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // capability flags (lower 2 bytes) [2 bytes] mc.flags = clientFlag(binary.LittleEndian.Uint16(data[pos : pos+2])) if mc.flags&clientProtocol41 == 0 { - return nil, ErrOldProtocol + return nil, "", ErrOldProtocol } if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - return nil, ErrNoTLS + if mc.cfg.TLSConfig == "preferred" { + mc.cfg.tls = nil + } else { + return nil, "", ErrNoTLS + } } pos += 2 @@ -186,32 +251,32 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // // The official Python library uses the fixed length 12 // which seems to work but technically could have a hidden bug. - cipher = append(cipher, data[pos:pos+12]...) + authData = append(authData, data[pos:pos+12]...) + pos += 13 - // TODO: Verify string termination // EOF if version (>= 5.5.7 and < 5.5.10) or (>= 5.6.0 and < 5.6.2) // \NUL otherwise - // - //if data[len(data)-1] == 0 { - // return - //} - //return ErrMalformPkt + if end := bytes.IndexByte(data[pos:], 0x00); end != -1 { + plugin = string(data[pos : pos+end]) + } else { + plugin = string(data[pos:]) + } // make a memory safe copy of the cipher slice var b [20]byte - copy(b[:], cipher) - return b[:], nil + copy(b[:], authData) + return b[:], plugin, nil } // make a memory safe copy of the cipher slice var b [8]byte - copy(b[:], cipher) - return b[:], nil + copy(b[:], authData) + return b[:], plugin, nil } // Client Authentication Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse -func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { +func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error { // Adjust client flags based on server support clientFlags := clientProtocol41 | clientSecureConn | @@ -219,9 +284,10 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientTransactions | clientLocalFiles | clientPluginAuth | + clientMultiResults | mc.flags&clientLongFlag - if mc.cfg.clientFoundRows { + if mc.cfg.ClientFoundRows { clientFlags |= clientFoundRows } @@ -230,23 +296,34 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientFlags |= clientSSL } - // User Password - scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.passwd)) + if mc.cfg.MultiStatements { + clientFlags |= clientMultiStatements + } - pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.user) + 1 + 1 + len(scrambleBuff) + 21 + 1 + // encode length of the auth plugin data + var authRespLEIBuf [9]byte + authRespLen := len(authResp) + authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen)) + if len(authRespLEI) > 1 { + // if the length can not be written in 1 byte, it must be written as a + // length encoded integer + clientFlags |= clientPluginAuthLenEncClientData + } + + pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1 // To specify a db name - if n := len(mc.cfg.dbname); n > 0 { + if n := len(mc.cfg.DBName); n > 0 { clientFlags |= clientConnectWithDB pktLen += n + 1 } // Calculate packet length and get buffer with that size - data := mc.buf.takeSmallBuffer(pktLen + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeSmallBuffer(pktLen + 4) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // ClientFlags [32 bit] @@ -262,7 +339,14 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { data[11] = 0x00 // Charset [1 byte] - data[12] = mc.cfg.collation + var found bool + data[12], found = collations[mc.cfg.Collation] + if !found { + // Note possibility for false negatives: + // could be triggered although the collation is valid if the + // collations map does not contain entries the server supports. + return errors.New("unknown collation") + } // SSL Connection Request Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest @@ -277,8 +361,9 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { if err := tlsConn.Handshake(); err != nil { return err } + mc.rawConn = mc.netConn mc.netConn = tlsConn - mc.buf.rd = tlsConn + mc.buf.nc = tlsConn } // Filler [23 bytes] (all 0x00) @@ -288,69 +373,43 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { } // User [null terminated string] - if len(mc.cfg.user) > 0 { - pos += copy(data[pos:], mc.cfg.user) + if len(mc.cfg.User) > 0 { + pos += copy(data[pos:], mc.cfg.User) } data[pos] = 0x00 pos++ - // ScrambleBuffer [length encoded integer] - data[pos] = byte(len(scrambleBuff)) - pos += 1 + copy(data[pos+1:], scrambleBuff) + // Auth Data [length encoded integer] + pos += copy(data[pos:], authRespLEI) + pos += copy(data[pos:], authResp) // Databasename [null terminated string] - if len(mc.cfg.dbname) > 0 { - pos += copy(data[pos:], mc.cfg.dbname) + if len(mc.cfg.DBName) > 0 { + pos += copy(data[pos:], mc.cfg.DBName) data[pos] = 0x00 pos++ } - // Assume native client during response - pos += copy(data[pos:], "mysql_native_password") + pos += copy(data[pos:], plugin) data[pos] = 0x00 + pos++ // Send Auth packet - return mc.writePacket(data) + return mc.writePacket(data[:pos]) } -// Client old authentication packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeOldAuthPacket(cipher []byte) error { - // User password - scrambleBuff := scrambleOldPassword(cipher, []byte(mc.cfg.passwd)) - - // Calculate the packet length and add a tailing 0 - pktLen := len(scrambleBuff) + 1 - data := mc.buf.takeSmallBuffer(4 + pktLen) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn +func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { + pktLen := 4 + len(authData) + data, err := mc.buf.takeSmallBuffer(pktLen) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } - // Add the scrambled password [null terminated string] - copy(data[4:], scrambleBuff) - data[4+pktLen-1] = 0x00 - - return mc.writePacket(data) -} - -// Client clear text authentication packet -// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeClearAuthPacket() error { - // Calculate the packet length and add a tailing 0 - pktLen := len(mc.cfg.passwd) + 1 - data := mc.buf.takeSmallBuffer(4 + pktLen) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // Add the clear password [null terminated string] - copy(data[4:], mc.cfg.passwd) - data[4+pktLen-1] = 0x00 - + // Add the auth data [EOF] + copy(data[4:], authData) return mc.writePacket(data) } @@ -362,11 +421,11 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // Add command byte @@ -381,11 +440,11 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { mc.sequence = 0 pktLen := 1 + len(arg) - data := mc.buf.takeBuffer(pktLen + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeBuffer(pktLen + 4) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // Add command byte @@ -402,11 +461,11 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1 + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // Add command byte @@ -426,37 +485,50 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { * Result Packets * ******************************************************************************/ +func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { + data, err := mc.readPacket() + if err != nil { + return nil, "", err + } + + // packet indicator + switch data[0] { + + case iOK: + return nil, "", mc.handleOkPacket(data) + + case iAuthMoreData: + return data[1:], "", err + + case iEOF: + if len(data) == 1 { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest + return nil, "mysql_old_password", nil + } + pluginEndIndex := bytes.IndexByte(data, 0x00) + if pluginEndIndex < 0 { + return nil, "", ErrMalformPkt + } + plugin := string(data[1:pluginEndIndex]) + authData := data[pluginEndIndex+1:] + return authData, plugin, nil + + default: // Error otherwise + return nil, "", mc.handleErrorPacket(data) + } +} + // Returns error if Packet is not an 'Result OK'-Packet func (mc *mysqlConn) readResultOK() error { data, err := mc.readPacket() - if err == nil { - // packet indicator - switch data[0] { - - case iOK: - return mc.handleOkPacket(data) - - case iEOF: - if len(data) > 1 { - plugin := string(data[1:bytes.IndexByte(data, 0x00)]) - if plugin == "mysql_old_password" { - // using old_passwords - return ErrOldPassword - } else if plugin == "mysql_clear_password" { - // using clear text password - return ErrCleartextPassword - } else { - return ErrUnknownPlugin - } - } else { - return ErrOldPassword - } - - default: // Error otherwise - return mc.handleErrorPacket(data) - } + if err != nil { + return err } - return err + + if data[0] == iOK { + return mc.handleOkPacket(data) + } + return mc.handleErrorPacket(data) } // Result Set Header Packet @@ -499,6 +571,22 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { // Error Number [16 bit uint] errno := binary.LittleEndian.Uint16(data[1:3]) + // 1792: ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION + // 1290: ER_OPTION_PREVENTS_STATEMENT (returned by Aurora during failover) + if (errno == 1792 || errno == 1290) && mc.cfg.RejectReadOnly { + // Oops; we are connected to a read-only connection, and won't be able + // to issue any write statements. Since RejectReadOnly is configured, + // we throw away this connection hoping this one would have write + // permission. This is specifically for a possible race condition + // during failover (e.g. on AWS Aurora). See README.md for more. + // + // We explicitly close the connection before returning + // driver.ErrBadConn to ensure that `database/sql` purges this + // connection and initiates a new one for next statement next time. + mc.Close() + return driver.ErrBadConn + } + pos := 3 // SQL State [optional: # + 5bytes string] @@ -514,6 +602,10 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { } } +func readStatus(b []byte) statusFlag { + return statusFlag(b[0]) | statusFlag(b[1])<<8 +} + // Ok Packet // http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet func (mc *mysqlConn) handleOkPacket(data []byte) error { @@ -528,18 +620,14 @@ func (mc *mysqlConn) handleOkPacket(data []byte) error { mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) // server_status [2 bytes] - mc.status = statusFlag(data[1+n+m]) | statusFlag(data[1+n+m+1])<<8 - - // warning count [2 bytes] - if !mc.strict { - return nil - } else { - pos := 1 + n + m + 2 - if binary.LittleEndian.Uint16(data[pos:pos+2]) > 0 { - return mc.getWarnings() - } + mc.status = readStatus(data[1+n+m : 1+n+m+2]) + if mc.status&statusMoreResultsExists != 0 { return nil } + + // warning count [2 bytes] + + return nil } // Read Packets as Field Packets until EOF-Packet or an Error appears @@ -558,7 +646,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { if i == count { return columns, nil } - return nil, fmt.Errorf("ColumnsCount mismatch n:%d len:%d", count, len(columns)) + return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns)) } // Catalog @@ -575,7 +663,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { pos += n // Table [len coded string] - if mc.cfg.columnsWithAlias { + if mc.cfg.ColumnsWithAlias { tableName, _, n, err := readLengthEncodedString(data[pos:]) if err != nil { return nil, err @@ -610,14 +698,21 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { if err != nil { return nil, err } + pos += n // Filler [uint8] + pos++ + // Charset [charset, collation uint8] + columns[i].charSet = data[pos] + pos += 2 + // Length [uint32] - pos += n + 1 + 2 + 4 + columns[i].length = binary.LittleEndian.Uint32(data[pos : pos+4]) + pos += 4 // Field type [uint8] - columns[i].fieldType = data[pos] + columns[i].fieldType = fieldType(data[pos]) pos++ // Flags [uint16] @@ -640,6 +735,10 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { func (rows *textRows) readRow(dest []driver.Value) error { mc := rows.mc + if rows.rs.done { + return io.EOF + } + data, err := mc.readPacket() if err != nil { return err @@ -647,7 +746,12 @@ func (rows *textRows) readRow(dest []driver.Value) error { // EOF Packet if data[0] == iEOF && len(data) == 5 { - rows.mc = nil + // server_status [2 bytes] + rows.mc.status = readStatus(data[3:]) + rows.rs.done = true + if !rows.HasNextResultSet() { + rows.mc = nil + } return io.EOF } if data[0] == iERR { @@ -669,12 +773,12 @@ func (rows *textRows) readRow(dest []driver.Value) error { if !mc.parseTime { continue } else { - switch rows.columns[i].fieldType { + switch rows.rs.columns[i].fieldType { case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeDate, fieldTypeNewDate: dest[i], err = parseDateTime( string(dest[i].([]byte)), - mc.cfg.loc, + mc.cfg.Loc, ) if err == nil { continue @@ -699,12 +803,19 @@ func (rows *textRows) readRow(dest []driver.Value) error { func (mc *mysqlConn) readUntilEOF() error { for { data, err := mc.readPacket() - - // No Err and no EOF Packet - if err == nil && data[0] != iEOF { - continue + if err != nil { + return err + } + + switch data[0] { + case iERR: + return mc.handleErrorPacket(data) + case iEOF: + if len(data) == 5 { + mc.status = readStatus(data[3:]) + } + return nil } - return err // Err or EOF } } @@ -734,22 +845,15 @@ func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) { // Reserved [8 bit] // Warning count [16 bit uint] - if !stmt.mc.strict { - return columnCount, nil - } else { - // Check for warnings count > 0, only available in MySQL > 4.1 - if len(data) >= 12 && binary.LittleEndian.Uint16(data[10:12]) > 0 { - return columnCount, stmt.mc.getWarnings() - } - return columnCount, nil - } + + return columnCount, nil } return 0, err } // http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { - maxLen := stmt.mc.maxPacketAllowed - 1 + maxLen := stmt.mc.maxAllowedPacket - 1 pktLen := maxLen // After the header (bytes 0-3) follows before the data: @@ -758,7 +862,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { // 2 bytes paramID const dataOffset = 1 + 4 + 2 - // Can not use the write buffer since + // Cannot use the write buffer since // a) the buffer is too small // b) it is in use data := make([]byte, 4+1+4+2+len(arg)) @@ -804,7 +908,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if len(args) != stmt.paramCount { return fmt.Errorf( - "Arguments count mismatch (Got: %d Has: %d)", + "argument count mismatch (got: %d; has: %d)", len(args), stmt.paramCount, ) @@ -813,20 +917,28 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { const minPktLen = 4 + 1 + 4 + 1 + 4 mc := stmt.mc + // Determine threshold dynamically to avoid packet size shortage. + longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1) + if longDataSize < 64 { + longDataSize = 64 + } + // Reset packet-sequence mc.sequence = 0 var data []byte + var err error if len(args) == 0 { - data = mc.buf.takeBuffer(minPktLen) + data, err = mc.buf.takeBuffer(minPktLen) } else { - data = mc.buf.takeCompleteBuffer() + data, err = mc.buf.takeCompleteBuffer() + // In this case the len(data) == cap(data) which is used to optimise the flow below. } - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // command [1 byte] @@ -851,7 +963,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { pos := minPktLen var nullMask []byte - if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { + if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= cap(data) { // buffer has to be extended but we don't know by how much so // we depend on append after all data with known sizes fit. // We stop at that because we deal with a lot of columns here @@ -860,10 +972,11 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { copy(tmp[:pos], data[:pos]) data = tmp nullMask = data[pos : pos+maskLen] + // No need to clean nullMask as make ensures that. pos += maskLen } else { nullMask = data[pos : pos+maskLen] - for i := 0; i < maskLen; i++ { + for i := range nullMask { nullMask[i] = 0 } pos += maskLen @@ -885,7 +998,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // build NULL-bitmap if arg == nil { nullMask[i/8] |= 1 << (uint(i) & 7) - paramTypes[i+i] = fieldTypeNULL + paramTypes[i+i] = byte(fieldTypeNULL) paramTypes[i+i+1] = 0x00 continue } @@ -893,7 +1006,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // cache types and values switch v := arg.(type) { case int64: - paramTypes[i+i] = fieldTypeLongLong + paramTypes[i+i] = byte(fieldTypeLongLong) paramTypes[i+i+1] = 0x00 if cap(paramValues)-len(paramValues)-8 >= 0 { @@ -908,8 +1021,24 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { ) } + case uint64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x80 // type is unsigned + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + case float64: - paramTypes[i+i] = fieldTypeDouble + paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 if cap(paramValues)-len(paramValues)-8 >= 0 { @@ -925,7 +1054,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { } case bool: - paramTypes[i+i] = fieldTypeTiny + paramTypes[i+i] = byte(fieldTypeTiny) paramTypes[i+i+1] = 0x00 if v { @@ -937,10 +1066,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { case []byte: // Common case (non-nil value) first if v != nil { - paramTypes[i+i] = fieldTypeString + paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < longDataSize { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -955,14 +1084,14 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // Handle []byte(nil) as a NULL value nullMask[i/8] |= 1 << (uint(i) & 7) - paramTypes[i+i] = fieldTypeNULL + paramTypes[i+i] = byte(fieldTypeNULL) paramTypes[i+i+1] = 0x00 case string: - paramTypes[i+i] = fieldTypeString + paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < longDataSize { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -974,23 +1103,25 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { } case time.Time: - paramTypes[i+i] = fieldTypeString + paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 - var val []byte + var a [64]byte + var b = a[:0] + if v.IsZero() { - val = []byte("0000-00-00") + b = append(b, "0000-00-00"...) } else { - val = []byte(v.In(mc.cfg.loc).Format(timeFormat)) + b = v.In(mc.cfg.Loc).AppendFormat(b, timeFormat) } paramValues = appendLengthEncodedInteger(paramValues, - uint64(len(val)), + uint64(len(b)), ) - paramValues = append(paramValues, val...) + paramValues = append(paramValues, b...) default: - return fmt.Errorf("Can't convert type: %T", arg) + return fmt.Errorf("cannot convert type: %T", arg) } } @@ -998,7 +1129,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In that case we must build the data packet with the new values buffer if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) - mc.buf.buf = data + if err = mc.buf.store(data); err != nil { + errLog.Print(err) + return errBadConnNoWrite + } } pos += len(paramValues) @@ -1008,6 +1142,26 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { return mc.writePacket(data) } +func (mc *mysqlConn) discardResults() error { + for mc.status&statusMoreResultsExists != 0 { + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return err + } + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { + return err + } + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } + } + } + return nil +} + // http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html func (rows *binaryRows) readRow(dest []driver.Value) error { data, err := rows.mc.readPacket() @@ -1017,14 +1171,20 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // packet indicator [1 byte] if data[0] != iOK { - rows.mc = nil // EOF Packet if data[0] == iEOF && len(data) == 5 { + rows.mc.status = readStatus(data[3:]) + rows.rs.done = true + if !rows.HasNextResultSet() { + rows.mc = nil + } return io.EOF } + mc := rows.mc + rows.mc = nil // Error otherwise - return rows.mc.handleErrorPacket(data) + return mc.handleErrorPacket(data) } // NULL-bitmap, [(column-count + 7 + 2) / 8 bytes] @@ -1040,14 +1200,14 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { } // Convert to byte-coded string - switch rows.columns[i].fieldType { + switch rows.rs.columns[i].fieldType { case fieldTypeNULL: dest[i] = nil continue // Numeric Types case fieldTypeTiny: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { dest[i] = int64(data[pos]) } else { dest[i] = int64(int8(data[pos])) @@ -1056,7 +1216,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeShort, fieldTypeYear: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { dest[i] = int64(binary.LittleEndian.Uint16(data[pos : pos+2])) } else { dest[i] = int64(int16(binary.LittleEndian.Uint16(data[pos : pos+2]))) @@ -1065,7 +1225,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeInt24, fieldTypeLong: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { dest[i] = int64(binary.LittleEndian.Uint32(data[pos : pos+4])) } else { dest[i] = int64(int32(binary.LittleEndian.Uint32(data[pos : pos+4]))) @@ -1074,7 +1234,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeLongLong: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { val := binary.LittleEndian.Uint64(data[pos : pos+8]) if val > math.MaxInt64 { dest[i] = uint64ToString(val) @@ -1088,7 +1248,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeFloat: - dest[i] = float64(math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4]))) + dest[i] = math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4])) pos += 4 continue @@ -1101,7 +1261,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry: + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON: var isNull bool var n int dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) @@ -1128,41 +1288,41 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { case isNull: dest[i] = nil continue - case rows.columns[i].fieldType == fieldTypeTime: + case rows.rs.columns[i].fieldType == fieldTypeTime: // database/sql does not support an equivalent to TIME, return a string var dstlen uint8 - switch decimals := rows.columns[i].decimals; decimals { + switch decimals := rows.rs.columns[i].decimals; decimals { case 0x00, 0x1f: dstlen = 8 case 1, 2, 3, 4, 5, 6: dstlen = 8 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", - rows.columns[i].decimals, + "protocol error, illegal decimals value %d", + rows.rs.columns[i].decimals, ) } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, true) + dest[i], err = formatBinaryTime(data[pos:pos+int(num)], dstlen) case rows.mc.parseTime: - dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.loc) + dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.Loc) default: var dstlen uint8 - if rows.columns[i].fieldType == fieldTypeDate { + if rows.rs.columns[i].fieldType == fieldTypeDate { dstlen = 10 } else { - switch decimals := rows.columns[i].decimals; decimals { + switch decimals := rows.rs.columns[i].decimals; decimals { case 0x00, 0x1f: dstlen = 19 case 1, 2, 3, 4, 5, 6: dstlen = 19 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", - rows.columns[i].decimals, + "protocol error, illegal decimals value %d", + rows.rs.columns[i].decimals, ) } } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, false) + dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen) } if err == nil { @@ -1174,7 +1334,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // Please report if this happens! default: - return fmt.Errorf("Unknown FieldType %d", rows.columns[i].fieldType) + return fmt.Errorf("unknown field type %d", rows.rs.columns[i].fieldType) } } diff --git a/vendor/github.com/go-sql-driver/mysql/packets_test.go b/vendor/github.com/go-sql-driver/mysql/packets_test.go new file mode 100644 index 0000000..b61e4db --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/packets_test.go @@ -0,0 +1,336 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "errors" + "net" + "testing" + "time" +) + +var ( + errConnClosed = errors.New("connection is closed") + errConnTooManyReads = errors.New("too many reads") + errConnTooManyWrites = errors.New("too many writes") +) + +// struct to mock a net.Conn for testing purposes +type mockConn struct { + laddr net.Addr + raddr net.Addr + data []byte + written []byte + queuedReplies [][]byte + closed bool + read int + reads int + writes int + maxReads int + maxWrites int +} + +func (m *mockConn) Read(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.reads++ + if m.maxReads > 0 && m.reads > m.maxReads { + return 0, errConnTooManyReads + } + + n = copy(b, m.data) + m.read += n + m.data = m.data[n:] + return +} +func (m *mockConn) Write(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.writes++ + if m.maxWrites > 0 && m.writes > m.maxWrites { + return 0, errConnTooManyWrites + } + + n = len(b) + m.written = append(m.written, b...) + + if n > 0 && len(m.queuedReplies) > 0 { + m.data = m.queuedReplies[0] + m.queuedReplies = m.queuedReplies[1:] + } + return +} +func (m *mockConn) Close() error { + m.closed = true + return nil +} +func (m *mockConn) LocalAddr() net.Addr { + return m.laddr +} +func (m *mockConn) RemoteAddr() net.Addr { + return m.raddr +} +func (m *mockConn) SetDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetReadDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetWriteDeadline(t time.Time) error { + return nil +} + +// make sure mockConn implements the net.Conn interface +var _ net.Conn = new(mockConn) + +func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: NewConfig(), + netConn: conn, + closech: make(chan struct{}), + maxAllowedPacket: defaultMaxAllowedPacket, + sequence: sequence, + } + return conn, mc +} + +func TestReadPacketSingleByte(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 1 { + t.Fatalf("unexpected packet length: expected %d, got %d", 1, len(packet)) + } + if packet[0] != 0xff { + t.Fatalf("unexpected packet content: expected %x, got %x", 0xff, packet[0]) + } +} + +func TestReadPacketWrongSequenceID(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + // too low sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + mc.sequence = 1 + _, err := mc.readPacket() + if err != ErrPktSync { + t.Errorf("expected ErrPktSync, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // too high sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x42, 0xff} + _, err = mc.readPacket() + if err != ErrPktSyncMul { + t.Errorf("expected ErrPktSyncMul, got %v", err) + } +} + +func TestReadPacketSplit(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + data := make([]byte, maxPacketSize*2+4*3) + const pkt2ofs = maxPacketSize + 4 + const pkt3ofs = 2 * (maxPacketSize + 4) + + // case 1: payload has length maxPacketSize + data = data[:pkt2ofs+4] + + // 1st packet has maxPacketSize length and sequence id 0 + // ff ff ff 00 ... + data[0] = 0xff + data[1] = 0xff + data[2] = 0xff + + // mark the payload start and end of 1st packet so that we can check if the + // content was correctly appended + data[4] = 0x11 + data[maxPacketSize+3] = 0x22 + + // 2nd packet has payload length 0 and squence id 1 + // 00 00 00 01 + data[pkt2ofs+3] = 0x01 + + conn.data = data + conn.maxReads = 3 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize-1] != 0x22 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x22, packet[maxPacketSize-1]) + } + + // case 2: payload has length which is a multiple of maxPacketSize + data = data[:cap(data)] + + // 2nd packet now has maxPacketSize length + data[pkt2ofs] = 0xff + data[pkt2ofs+1] = 0xff + data[pkt2ofs+2] = 0xff + + // mark the payload start and end of the 2nd packet + data[pkt2ofs+4] = 0x33 + data[pkt2ofs+maxPacketSize+3] = 0x44 + + // 3rd packet has payload length 0 and squence id 2 + // 00 00 00 02 + data[pkt3ofs+3] = 0x02 + + conn.data = data + conn.reads = 0 + conn.maxReads = 5 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 2*maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", 2*maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[2*maxPacketSize-1] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[2*maxPacketSize-1]) + } + + // case 3: payload has a length larger maxPacketSize, which is not an exact + // multiple of it + data = data[:pkt2ofs+4+42] + data[pkt2ofs] = 0x2a + data[pkt2ofs+1] = 0x00 + data[pkt2ofs+2] = 0x00 + data[pkt2ofs+4+41] = 0x44 + + conn.data = data + conn.reads = 0 + conn.maxReads = 4 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize+42 { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize+42, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize+41] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[maxPacketSize+41]) + } +} + +func TestReadPacketFail(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + closech: make(chan struct{}), + } + + // illegal empty (stand-alone) packet + conn.data = []byte{0x00, 0x00, 0x00, 0x00} + conn.maxReads = 1 + _, err := mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read header + conn.closed = true + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.closed = false + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read body + conn.maxReads = 1 + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } +} + +// https://github.com/go-sql-driver/mysql/pull/801 +// not-NUL terminated plugin_name in init packet +func TestRegression801(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: new(Config), + sequence: 42, + closech: make(chan struct{}), + } + + conn.data = []byte{72, 0, 0, 42, 10, 53, 46, 53, 46, 56, 0, 165, 0, 0, 0, + 60, 70, 63, 58, 68, 104, 34, 97, 0, 223, 247, 33, 2, 0, 15, 128, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 120, 114, 47, 85, 75, 109, 99, 51, 77, + 50, 64, 0, 109, 121, 115, 113, 108, 95, 110, 97, 116, 105, 118, 101, 95, + 112, 97, 115, 115, 119, 111, 114, 100} + conn.maxReads = 1 + + authData, pluginName, err := mc.readHandshakePacket() + if err != nil { + t.Fatalf("got error: %v", err) + } + + if pluginName != "mysql_native_password" { + t.Errorf("expected plugin name 'mysql_native_password', got '%s'", pluginName) + } + + expectedAuthData := []byte{60, 70, 63, 58, 68, 104, 34, 97, 98, 120, 114, + 47, 85, 75, 109, 99, 51, 77, 50, 64} + if !bytes.Equal(authData, expectedAuthData) { + t.Errorf("expected authData '%v', got '%v'", expectedAuthData, authData) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/go-sql-driver/mysql/rows.go index ba606e1..888bdb5 100644 --- a/vendor/github.com/go-sql-driver/mysql/rows.go +++ b/vendor/github.com/go-sql-driver/mysql/rows.go @@ -11,19 +11,20 @@ package mysql import ( "database/sql/driver" "io" + "math" + "reflect" ) -type mysqlField struct { - tableName string - name string - flags fieldFlag - fieldType byte - decimals byte +type resultSet struct { + columns []mysqlField + columnNames []string + done bool } type mysqlRows struct { - mc *mysqlConn - columns []mysqlField + mc *mysqlConn + rs resultSet + finish func() } type binaryRows struct { @@ -34,45 +35,163 @@ type textRows struct { mysqlRows } -type emptyRows struct{} - func (rows *mysqlRows) Columns() []string { - columns := make([]string, len(rows.columns)) - if rows.mc.cfg.columnsWithAlias { + if rows.rs.columnNames != nil { + return rows.rs.columnNames + } + + columns := make([]string, len(rows.rs.columns)) + if rows.mc != nil && rows.mc.cfg.ColumnsWithAlias { for i := range columns { - if tableName := rows.columns[i].tableName; len(tableName) > 0 { - columns[i] = tableName + "." + rows.columns[i].name + if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 { + columns[i] = tableName + "." + rows.rs.columns[i].name } else { - columns[i] = rows.columns[i].name + columns[i] = rows.rs.columns[i].name } } } else { for i := range columns { - columns[i] = rows.columns[i].name + columns[i] = rows.rs.columns[i].name } } + + rows.rs.columnNames = columns return columns } -func (rows *mysqlRows) Close() error { +func (rows *mysqlRows) ColumnTypeDatabaseTypeName(i int) string { + return rows.rs.columns[i].typeDatabaseName() +} + +// func (rows *mysqlRows) ColumnTypeLength(i int) (length int64, ok bool) { +// return int64(rows.rs.columns[i].length), true +// } + +func (rows *mysqlRows) ColumnTypeNullable(i int) (nullable, ok bool) { + return rows.rs.columns[i].flags&flagNotNULL == 0, true +} + +func (rows *mysqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) { + column := rows.rs.columns[i] + decimals := int64(column.decimals) + + switch column.fieldType { + case fieldTypeDecimal, fieldTypeNewDecimal: + if decimals > 0 { + return int64(column.length) - 2, decimals, true + } + return int64(column.length) - 1, decimals, true + case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeTime: + return decimals, decimals, true + case fieldTypeFloat, fieldTypeDouble: + if decimals == 0x1f { + return math.MaxInt64, math.MaxInt64, true + } + return math.MaxInt64, decimals, true + } + + return 0, 0, false +} + +func (rows *mysqlRows) ColumnTypeScanType(i int) reflect.Type { + return rows.rs.columns[i].scanType() +} + +func (rows *mysqlRows) Close() (err error) { + if f := rows.finish; f != nil { + f() + rows.finish = nil + } + mc := rows.mc if mc == nil { return nil } - if mc.netConn == nil { - return ErrInvalidConn + if err := mc.error(); err != nil { + return err + } + + // flip the buffer for this connection if we need to drain it. + // note that for a successful query (i.e. one where rows.next() + // has been called until it returns false), `rows.mc` will be nil + // by the time the user calls `(*Rows).Close`, so we won't reach this + // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 + mc.buf.flip() + + // Remove unread packets from stream + if !rows.rs.done { + err = mc.readUntilEOF() + } + if err == nil { + if err = mc.discardResults(); err != nil { + return err + } + } + + rows.mc = nil + return err +} + +func (rows *mysqlRows) HasNextResultSet() (b bool) { + if rows.mc == nil { + return false + } + return rows.mc.status&statusMoreResultsExists != 0 +} + +func (rows *mysqlRows) nextResultSet() (int, error) { + if rows.mc == nil { + return 0, io.EOF + } + if err := rows.mc.error(); err != nil { + return 0, err } // Remove unread packets from stream - err := mc.readUntilEOF() - rows.mc = nil + if !rows.rs.done { + if err := rows.mc.readUntilEOF(); err != nil { + return 0, err + } + rows.rs.done = true + } + + if !rows.HasNextResultSet() { + rows.mc = nil + return 0, io.EOF + } + rows.rs = resultSet{} + return rows.mc.readResultSetHeaderPacket() +} + +func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) { + for { + resLen, err := rows.nextResultSet() + if err != nil { + return 0, err + } + + if resLen > 0 { + return resLen, nil + } + + rows.rs.done = true + } +} + +func (rows *binaryRows) NextResultSet() error { + resLen, err := rows.nextNotEmptyResultSet() + if err != nil { + return err + } + + rows.rs.columns, err = rows.mc.readColumns(resLen) return err } func (rows *binaryRows) Next(dest []driver.Value) error { if mc := rows.mc; mc != nil { - if mc.netConn == nil { - return ErrInvalidConn + if err := mc.error(); err != nil { + return err } // Fetch next row from stream @@ -81,10 +200,20 @@ func (rows *binaryRows) Next(dest []driver.Value) error { return io.EOF } +func (rows *textRows) NextResultSet() (err error) { + resLen, err := rows.nextNotEmptyResultSet() + if err != nil { + return err + } + + rows.rs.columns, err = rows.mc.readColumns(resLen) + return err +} + func (rows *textRows) Next(dest []driver.Value) error { if mc := rows.mc; mc != nil { - if mc.netConn == nil { - return ErrInvalidConn + if err := mc.error(); err != nil { + return err } // Fetch next row from stream @@ -92,15 +221,3 @@ func (rows *textRows) Next(dest []driver.Value) error { } return io.EOF } - -func (rows emptyRows) Columns() []string { - return nil -} - -func (rows emptyRows) Close() error { - return nil -} - -func (rows emptyRows) Next(dest []driver.Value) error { - return io.EOF -} diff --git a/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/go-sql-driver/mysql/statement.go index 6e869b3..f7e3709 100644 --- a/vendor/github.com/go-sql-driver/mysql/statement.go +++ b/vendor/github.com/go-sql-driver/mysql/statement.go @@ -11,20 +11,22 @@ package mysql import ( "database/sql/driver" "fmt" + "io" "reflect" - "strconv" ) type mysqlStmt struct { mc *mysqlConn id uint32 paramCount int - columns []mysqlField // cached from the first query } func (stmt *mysqlStmt) Close() error { - if stmt.mc == nil || stmt.mc.netConn == nil { - errLog.Print(ErrInvalidConn) + if stmt.mc == nil || stmt.mc.closed.IsSet() { + // driver.Stmt.Close can be called more than once, thus this function + // has to be idempotent. + // See also Issue #450 and golang/go#16019. + //errLog.Print(ErrInvalidConn) return driver.ErrBadConn } @@ -42,14 +44,14 @@ func (stmt *mysqlStmt) ColumnConverter(idx int) driver.ValueConverter { } func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { - if stmt.mc.netConn == nil { + if stmt.mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command err := stmt.writeExecutePacket(args) if err != nil { - return nil, err + return nil, stmt.mc.markBadConn(err) } mc := stmt.mc @@ -59,37 +61,45 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { // Read Result resLen, err := mc.readResultSetHeaderPacket() - if err == nil { - if resLen > 0 { - // Columns - err = mc.readUntilEOF() - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } - // Rows - err = mc.readUntilEOF() + if resLen > 0 { + // Columns + if err = mc.readUntilEOF(); err != nil { + return nil, err } - if err == nil { - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, nil + + // Rows + if err := mc.readUntilEOF(); err != nil { + return nil, err } } - return nil, err + if err := mc.discardResults(); err != nil { + return nil, err + } + + return &mysqlResult{ + affectedRows: int64(mc.affectedRows), + insertId: int64(mc.insertId), + }, nil } func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { - if stmt.mc.netConn == nil { + return stmt.query(args) +} + +func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { + if stmt.mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command err := stmt.writeExecutePacket(args) if err != nil { - return nil, err + return nil, stmt.mc.markBadConn(err) } mc := stmt.mc @@ -101,17 +111,18 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { } rows := new(binaryRows) - rows.mc = mc if resLen > 0 { - // Columns - // If not cached, read them and cache them - if stmt.columns == nil { - rows.columns, err = mc.readColumns(resLen) - stmt.columns = rows.columns - } else { - rows.columns = stmt.columns - err = mc.readUntilEOF() + rows.mc = mc + rows.rs.columns, err = mc.readColumns(resLen) + } else { + rows.rs.done = true + + switch err := rows.NextResultSet(); err { + case nil, io.EOF: + return rows, nil + default: + return nil, err } } @@ -120,31 +131,74 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { type converter struct{} +// ConvertValue mirrors the reference/default converter in database/sql/driver +// with _one_ exception. We support uint64 with their high bit and the default +// implementation does not. This function should be kept in sync with +// database/sql/driver defaultConverter.ConvertValue() except for that +// deliberate difference. func (c converter) ConvertValue(v interface{}) (driver.Value, error) { if driver.IsValue(v) { return v, nil } + if vr, ok := v.(driver.Valuer); ok { + sv, err := callValuerValue(vr) + if err != nil { + return nil, err + } + if !driver.IsValue(sv) { + return nil, fmt.Errorf("non-Value type %T returned from Value", sv) + } + return sv, nil + } + rv := reflect.ValueOf(v) switch rv.Kind() { case reflect.Ptr: // indirect pointers if rv.IsNil() { return nil, nil + } else { + return c.ConvertValue(rv.Elem().Interface()) } - return c.ConvertValue(rv.Elem().Interface()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint(), nil case reflect.Float32, reflect.Float64: return rv.Float(), nil + case reflect.Bool: + return rv.Bool(), nil + case reflect.Slice: + ek := rv.Type().Elem().Kind() + if ek == reflect.Uint8 { + return rv.Bytes(), nil + } + return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek) + case reflect.String: + return rv.String(), nil } return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) } + +var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() + +// callValuerValue returns vr.Value(), with one exception: +// If vr.Value is an auto-generated method on a pointer type and the +// pointer is nil, it would panic at runtime in the panicwrap +// method. Treat it like nil instead. +// +// This is so people can implement driver.Value on value types and +// still use nil pointers to those types to mean nil/NULL, just like +// string/*string. +// +// This is an exact copy of the same-named unexported function from the +// database/sql package. +func callValuerValue(vr driver.Valuer) (v driver.Value, err error) { + if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr && + rv.IsNil() && + rv.Type().Elem().Implements(valuerReflectType) { + return nil, nil + } + return vr.Value() +} diff --git a/vendor/github.com/go-sql-driver/mysql/statement_test.go b/vendor/github.com/go-sql-driver/mysql/statement_test.go new file mode 100644 index 0000000..4b9914f --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/statement_test.go @@ -0,0 +1,126 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "testing" +) + +func TestConvertDerivedString(t *testing.T) { + type derived string + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Derived string type not convertible", err) + } + + if output != "value" { + t.Fatalf("Derived string type not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedByteSlice(t *testing.T) { + type derived []uint8 + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Byte slice not convertible", err) + } + + if bytes.Compare(output.([]byte), []byte("value")) != 0 { + t.Fatalf("Byte slice not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedUnsupportedSlice(t *testing.T) { + type derived []int + + _, err := converter{}.ConvertValue(derived{1}) + if err == nil || err.Error() != "unsupported type mysql.derived, a slice of int" { + t.Fatal("Unexpected error", err) + } +} + +func TestConvertDerivedBool(t *testing.T) { + type derived bool + + output, err := converter{}.ConvertValue(derived(true)) + if err != nil { + t.Fatal("Derived bool type not convertible", err) + } + + if output != true { + t.Fatalf("Derived bool type not converted, got %#v %T", output, output) + } +} + +func TestConvertPointer(t *testing.T) { + str := "value" + + output, err := converter{}.ConvertValue(&str) + if err != nil { + t.Fatal("Pointer type not convertible", err) + } + + if output != "value" { + t.Fatalf("Pointer type not converted, got %#v %T", output, output) + } +} + +func TestConvertSignedIntegers(t *testing.T) { + values := []interface{}{ + int8(-42), + int16(-42), + int32(-42), + int64(-42), + int(-42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != int64(-42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } +} + +func TestConvertUnsignedIntegers(t *testing.T) { + values := []interface{}{ + uint8(42), + uint16(42), + uint32(42), + uint64(42), + uint(42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != uint64(42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } + + output, err := converter{}.ConvertValue(^uint64(0)) + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if output != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", output, output) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/transaction.go b/vendor/github.com/go-sql-driver/mysql/transaction.go index 33c749b..417d727 100644 --- a/vendor/github.com/go-sql-driver/mysql/transaction.go +++ b/vendor/github.com/go-sql-driver/mysql/transaction.go @@ -13,7 +13,7 @@ type mysqlTx struct { } func (tx *mysqlTx) Commit() (err error) { - if tx.mc == nil || tx.mc.netConn == nil { + if tx.mc == nil || tx.mc.closed.IsSet() { return ErrInvalidConn } err = tx.mc.exec("COMMIT") @@ -22,7 +22,7 @@ func (tx *mysqlTx) Commit() (err error) { } func (tx *mysqlTx) Rollback() (err error) { - if tx.mc == nil || tx.mc.netConn == nil { + if tx.mc == nil || tx.mc.closed.IsSet() { return ErrInvalidConn } err = tx.mc.exec("ROLLBACK") diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index 6a26ad1..cfa10e9 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -9,35 +9,32 @@ package mysql import ( - "crypto/sha1" "crypto/tls" + "database/sql" "database/sql/driver" "encoding/binary" "errors" "fmt" "io" - "net" - "net/url" + "strconv" "strings" + "sync" + "sync/atomic" "time" ) +// Registry for custom tls.Configs var ( - tlsConfigRegister map[string]*tls.Config // Register for custom tls.Configs - - errInvalidDSNUnescaped = errors.New("Invalid DSN: Did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("Invalid DSN: Network Address not terminated (missing closing brace)") - errInvalidDSNNoSlash = errors.New("Invalid DSN: Missing the slash separating the database name") - errInvalidDSNUnsafeCollation = errors.New("Invalid DSN: interpolateParams can be used with ascii, latin1, utf8 and utf8mb4 charset") + tlsConfigLock sync.RWMutex + tlsConfigRegistry map[string]*tls.Config ) -func init() { - tlsConfigRegister = make(map[string]*tls.Config) -} - // RegisterTLSConfig registers a custom tls.Config to be used with sql.Open. // Use the key as a value in the DSN where tls=value. // +// Note: The provided tls.Config is exclusively owned by the driver after +// registering it. +// // rootCertPool := x509.NewCertPool() // pem, err := ioutil.ReadFile("/path/ca-cert.pem") // if err != nil { @@ -59,243 +56,35 @@ func init() { // db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") // func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { - return fmt.Errorf("Key '%s' is reserved", key) + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { + return fmt.Errorf("key '%s' is reserved", key) } - tlsConfigRegister[key] = config + tlsConfigLock.Lock() + if tlsConfigRegistry == nil { + tlsConfigRegistry = make(map[string]*tls.Config) + } + + tlsConfigRegistry[key] = config + tlsConfigLock.Unlock() return nil } // DeregisterTLSConfig removes the tls.Config associated with key. func DeregisterTLSConfig(key string) { - delete(tlsConfigRegister, key) + tlsConfigLock.Lock() + if tlsConfigRegistry != nil { + delete(tlsConfigRegistry, key) + } + tlsConfigLock.Unlock() } -// parseDSN parses the DSN string to a config -func parseDSN(dsn string) (cfg *config, err error) { - // New config with some default values - cfg = &config{ - loc: time.UTC, - collation: defaultCollation, +func getTLSConfigClone(key string) (config *tls.Config) { + tlsConfigLock.RLock() + if v, ok := tlsConfigRegistry[key]; ok { + config = v.Clone() } - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.passwd = dsn[k+1 : j] - break - } - } - cfg.user = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - cfg.addr = dsn[k+1 : i-1] - break - } - } - cfg.net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.dbname = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - if cfg.interpolateParams && unsafeCollations[cfg.collation] { - return nil, errInvalidDSNUnsafeCollation - } - - // Set default network if empty - if cfg.net == "" { - cfg.net = "tcp" - } - - // Set default address if empty - if cfg.addr == "" { - switch cfg.net { - case "tcp": - cfg.addr = "127.0.0.1:3306" - case "unix": - cfg.addr = "/tmp/mysql.sock" - default: - return nil, errors.New("Default addr for network '" + cfg.net + "' unknown") - } - - } - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.interpolateParams, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Disable INFILE whitelist / enable all files - case "allowAllFiles": - var isBool bool - cfg.allowAllFiles, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Use cleartext authentication mode (MySQL 5.5.10+) - case "allowCleartextPasswords": - var isBool bool - cfg.allowCleartextPasswords, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Use old authentication mode (pre MySQL 4.1) - case "allowOldPasswords": - var isBool bool - cfg.allowOldPasswords, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Switch "rowsAffected" mode - case "clientFoundRows": - var isBool bool - cfg.clientFoundRows, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Collation - case "collation": - collation, ok := collations[value] - if !ok { - // Note possibility for false negatives: - // could be triggered although the collation is valid if the - // collations map does not contain entries the server supports. - err = errors.New("unknown collation") - return - } - cfg.collation = collation - break - - case "columnsWithAlias": - var isBool bool - cfg.columnsWithAlias, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // Dial Timeout - case "timeout": - cfg.timeout, err = time.ParseDuration(value) - if err != nil { - return - } - - // TLS-Encryption - case "tls": - boolValue, isBool := readBool(value) - if isBool { - if boolValue { - cfg.tls = &tls.Config{} - } - } else { - if strings.ToLower(value) == "skip-verify" { - cfg.tls = &tls.Config{InsecureSkipVerify: true} - } else if tlsConfig, ok := tlsConfigRegister[value]; ok { - if len(tlsConfig.ServerName) == 0 && !tlsConfig.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.addr) - if err == nil { - tlsConfig.ServerName = host - } - } - - cfg.tls = tlsConfig - } else { - return fmt.Errorf("Invalid value / unknown config name: %s", value) - } - } - - default: - // lazy init - if cfg.params == nil { - cfg.params = make(map[string]string) - } - - if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - + tlsConfigLock.RUnlock() return } @@ -313,119 +102,6 @@ func readBool(input string) (value bool, valid bool) { return } -/****************************************************************************** -* Authentication * -******************************************************************************/ - -// Encrypt password using 4.1+ method -func scramblePassword(scramble, password []byte) []byte { - if len(password) == 0 { - return nil - } - - // stage1Hash = SHA1(password) - crypt := sha1.New() - crypt.Write(password) - stage1 := crypt.Sum(nil) - - // scrambleHash = SHA1(scramble + SHA1(stage1Hash)) - // inner Hash - crypt.Reset() - crypt.Write(stage1) - hash := crypt.Sum(nil) - - // outer Hash - crypt.Reset() - crypt.Write(scramble) - crypt.Write(hash) - scramble = crypt.Sum(nil) - - // token = scrambleHash XOR stage1Hash - for i := range scramble { - scramble[i] ^= stage1[i] - } - return scramble -} - -// Encrypt password using pre 4.1 (old password) method -// https://github.com/atcurtis/mariadb/blob/master/mysys/my_rnd.c -type myRnd struct { - seed1, seed2 uint32 -} - -const myRndMaxVal = 0x3FFFFFFF - -// Pseudo random number generator -func newMyRnd(seed1, seed2 uint32) *myRnd { - return &myRnd{ - seed1: seed1 % myRndMaxVal, - seed2: seed2 % myRndMaxVal, - } -} - -// Tested to be equivalent to MariaDB's floating point variant -// http://play.golang.org/p/QHvhd4qved -// http://play.golang.org/p/RG0q4ElWDx -func (r *myRnd) NextByte() byte { - r.seed1 = (r.seed1*3 + r.seed2) % myRndMaxVal - r.seed2 = (r.seed1 + r.seed2 + 33) % myRndMaxVal - - return byte(uint64(r.seed1) * 31 / myRndMaxVal) -} - -// Generate binary hash from byte string using insecure pre 4.1 method -func pwHash(password []byte) (result [2]uint32) { - var add uint32 = 7 - var tmp uint32 - - result[0] = 1345345333 - result[1] = 0x12345671 - - for _, c := range password { - // skip spaces and tabs in password - if c == ' ' || c == '\t' { - continue - } - - tmp = uint32(c) - result[0] ^= (((result[0] & 63) + add) * tmp) + (result[0] << 8) - result[1] += (result[1] << 8) ^ result[0] - add += tmp - } - - // Remove sign bit (1<<31)-1) - result[0] &= 0x7FFFFFFF - result[1] &= 0x7FFFFFFF - - return -} - -// Encrypt password using insecure pre 4.1 method -func scrambleOldPassword(scramble, password []byte) []byte { - if len(password) == 0 { - return nil - } - - scramble = scramble[:8] - - hashPw := pwHash(password) - hashSc := pwHash(scramble) - - r := newMyRnd(hashPw[0]^hashSc[0], hashPw[1]^hashSc[1]) - - var out [8]byte - for i := range out { - out[i] = r.NextByte() + 64 - } - - mask := r.NextByte() - for i := range out { - out[i] ^= mask - } - - return out[:] -} - /****************************************************************************** * Time related utils * ******************************************************************************/ @@ -493,7 +169,7 @@ func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { } t, err = time.Parse(timeFormat[:len(str)], str) default: - err = fmt.Errorf("Invalid Time-String: %s", str) + err = fmt.Errorf("invalid time string: %s", str) return } @@ -542,7 +218,7 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va loc, ), nil } - return nil, fmt.Errorf("Invalid DATETIME-packet length %d", num) + return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } // zeroDateTime is used in formatBinaryDateTime to avoid an allocation @@ -554,87 +230,104 @@ var zeroDateTime = []byte("0000-00-00 00:00:00.000000") const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" -func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value, error) { +func appendMicrosecs(dst, src []byte, decimals int) []byte { + if decimals <= 0 { + return dst + } + if len(src) == 0 { + return append(dst, ".000000"[:decimals+1]...) + } + + microsecs := binary.LittleEndian.Uint32(src[:4]) + p1 := byte(microsecs / 10000) + microsecs -= 10000 * uint32(p1) + p2 := byte(microsecs / 100) + microsecs -= 100 * uint32(p2) + p3 := byte(microsecs) + + switch decimals { + default: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + digits10[p3], digits01[p3], + ) + case 1: + return append(dst, '.', + digits10[p1], + ) + case 2: + return append(dst, '.', + digits10[p1], digits01[p1], + ) + case 3: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], + ) + case 4: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + ) + case 5: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + digits10[p3], + ) + } +} + +func formatBinaryDateTime(src []byte, length uint8) (driver.Value, error) { // length expects the deterministic length of the zero value, // negative time and 100+ hours are automatically added if needed if len(src) == 0 { - if justTime { - return zeroDateTime[11 : 11+length], nil - } return zeroDateTime[:length], nil } - var dst []byte // return value - var pt, p1, p2, p3 byte // current digit pair - var zOffs byte // offset of value in zeroDateTime - if justTime { - switch length { - case - 8, // time (can be up to 10 when negative and 100+ hours) - 10, 11, 12, 13, 14, 15: // time with fractional seconds - default: - return nil, fmt.Errorf("illegal TIME length %d", length) + var dst []byte // return value + var p1, p2, p3 byte // current digit pair + + switch length { + case 10, 19, 21, 22, 23, 24, 25, 26: + default: + t := "DATE" + if length > 10 { + t += "TIME" } - switch len(src) { - case 8, 12: - default: - return nil, fmt.Errorf("Invalid TIME-packet length %d", len(src)) - } - // +2 to enable negative time and 100+ hours - dst = make([]byte, 0, length+2) - if src[0] == 1 { - dst = append(dst, '-') - } - if src[1] != 0 { - hour := uint16(src[1])*24 + uint16(src[5]) - pt = byte(hour / 100) - p1 = byte(hour - 100*uint16(pt)) - dst = append(dst, digits01[pt]) - } else { - p1 = src[5] - } - zOffs = 11 - src = src[6:] - } else { - switch length { - case 10, 19, 21, 22, 23, 24, 25, 26: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s length %d", t, length) - } - switch len(src) { - case 4, 7, 11: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s-packet length %d", t, len(src)) - } - dst = make([]byte, 0, length) - // start with the date - year := binary.LittleEndian.Uint16(src[:2]) - pt = byte(year / 100) - p1 = byte(year - 100*uint16(pt)) - p2, p3 = src[2], src[3] - dst = append(dst, - digits10[pt], digits01[pt], - digits10[p1], digits01[p1], '-', - digits10[p2], digits01[p2], '-', - digits10[p3], digits01[p3], - ) - if length == 10 { - return dst, nil - } - if len(src) == 4 { - return append(dst, zeroDateTime[10:length]...), nil - } - dst = append(dst, ' ') - p1 = src[4] // hour - src = src[5:] + return nil, fmt.Errorf("illegal %s length %d", t, length) } + switch len(src) { + case 4, 7, 11: + default: + t := "DATE" + if length > 10 { + t += "TIME" + } + return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) + } + dst = make([]byte, 0, length) + // start with the date + year := binary.LittleEndian.Uint16(src[:2]) + pt := year / 100 + p1 = byte(year - 100*uint16(pt)) + p2, p3 = src[2], src[3] + dst = append(dst, + digits10[pt], digits01[pt], + digits10[p1], digits01[p1], '-', + digits10[p2], digits01[p2], '-', + digits10[p3], digits01[p3], + ) + if length == 10 { + return dst, nil + } + if len(src) == 4 { + return append(dst, zeroDateTime[10:length]...), nil + } + dst = append(dst, ' ') + p1 = src[4] // hour + src = src[5:] + // p1 is 2-digit hour, src is after hour p2, p3 = src[0], src[1] dst = append(dst, @@ -642,51 +335,49 @@ func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value digits10[p2], digits01[p2], ':', digits10[p3], digits01[p3], ) - if length <= byte(len(dst)) { - return dst, nil - } - src = src[2:] + return appendMicrosecs(dst, src[2:], int(length)-20), nil +} + +func formatBinaryTime(src []byte, length uint8) (driver.Value, error) { + // length expects the deterministic length of the zero value, + // negative time and 100+ hours are automatically added if needed if len(src) == 0 { - return append(dst, zeroDateTime[19:zOffs+length]...), nil + return zeroDateTime[11 : 11+length], nil } - microsecs := binary.LittleEndian.Uint32(src[:4]) - p1 = byte(microsecs / 10000) - microsecs -= 10000 * uint32(p1) - p2 = byte(microsecs / 100) - microsecs -= 100 * uint32(p2) - p3 = byte(microsecs) - switch decimals := zOffs + length - 20; decimals { + var dst []byte // return value + + switch length { + case + 8, // time (can be up to 10 when negative and 100+ hours) + 10, 11, 12, 13, 14, 15: // time with fractional seconds default: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - digits10[p3], digits01[p3], - ), nil - case 1: - return append(dst, '.', - digits10[p1], - ), nil - case 2: - return append(dst, '.', - digits10[p1], digits01[p1], - ), nil - case 3: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], - ), nil - case 4: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - ), nil - case 5: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - digits10[p3], - ), nil + return nil, fmt.Errorf("illegal TIME length %d", length) } + switch len(src) { + case 8, 12: + default: + return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) + } + // +2 to enable negative time and 100+ hours + dst = make([]byte, 0, length+2) + if src[0] == 1 { + dst = append(dst, '-') + } + days := binary.LittleEndian.Uint32(src[1:5]) + hours := int64(days)*24 + int64(src[5]) + + if hours >= 100 { + dst = strconv.AppendInt(dst, hours, 10) + } else { + dst = append(dst, digits10[hours], digits01[hours]) + } + + min, sec := src[6], src[7] + dst = append(dst, ':', + digits10[min], digits01[min], ':', + digits10[sec], digits01[sec], + ) + return appendMicrosecs(dst, src[8:], int(length)-9), nil } /****************************************************************************** @@ -752,7 +443,7 @@ func readLengthEncodedString(b []byte) ([]byte, bool, int, error) { // Check data length if len(b) >= n { - return b[n-int(num) : n], false, n, nil + return b[n-int(num) : n : n], false, n, nil } return nil, false, n, io.EOF } @@ -781,8 +472,8 @@ func readLengthEncodedInteger(b []byte) (uint64, bool, int) { if len(b) == 0 { return 0, true, 1 } - switch b[0] { + switch b[0] { // 251: NULL case 0xfb: return 0, true, 1 @@ -877,7 +568,7 @@ func escapeBytesBackslash(buf, v []byte) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } } @@ -922,7 +613,7 @@ func escapeStringBackslash(buf []byte, v string) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } } @@ -971,3 +662,94 @@ func escapeStringQuotes(buf []byte, v string) []byte { return buf[:pos] } + +/****************************************************************************** +* Sync utils * +******************************************************************************/ + +// noCopy may be embedded into structs which must not be copied +// after the first use. +// +// See https://github.com/golang/go/issues/8005#issuecomment-190753527 +// for details. +type noCopy struct{} + +// Lock is a no-op used by -copylocks checker from `go vet`. +func (*noCopy) Lock() {} + +// atomicBool is a wrapper around uint32 for usage as a boolean value with +// atomic access. +type atomicBool struct { + _noCopy noCopy + value uint32 +} + +// IsSet returns whether the current boolean value is true +func (ab *atomicBool) IsSet() bool { + return atomic.LoadUint32(&ab.value) > 0 +} + +// Set sets the value of the bool regardless of the previous value +func (ab *atomicBool) Set(value bool) { + if value { + atomic.StoreUint32(&ab.value, 1) + } else { + atomic.StoreUint32(&ab.value, 0) + } +} + +// TrySet sets the value of the bool and returns whether the value changed +func (ab *atomicBool) TrySet(value bool) bool { + if value { + return atomic.SwapUint32(&ab.value, 1) == 0 + } + return atomic.SwapUint32(&ab.value, 0) > 0 +} + +// atomicError is a wrapper for atomically accessed error values +type atomicError struct { + _noCopy noCopy + value atomic.Value +} + +// Set sets the error value regardless of the previous value. +// The value must not be nil +func (ae *atomicError) Set(value error) { + ae.value.Store(value) +} + +// Value returns the current error value +func (ae *atomicError) Value() error { + if v := ae.value.Load(); v != nil { + // this will panic if the value doesn't implement the error interface + return v.(error) + } + return nil +} + +func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { + dargs := make([]driver.Value, len(named)) + for n, param := range named { + if len(param.Name) > 0 { + // TODO: support the use of Named Parameters #561 + return nil, errors.New("mysql: driver does not support the use of Named Parameters") + } + dargs[n] = param.Value + } + return dargs, nil +} + +func mapIsolationLevel(level driver.IsolationLevel) (string, error) { + switch sql.IsolationLevel(level) { + case sql.LevelRepeatableRead: + return "REPEATABLE READ", nil + case sql.LevelReadCommitted: + return "READ COMMITTED", nil + case sql.LevelReadUncommitted: + return "READ UNCOMMITTED", nil + case sql.LevelSerializable: + return "SERIALIZABLE", nil + default: + return "", fmt.Errorf("mysql: unsupported isolation level: %v", level) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/utils_test.go b/vendor/github.com/go-sql-driver/mysql/utils_test.go new file mode 100644 index 0000000..8951a7a --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/utils_test.go @@ -0,0 +1,334 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "database/sql" + "database/sql/driver" + "encoding/binary" + "testing" + "time" +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} + +func TestLengthEncodedInteger(t *testing.T) { + var integerTests = []struct { + num uint64 + encoded []byte + }{ + {0x0000000000000000, []byte{0x00}}, + {0x0000000000000012, []byte{0x12}}, + {0x00000000000000fa, []byte{0xfa}}, + {0x0000000000000100, []byte{0xfc, 0x00, 0x01}}, + {0x0000000000001234, []byte{0xfc, 0x34, 0x12}}, + {0x000000000000ffff, []byte{0xfc, 0xff, 0xff}}, + {0x0000000000010000, []byte{0xfd, 0x00, 0x00, 0x01}}, + {0x0000000000123456, []byte{0xfd, 0x56, 0x34, 0x12}}, + {0x0000000000ffffff, []byte{0xfd, 0xff, 0xff, 0xff}}, + {0x0000000001000000, []byte{0xfe, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}}, + {0x123456789abcdef0, []byte{0xfe, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12}}, + {0xffffffffffffffff, []byte{0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + } + + for _, tst := range integerTests { + num, isNull, numLen := readLengthEncodedInteger(tst.encoded) + if isNull { + t.Errorf("%x: expected %d, got NULL", tst.encoded, tst.num) + } + if num != tst.num { + t.Errorf("%x: expected %d, got %d", tst.encoded, tst.num, num) + } + if numLen != len(tst.encoded) { + t.Errorf("%x: expected size %d, got %d", tst.encoded, len(tst.encoded), numLen) + } + encoded := appendLengthEncodedInteger(nil, num) + if !bytes.Equal(encoded, tst.encoded) { + t.Errorf("%v: expected %x, got %x", num, tst.encoded, encoded) + } + } +} + +func TestFormatBinaryDateTime(t *testing.T) { + rawDate := [11]byte{} + binary.LittleEndian.PutUint16(rawDate[:2], 1978) // years + rawDate[2] = 12 // months + rawDate[3] = 30 // days + rawDate[4] = 15 // hours + rawDate[5] = 46 // minutes + rawDate[6] = 23 // seconds + binary.LittleEndian.PutUint32(rawDate[7:], 987654) // microseconds + expect := func(expected string, inlen, outlen uint8) { + actual, _ := formatBinaryDateTime(rawDate[:inlen], outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for length in %d, out %d", + expected, actual, inlen, outlen, + ) + } + } + expect("0000-00-00", 0, 10) + expect("0000-00-00 00:00:00", 0, 19) + expect("1978-12-30", 4, 10) + expect("1978-12-30 15:46:23", 7, 19) + expect("1978-12-30 15:46:23.987654", 11, 26) +} + +func TestFormatBinaryTime(t *testing.T) { + expect := func(expected string, src []byte, outlen uint8) { + actual, _ := formatBinaryTime(src, outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for src=%q and outlen=%d", + expected, actual, src, outlen) + } + } + + // binary format: + // sign (0: positive, 1: negative), days(4), hours, minutes, seconds, micro(4) + + // Zeros + expect("00:00:00", []byte{}, 8) + expect("00:00:00.0", []byte{}, 10) + expect("00:00:00.000000", []byte{}, 15) + + // Without micro(4) + expect("12:34:56", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("-12:34:56", []byte{1, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 11) + expect("24:34:56", []byte{0, 1, 0, 0, 0, 0, 34, 56}, 8) + expect("-99:34:56", []byte{1, 4, 0, 0, 0, 3, 34, 56}, 8) + expect("103079215103:34:56", []byte{0, 255, 255, 255, 255, 23, 34, 56}, 8) + + // With micro(4) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 11) + expect("12:34:56.000099", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 15) +} + +func TestEscapeBackslash(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesBackslash([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringBackslash([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\\0bar", "foo\x00bar") + expect("foo\\nbar", "foo\nbar") + expect("foo\\rbar", "foo\rbar") + expect("foo\\Zbar", "foo\x1abar") + expect("foo\\\"bar", "foo\"bar") + expect("foo\\\\bar", "foo\\bar") + expect("foo\\'bar", "foo'bar") +} + +func TestEscapeQuotes(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesQuotes([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringQuotes([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\x00bar", "foo\x00bar") // not affected + expect("foo\nbar", "foo\nbar") // not affected + expect("foo\rbar", "foo\rbar") // not affected + expect("foo\x1abar", "foo\x1abar") // not affected + expect("foo''bar", "foo'bar") // affected + expect("foo\"bar", "foo\"bar") // not affected +} + +func TestAtomicBool(t *testing.T) { + var ab atomicBool + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(true) + if ab.value != 1 { + t.Fatal("Set(true) did not set value to 1") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(false) + if ab.value != 0 { + t.Fatal("Set(false) did not set value to 0") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(false) + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + if ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to fail") + } + if !ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to succeed") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + if ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to fail") + } + if !ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to succeed") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab._noCopy.Lock() // we've "tested" it ¯\_(ツ)_/¯ +} + +func TestAtomicError(t *testing.T) { + var ae atomicError + if ae.Value() != nil { + t.Fatal("Expected value to be nil") + } + + ae.Set(ErrMalformPkt) + if v := ae.Value(); v != ErrMalformPkt { + if v == nil { + t.Fatal("Value is still nil") + } + t.Fatal("Error did not match") + } + ae.Set(ErrPktSync) + if ae.Value() == ErrMalformPkt { + t.Fatal("Error still matches old error") + } + if v := ae.Value(); v != ErrPktSync { + t.Fatal("Error did not match") + } +} + +func TestIsolationLevelMapping(t *testing.T) { + data := []struct { + level driver.IsolationLevel + expected string + }{ + { + level: driver.IsolationLevel(sql.LevelReadCommitted), + expected: "READ COMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelRepeatableRead), + expected: "REPEATABLE READ", + }, + { + level: driver.IsolationLevel(sql.LevelReadUncommitted), + expected: "READ UNCOMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelSerializable), + expected: "SERIALIZABLE", + }, + } + + for i, td := range data { + if actual, err := mapIsolationLevel(td.level); actual != td.expected || err != nil { + t.Fatal(i, td.expected, actual, err) + } + } + + // check unsupported mapping + expectedErr := "mysql: unsupported isolation level: 7" + actual, err := mapIsolationLevel(driver.IsolationLevel(sql.LevelLinearizable)) + if actual != "" || err == nil { + t.Fatal("Expected error on unsupported isolation level") + } + if err.Error() != expectedErr { + t.Fatalf("Expected error to be %q, got %q", expectedErr, err) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..d9771f1 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,21 @@ +### Issue description +Tell us what should happen and what happens instead + +### Example code +```go +If possible, please enter some example code here to reproduce the issue. +``` + +### Error log +``` +If you have an error log, please paste it here. +``` + +### Configuration +*Driver version (or git SHA):* + +*Go version:* run `go version` in your console + +*Server version:* E.g. MySQL 5.6, MariaDB 10.0.20 + +*Server OS:* E.g. Debian 8.1 (Jessie), Windows 10 diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6f5c7eb --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +### Description +Please explain the changes you made here. + +### Checklist +- [ ] Code compiles correctly +- [ ] Created tests which fail without the change (if possible) +- [ ] All tests passing +- [ ] Extended the README / documentation, if necessary +- [ ] Added myself / the copyright holder to the AUTHORS file diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore new file mode 100644 index 0000000..2de28da --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +Icon? +ehthumbs.db +Thumbs.db +.idea diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml new file mode 100644 index 0000000..eae311b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -0,0 +1,128 @@ +sudo: false +language: go +go: + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - master + +before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + +before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB" | sudo tee -a /etc/mysql/my.cnf + - sudo service mysql restart + - .travis/wait_mysql.sh + - mysql -e 'create database gotest;' + +matrix: + include: + - env: DB=MYSQL8 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:8.0 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:8.0 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MYSQL57 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:5.7 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:5.7 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA55 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:5.5 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:5.5 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA10_1 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:10.1 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:10.1 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - os: osx + osx_image: xcode10.1 + addons: + homebrew: + packages: + - mysql + go: 1.12.x + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + - mysql.server start + - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + - mysql -uroot -e 'create database gotest;' + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3306 + - export MYSQL_TEST_CONCURRENT=1 + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - go vet ./... + - .travis/gofmt.sh +after_script: + - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf new file mode 100644 index 0000000..e57754e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf @@ -0,0 +1,5 @@ +[client] +user = gotest +password = secret +host = 127.0.0.1 +port = 3307 diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh new file mode 100755 index 0000000..9bf0d16 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ev + +# Only check for go1.10+ since the gofmt style changed +if [[ $(go version) =~ go1\.([0-9]+) ]] && ((${BASH_REMATCH[1]} >= 10)); then + test -z "$(gofmt -d -s . | tee /dev/stderr)" +fi diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh new file mode 100755 index 0000000..e87993e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh @@ -0,0 +1,8 @@ +#!/bin/sh +while : +do + if mysql -e 'select version()' 2>&1 | grep 'version()\|ERROR 2059 (HY000):'; then + break + fi + sleep 3 +done diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS index fbe4ec4..bfe74c4 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -27,6 +27,7 @@ Daniël van Eeden Dave Protasowski DisposaBoy Egor Smolyakov +Erwan Martin Evan Shaw Frederick Mayle Gustavo Kristic @@ -34,12 +35,15 @@ Hajime Nakagami Hanno Braun Henri Yandell Hirotaka Yamamoto +Huyiguang ICHINOSE Shogo +Ilia Cimpoes INADA Naoki Jacek Szwec James Harr Jeff Hodges Jeffrey Charles +Jerome Meyer Jian Zhen Joshua Prunier Julien Lefevre @@ -69,10 +73,14 @@ Richard Wilkes Robert Russell Runrioter Wung Shuode Li +Simon J Mudd Soroush Pour Stan Putrya Stanley Gunawan +Steven Hartland Thomas Wodarek +Tim Ruffles +Tom Jenkinson Xiangyu Hu Xiaobing Jiang Xiuming Chen @@ -82,9 +90,12 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +Facebook Inc. +GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. +Multiplay Ltd. Percona LLC Pivotal Inc. Stripe Inc. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md new file mode 100644 index 0000000..2d87d74 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -0,0 +1,167 @@ +## Version 1.4 (2018-06-03) + +Changes: + + - Documentation fixes (#530, #535, #567) + - Refactoring (#575, #579, #580, #581, #603, #615, #704) + - Cache column names (#444) + - Sort the DSN parameters in DSNs generated from a config (#637) + - Allow native password authentication by default (#644) + - Use the default port if it is missing in the DSN (#668) + - Removed the `strict` mode (#676) + - Do not query `max_allowed_packet` by default (#680) + - Dropped support Go 1.6 and lower (#696) + - Updated `ConvertValue()` to match the database/sql/driver implementation (#760) + - Document the usage of `0000-00-00T00:00:00` as the time.Time zero value (#783) + - Improved the compatibility of the authentication system (#807) + +New Features: + + - Multi-Results support (#537) + - `rejectReadOnly` DSN option (#604) + - `context.Context` support (#608, #612, #627, #761) + - Transaction isolation level support (#619, #744) + - Read-Only transactions support (#618, #634) + - `NewConfig` function which initializes a config with default values (#679) + - Implemented the `ColumnType` interfaces (#667, #724) + - Support for custom string types in `ConvertValue` (#623) + - Implemented `NamedValueChecker`, improving support for uint64 with high bit set (#690, #709, #710) + - `caching_sha2_password` authentication plugin support (#794, #800, #801, #802) + - Implemented `driver.SessionResetter` (#779) + - `sha256_password` authentication plugin support (#808) + +Bugfixes: + + - Use the DSN hostname as TLS default ServerName if `tls=true` (#564, #718) + - Fixed LOAD LOCAL DATA INFILE for empty files (#590) + - Removed columns definition cache since it sometimes cached invalid data (#592) + - Don't mutate registered TLS configs (#600) + - Make RegisterTLSConfig concurrency-safe (#613) + - Handle missing auth data in the handshake packet correctly (#646) + - Do not retry queries when data was written to avoid data corruption (#302, #736) + - Cache the connection pointer for error handling before invalidating it (#678) + - Fixed imports for appengine/cloudsql (#700) + - Fix sending STMT_LONG_DATA for 0 byte data (#734) + - Set correct capacity for []bytes read from length-encoded strings (#766) + - Make RegisterDial concurrency-safe (#773) + + +## Version 1.3 (2016-12-01) + +Changes: + + - Go 1.1 is no longer supported + - Use decimals fields in MySQL to format time types (#249) + - Buffer optimizations (#269) + - TLS ServerName defaults to the host (#283) + - Refactoring (#400, #410, #437) + - Adjusted documentation for second generation CloudSQL (#485) + - Documented DSN system var quoting rules (#502) + - Made statement.Close() calls idempotent to avoid errors in Go 1.6+ (#512) + +New Features: + + - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) + - Support for returning table alias on Columns() (#289, #359, #382) + - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490) + - Support for uint64 parameters with high bit set (#332, #345) + - Cleartext authentication plugin support (#327) + - Exported ParseDSN function and the Config struct (#403, #419, #429) + - Read / Write timeouts (#401) + - Support for JSON field type (#414) + - Support for multi-statements and multi-results (#411, #431) + - DSN parameter to set the driver-side max_allowed_packet value manually (#489) + - Native password authentication plugin support (#494, #524) + +Bugfixes: + + - Fixed handling of queries without columns and rows (#255) + - Fixed a panic when SetKeepAlive() failed (#298) + - Handle ERR packets while reading rows (#321) + - Fixed reading NULL length-encoded integers in MySQL 5.6+ (#349) + - Fixed absolute paths support in LOAD LOCAL DATA INFILE (#356) + - Actually zero out bytes in handshake response (#378) + - Fixed race condition in registering LOAD DATA INFILE handler (#383) + - Fixed tests with MySQL 5.7.9+ (#380) + - QueryUnescape TLS config names (#397) + - Fixed "broken pipe" error by writing to closed socket (#390) + - Fixed LOAD LOCAL DATA INFILE buffering (#424) + - Fixed parsing of floats into float64 when placeholders are used (#434) + - Fixed DSN tests with Go 1.7+ (#459) + - Handle ERR packets while waiting for EOF (#473) + - Invalidate connection on error while discarding additional results (#513) + - Allow terminating packets of length 0 (#516) + + +## Version 1.2 (2014-06-03) + +Changes: + + - We switched back to a "rolling release". `go get` installs the current master branch again + - Version v1 of the driver will not be maintained anymore. Go 1.0 is no longer supported by this driver + - Exported errors to allow easy checking from application code + - Enabled TCP Keepalives on TCP connections + - Optimized INFILE handling (better buffer size calculation, lazy init, ...) + - The DSN parser also checks for a missing separating slash + - Faster binary date / datetime to string formatting + - Also exported the MySQLWarning type + - mysqlConn.Close returns the first error encountered instead of ignoring all errors + - writePacket() automatically writes the packet size to the header + - readPacket() uses an iterative approach instead of the recursive approach to merge splitted packets + +New Features: + + - `RegisterDial` allows the usage of a custom dial function to establish the network connection + - Setting the connection collation is possible with the `collation` DSN parameter. This parameter should be preferred over the `charset` parameter + - Logging of critical errors is configurable with `SetLogger` + - Google CloudSQL support + +Bugfixes: + + - Allow more than 32 parameters in prepared statements + - Various old_password fixes + - Fixed TestConcurrent test to pass Go's race detection + - Fixed appendLengthEncodedInteger for large numbers + - Renamed readLengthEnodedString to readLengthEncodedString and skipLengthEnodedString to skipLengthEncodedString (fixed typo) + + +## Version 1.1 (2013-11-02) + +Changes: + + - Go-MySQL-Driver now requires Go 1.1 + - Connections now use the collation `utf8_general_ci` by default. Adding `&charset=UTF8` to the DSN should not be necessary anymore + - Made closing rows and connections error tolerant. This allows for example deferring rows.Close() without checking for errors + - `[]byte(nil)` is now treated as a NULL value. Before, it was treated like an empty string / `[]byte("")` + - DSN parameter values must now be url.QueryEscape'ed. This allows text values to contain special characters, such as '&'. + - Use the IO buffer also for writing. This results in zero allocations (by the driver) for most queries + - Optimized the buffer for reading + - stmt.Query now caches column metadata + - New Logo + - Changed the copyright header to include all contributors + - Improved the LOAD INFILE documentation + - The driver struct is now exported to make the driver directly accessible + - Refactored the driver tests + - Added more benchmarks and moved all to a separate file + - Other small refactoring + +New Features: + + - Added *old_passwords* support: Required in some cases, but must be enabled by adding `allowOldPasswords=true` to the DSN since it is insecure + - Added a `clientFoundRows` parameter: Return the number of matching rows instead of the number of rows changed on UPDATEs + - Added TLS/SSL support: Use a TLS/SSL encrypted connection to the server. Custom TLS configs can be registered and used + +Bugfixes: + + - Fixed MySQL 4.1 support: MySQL 4.1 sends packets with lengths which differ from the specification + - Convert to DB timezone when inserting `time.Time` + - Splitted packets (more than 16MB) are now merged correctly + - Fixed false positive `io.EOF` errors when the data was fully read + - Avoid panics on reuse of closed connections + - Fixed empty string producing false nil values + - Fixed sign byte for positive TIME fields + + +## Version 1.0 (2013-05-14) + +Initial Release diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md new file mode 100644 index 0000000..8fe16bc --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md @@ -0,0 +1,23 @@ +# Contributing Guidelines + +## Reporting Issues + +Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed). + +## Contributing Code + +By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file. +Don't forget to add yourself to the AUTHORS file. + +### Code Review + +Everyone is invited to review and comment on pull requests. +If it looks fine to you, comment with "LGTM" (Looks good to me). + +If changes are required, notice the reviewers with "PTAL" (Please take another look) after committing the fixes. + +Before merging the Pull Request, at least one [team member](https://github.com/go-sql-driver?tab=members) must have commented with "LGTM". + +## Development Ideas + +If you are looking for ideas for code contributions, please check our [Development Ideas](https://github.com/go-sql-driver/mysql/wiki/Development-Ideas) Wiki page. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md new file mode 100644 index 0000000..c6adf1d --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md @@ -0,0 +1,495 @@ +# Go-MySQL-Driver + +A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package + +![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin") + +--------------------------------------- + * [Features](#features) + * [Requirements](#requirements) + * [Installation](#installation) + * [Usage](#usage) + * [DSN (Data Source Name)](#dsn-data-source-name) + * [Password](#password) + * [Protocol](#protocol) + * [Address](#address) + * [Parameters](#parameters) + * [Examples](#examples) + * [Connection pool and timeouts](#connection-pool-and-timeouts) + * [context.Context Support](#contextcontext-support) + * [ColumnType Support](#columntype-support) + * [LOAD DATA LOCAL INFILE support](#load-data-local-infile-support) + * [time.Time support](#timetime-support) + * [Unicode support](#unicode-support) + * [Testing / Development](#testing--development) + * [License](#license) + +--------------------------------------- + +## Features + * Lightweight and [fast](https://github.com/go-sql-driver/sql-benchmark "golang MySQL-Driver performance") + * Native Go implementation. No C-bindings, just pure Go + * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](https://godoc.org/github.com/go-sql-driver/mysql#DialFunc) + * Automatic handling of broken connections + * Automatic Connection Pooling *(by database/sql package)* + * Supports queries larger than 16MB + * Full [`sql.RawBytes`](https://golang.org/pkg/database/sql/#RawBytes) support. + * Intelligent `LONG DATA` handling in prepared statements + * Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support + * Optional `time.Time` parsing + * Optional placeholder interpolation + +## Requirements + * Go 1.9 or higher. We aim to support the 3 latest versions of Go. + * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) + +--------------------------------------- + +## Installation +Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell: +```bash +$ go get -u github.com/go-sql-driver/mysql +``` +Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`. + +## Usage +_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](https://golang.org/pkg/database/sql/) API then. + +Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: +```go +import "database/sql" +import _ "github.com/go-sql-driver/mysql" + +db, err := sql.Open("mysql", "user:password@/dbname") +``` + +[Examples are available in our Wiki](https://github.com/go-sql-driver/mysql/wiki/Examples "Go-MySQL-Driver Examples"). + + +### DSN (Data Source Name) + +The Data Source Name has a common format, like e.g. [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php) uses it, but without type-prefix (optional parts marked by squared brackets): +``` +[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] +``` + +A DSN in its fullest form: +``` +username:password@protocol(address)/dbname?param=value +``` + +Except for the databasename, all values are optional. So the minimal DSN is: +``` +/dbname +``` + +If you do not want to preselect a database, leave `dbname` empty: +``` +/ +``` +This has the same effect as an empty DSN string: +``` + +``` + +Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. + +#### Password +Passwords can consist of any character. Escaping is **not** necessary. + +#### Protocol +See [net.Dial](https://golang.org/pkg/net/#Dial) for more information which networks are available. +In general you should use an Unix domain socket if available and TCP otherwise for best performance. + +#### Address +For TCP and UDP networks, addresses have the form `host[:port]`. +If `port` is omitted, the default port will be used. +If `host` is a literal IPv6 address, it must be enclosed in square brackets. +The functions [net.JoinHostPort](https://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](https://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. + +For Unix domain sockets the address is the absolute path to the MySQL-Server-socket, e.g. `/var/run/mysqld/mysqld.sock` or `/tmp/mysql.sock`. + +#### Parameters +*Parameters are case-sensitive!* + +Notice that any of `true`, `TRUE`, `True` or `1` is accepted to stand for a true boolean value. Not surprisingly, false can be specified as any of: `false`, `FALSE`, `False` or `0`. + +##### `allowAllFiles` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowAllFiles=true` disables the file Whitelist for `LOAD DATA LOCAL INFILE` and allows *all* files. +[*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) + +##### `allowCleartextPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. + +##### `allowNativePasswords` + +``` +Type: bool +Valid Values: true, false +Default: true +``` +`allowNativePasswords=false` disallows the usage of MySQL native password method. + +##### `allowOldPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` +`allowOldPasswords=true` allows the usage of the insecure old password method. This should be avoided, but is necessary in some cases. See also [the old_passwords wiki page](https://github.com/go-sql-driver/mysql/wiki/old_passwords). + +##### `charset` + +``` +Type: string +Valid Values: +Default: none +``` + +Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). + +Usage of the `charset` parameter is discouraged because it issues additional queries to the server. +Unless you need the fallback behavior, please use `collation` instead. + +##### `collation` + +``` +Type: string +Valid Values: +Default: utf8mb4_general_ci +``` + +Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. + +A list of valid charsets for a server is retrievable with `SHOW COLLATION`. + +The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You should use an older collation (e.g. `utf8_general_ci`) for older MySQL. + +Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). + + +##### `clientFoundRows` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`clientFoundRows=true` causes an UPDATE to return the number of matching rows instead of the number of rows changed. + +##### `columnsWithAlias` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +When `columnsWithAlias` is true, calls to `sql.Rows.Columns()` will return the table alias and the column name separated by a dot. For example: + +``` +SELECT u.id FROM users as u +``` + +will return `u.id` instead of just `id` if `columnsWithAlias=true`. + +##### `interpolateParams` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +If `interpolateParams` is true, placeholders (`?`) in calls to `db.Query()` and `db.Exec()` are interpolated into a single query string with given parameters. This reduces the number of roundtrips, since the driver has to prepare a statement, execute it with given parameters and close the statement again with `interpolateParams=false`. + +*This can not be used together with the multibyte encodings BIG5, CP932, GB2312, GBK or SJIS. These are blacklisted as they may [introduce a SQL injection vulnerability](http://stackoverflow.com/a/12118602/3430118)!* + +##### `loc` + +``` +Type: string +Valid Values: +Default: UTC +``` + +Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](https://golang.org/pkg/time/#LoadLocation) for details. + +Note that this sets the location for time.Time values but does not change MySQL's [time_zone setting](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html). For that see the [time_zone system variable](#system-variables), which can also be set as a DSN parameter. + +Please keep in mind, that param values must be [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. + +##### `maxAllowedPacket` +``` +Type: decimal number +Default: 4194304 +``` + +Max packet size allowed in bytes. The default value is 4 MiB and should be adjusted to match the server settings. `maxAllowedPacket=0` can be used to automatically fetch the `max_allowed_packet` variable from server *on every connection*. + +##### `multiStatements` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. + +When `multiStatements` is used, `?` parameters must only be used in the first statement. + +##### `parseTime` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`parseTime=true` changes the output type of `DATE` and `DATETIME` values to `time.Time` instead of `[]byte` / `string` +The date or datetime like `0000-00-00 00:00:00` is converted into zero value of `time.Time`. + + +##### `readTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O read timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + +##### `rejectReadOnly` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + + +`rejectReadOnly=true` causes the driver to reject read-only connections. This +is for a possible race condition during an automatic failover, where the mysql +client gets connected to a read-only replica after the failover. + +Note that this should be a fairly rare case, as an automatic failover normally +happens when the primary is down, and the race condition shouldn't happen +unless it comes back up online as soon as the failover is kicked off. On the +other hand, when this happens, a MySQL application can get stuck on a +read-only connection until restarted. It is however fairly easy to reproduce, +for example, using a manual failover on AWS Aurora's MySQL-compatible cluster. + +If you are not relying on read-only transactions to reject writes that aren't +supposed to happen, setting this on some MySQL providers (such as AWS Aurora) +is safer for failovers. + +Note that ERROR 1290 can be returned for a `read-only` server and this option will +cause a retry for that error. However the same error number is used for some +other cases. You should ensure your application will never cause an ERROR 1290 +except for `read-only` mode when enabling this option. + + +##### `serverPubKey` + +``` +Type: string +Valid Values: +Default: none +``` + +Server public keys can be registered with [`mysql.RegisterServerPubKey`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterServerPubKey), which can then be used by the assigned name in the DSN. +Public keys are used to transmit encrypted data, e.g. for authentication. +If the server's public key is known, it should be set manually to avoid expensive and potentially insecure transmissions of the public key from the server to the client each time it is required. + + +##### `timeout` + +``` +Type: duration +Default: OS default +``` + +Timeout for establishing connections, aka dial timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + + +##### `tls` + +``` +Type: bool / string +Valid Values: true, false, skip-verify, preferred, +Default: false +``` + +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side) or use `preferred` to use TLS only when advertised by the server. This is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Neither `skip-verify` nor `preferred` add any reliable security. You can use a custom TLS config after registering it with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). + + +##### `writeTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O write timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + + +##### System Variables + +Any other parameters are interpreted as system variables: + * `=`: `SET =` + * `=`: `SET =` + * `=%27%27`: `SET =''` + +Rules: +* The values for string variables must be quoted with `'`. +* The values must also be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed! + (which implies values of string variables must be wrapped with `%27`). + +Examples: + * `autocommit=1`: `SET autocommit=1` + * [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'` + * [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'` + + +#### Examples +``` +user@unix(/path/to/socket)/dbname +``` + +``` +root:pw@unix(/tmp/mysql.sock)/myDatabase?loc=Local +``` + +``` +user:password@tcp(localhost:5555)/dbname?tls=skip-verify&autocommit=true +``` + +Treat warnings as errors by setting the system variable [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html): +``` +user:password@/dbname?sql_mode=TRADITIONAL +``` + +TCP via IPv6: +``` +user:password@tcp([de:ad:be:ef::ca:fe]:80)/dbname?timeout=90s&collation=utf8mb4_unicode_ci +``` + +TCP on a remote host, e.g. Amazon RDS: +``` +id:password@tcp(your-amazonaws-uri.com:3306)/dbname +``` + +Google Cloud SQL on App Engine (First Generation MySQL Server): +``` +user@cloudsql(project-id:instance-name)/dbname +``` + +Google Cloud SQL on App Engine (Second Generation MySQL Server): +``` +user@cloudsql(project-id:regionname:instance-name)/dbname +``` + +TCP using default port (3306) on localhost: +``` +user:password@tcp/dbname?charset=utf8mb4,utf8&sys_var=esc%40ped +``` + +Use the default protocol (tcp) and host (localhost:3306): +``` +user:password@/dbname +``` + +No Database preselected: +``` +user:password@/ +``` + + +### Connection pool and timeouts +The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively. + +## `ColumnType` Support +This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. + +## `context.Context` Support +Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts. +See [context support in the database/sql package](https://golang.org/doc/go1.8#database_sql) for more details. + + +### `LOAD DATA LOCAL INFILE` support +For this feature you need direct access to the package. Therefore you must change the import path (no `_`): +```go +import "github.com/go-sql-driver/mysql" +``` + +Files must be whitelisted by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the Whitelist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). + +To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. + +See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. + + +### `time.Time` support +The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program. + +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical equivalent in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. + +**Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). + +Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. + + +### Unicode support +Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. + +Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. + +Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. + +See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. + +## Testing / Development +To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. + +Go-MySQL-Driver is not feature-complete yet. Your help is very appreciated. +If you want to contribute, you can work on an [open issue](https://github.com/go-sql-driver/mysql/issues?state=open) or review a [pull request](https://github.com/go-sql-driver/mysql/pulls). + +See the [Contribution Guidelines](https://github.com/go-sql-driver/mysql/blob/master/CONTRIBUTING.md) for details. + +--------------------------------------- + +## License +Go-MySQL-Driver is licensed under the [Mozilla Public License Version 2.0](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) + +Mozilla summarizes the license scope as follows: +> MPL: The copyleft applies to any files containing MPLed code. + + +That means: + * You can **use** the **unchanged** source code both in private and commercially. + * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0). + * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged**. + +Please read the [MPL 2.0 FAQ](https://www.mozilla.org/en-US/MPL/2.0/FAQ/) if you have further questions regarding the license. + +You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE). + +![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow") + diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go index be41f2e..914e662 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go @@ -11,9 +11,15 @@ package mysql import ( + "context" + "net" + "google.golang.org/appengine/cloudsql" ) func init() { - RegisterDial("cloudsql", cloudsql.Dial) + RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { + // XXX: the cloudsql driver still does not export a Context-aware dialer. + return cloudsql.Dial(instance) + }) } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go index 2f61ecd..fec7040 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go @@ -234,64 +234,64 @@ func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) erro if err != nil { return err } - return mc.writeAuthSwitchPacket(enc, false) + return mc.writeAuthSwitchPacket(enc) } -func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, bool, error) { +func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { switch plugin { case "caching_sha2_password": authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) - return authResp, false, nil + return authResp, nil case "mysql_old_password": if !mc.cfg.AllowOldPasswords { - return nil, false, ErrOldPassword + return nil, ErrOldPassword } // Note: there are edge cases where this should work but doesn't; // this is currently "wontfix": // https://github.com/go-sql-driver/mysql/issues/184 - authResp := scrambleOldPassword(authData[:8], mc.cfg.Passwd) - return authResp, true, nil + authResp := append(scrambleOldPassword(authData[:8], mc.cfg.Passwd), 0) + return authResp, nil case "mysql_clear_password": if !mc.cfg.AllowCleartextPasswords { - return nil, false, ErrCleartextPassword + return nil, ErrCleartextPassword } // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil case "mysql_native_password": if !mc.cfg.AllowNativePasswords { - return nil, false, ErrNativePassword + return nil, ErrNativePassword } // https://dev.mysql.com/doc/internals/en/secure-password-authentication.html // Native password authentication only need and will need 20-byte challenge. authResp := scramblePassword(authData[:20], mc.cfg.Passwd) - return authResp, false, nil + return authResp, nil case "sha256_password": if len(mc.cfg.Passwd) == 0 { - return nil, true, nil + return []byte{0}, nil } if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil } pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - return []byte{1}, false, nil + return []byte{1}, nil } // encrypted password enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) - return enc, false, err + return enc, err default: errLog.Print("unknown auth plugin:", plugin) - return nil, false, ErrUnknownPlugin + return nil, ErrUnknownPlugin } } @@ -315,11 +315,11 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { plugin = newPlugin - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { return err } - if err = mc.writeAuthSwitchPacket(authResp, addNUL); err != nil { + if err = mc.writeAuthSwitchPacket(authResp); err != nil { return err } @@ -352,7 +352,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { case cachingSha2PasswordPerformFullAuthentication: if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - err = mc.writeAuthSwitchPacket([]byte(mc.cfg.Passwd), true) + err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0)) if err != nil { return err } @@ -360,13 +360,15 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - data := mc.buf.takeSmallBuffer(4 + 1) + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + return err + } data[4] = cachingSha2PasswordRequestPublicKey mc.writePacket(data) // parse public key - data, err := mc.readPacket() - if err != nil { + if data, err = mc.readPacket(); err != nil { return err } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go new file mode 100644 index 0000000..1920ef3 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go @@ -0,0 +1,1330 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "testing" +) + +var testPubKey = []byte("-----BEGIN PUBLIC KEY-----\n" + + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAol0Z8G8U+25Btxk/g/fm\n" + + "UAW/wEKjQCTjkibDE4B+qkuWeiumg6miIRhtilU6m9BFmLQSy1ltYQuu4k17A4tQ\n" + + "rIPpOQYZges/qsDFkZh3wyK5jL5WEFVdOasf6wsfszExnPmcZS4axxoYJfiuilrN\n" + + "hnwinBAqfi3S0sw5MpSI4Zl1AbOrHG4zDI62Gti2PKiMGyYDZTS9xPrBLbN95Kby\n" + + "FFclQLEzA9RJcS1nHFsWtRgHjGPhhjCQxEm9NQ1nePFhCfBfApyfH1VM2VCOQum6\n" + + "Ci9bMuHWjTjckC84mzF99kOxOWVU7mwS6gnJqBzpuz8t3zq8/iQ2y7QrmZV+jTJP\n" + + "WQIDAQAB\n" + + "-----END PUBLIC KEY-----\n") + +var testPubKeyRSA *rsa.PublicKey + +func init() { + block, _ := pem.Decode(testPubKey) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + panic(err) + } + testPubKeyRSA = pub.(*rsa.PublicKey) +} + +func TestScrambleOldPass(t *testing.T) { + scramble := []byte{9, 8, 7, 6, 5, 4, 3, 2} + vectors := []struct { + pass string + out string + }{ + {" pass", "47575c5a435b4251"}, + {"pass ", "47575c5a435b4251"}, + {"123\t456", "575c47505b5b5559"}, + {"C0mpl!ca ted#PASS123", "5d5d554849584a45"}, + } + for _, tuple := range vectors { + ours := scrambleOldPassword(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed old password %q", tuple.pass) + } + } +} + +func TestScrambleSHA256Pass(t *testing.T) { + scramble := []byte{10, 47, 74, 111, 75, 73, 34, 48, 88, 76, 114, 74, 37, 13, 3, 80, 82, 2, 23, 21} + vectors := []struct { + pass string + out string + }{ + {"secret", "f490e76f66d9d86665ce54d98c78d0acfe2fb0b08b423da807144873d30b312c"}, + {"secret2", "abc3934a012cf342e876071c8ee202de51785b430258a7a0138bc79c4d800bc6"}, + } + for _, tuple := range vectors { + ours := scrambleSHA256Password(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed SHA256 password %q", tuple.pass) + } + } +} + +func TestAuthFastCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{102, 32, 5, 35, 143, 161, 140, 241, 171, 232, 56, + 139, 43, 14, 107, 196, 249, 170, 147, 60, 220, 204, 120, 178, 214, 15, + 184, 150, 26, 61, 57, 235} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 3, // Fast Auth Success + 7, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // pub key response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{1, 0, 0, 3, 2, 0, 1, 0, 5}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCleartextPasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthFastCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowNativePasswords = false + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthFastNativePassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{53, 177, 140, 159, 251, 189, 127, 53, 109, 252, + 172, 50, 211, 192, 240, 164, 26, 48, 207, 45} + if writtenAuthRespLen != 20 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{1} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + // hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + + // unset TLS config to prevent the actual establishment of a TLS wrapper + mc.cfg.tls = nil + + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 6, 1}, testPubKey...), + + // OK + {7, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 4 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Pub Key Request + 1, 0, 0, 5, 2, + + // 3. Packet: Encrypted Password + 0, 1, 0, 7, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {2, 0, 0, 4, 1, 4}, // Perform Full Authentication + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Cleartext password + 7, 0, 0, 5, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + conn.maxReads = 1 + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthSwitchCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = false + + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + conn.maxReads = 1 + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthSwitchNativePassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{20, 0, 0, 3, 202, 41, 195, 164, 34, 226, 49, 103, + 21, 211, 167, 199, 227, 116, 8, 48, 57, 71, 149, 146} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchOldPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +// Same to TestAuthSwitchOldPasswordNotAllowed, but use OldAuthSwitch request. +func TestOldAuthSwitchNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +func TestAuthSwitchOldPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPassword, but use OldAuthSwitch request. +func TestOldAuthSwitch(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} +func TestAuthSwitchOldPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPasswordEmpty, but use OldAuthSwitch request. +func TestOldAuthSwitchPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // OldAuthSwitch request. + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Empty Password + 1, 0, 0, 3, 0, + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Pub Key Request + 1, 0, 0, 3, 1, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Encrypted Password + 0, 1, 0, 3, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Cleartext Password + 7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go new file mode 100644 index 0000000..3e25a3b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go @@ -0,0 +1,373 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "fmt" + "math" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +type TB testing.B + +func (tb *TB) check(err error) { + if err != nil { + tb.Fatal(err) + } +} + +func (tb *TB) checkDB(db *sql.DB, err error) *sql.DB { + tb.check(err) + return db +} + +func (tb *TB) checkRows(rows *sql.Rows, err error) *sql.Rows { + tb.check(err) + return rows +} + +func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt { + tb.check(err) + return stmt +} + +func initDB(b *testing.B, queries ...string) *sql.DB { + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + for _, query := range queries { + if _, err := db.Exec(query); err != nil { + b.Fatalf("error on %q: %v", query, err) + } + } + return db +} + +const concurrencyLevel = 10 + +func BenchmarkQuery(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + var got string + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Errorf("query = %q; want one", got) + wg.Done() + return + } + } + }() + } +} + +func BenchmarkExec(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := tb.checkDB(sql.Open("mysql", dsn)) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("DO 1")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + if _, err := stmt.Exec(); err != nil { + b.Fatal(err.Error()) + } + } + }() + } +} + +// data, but no db writes +var roundtripSample []byte + +func initRoundtripBenchmarks() ([]byte, int, int) { + if roundtripSample == nil { + roundtripSample = []byte(strings.Repeat("0123456789abcdef", 1024*1024)) + } + return roundtripSample, 16, len(roundtripSample) +} + +func BenchmarkRoundtripTxt(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + sampleString := string(sample) + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + b.StartTimer() + var result string + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sampleString[0:length] + rows := tb.checkRows(db.Query(`SELECT "` + test + `"`)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if result != test { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkRoundtripBin(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + stmt := tb.checkStmt(db.Prepare("SELECT ?")) + defer stmt.Close() + b.StartTimer() + var result sql.RawBytes + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sample[0:length] + rows := tb.checkRows(stmt.Query(test)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if !bytes.Equal(result, test) { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkInterpolation(b *testing.B) { + mc := &mysqlConn{ + cfg: &Config{ + InterpolateParams: true, + Loc: time.UTC, + }, + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + buf: newBuffer(nil), + } + + args := []driver.Value{ + int64(42424242), + float64(math.Pi), + false, + time.Unix(1423411542, 807015000), + []byte("bytes containing special chars ' \" \a \x00"), + "string containing special chars ' \" \a \x00", + } + q := "SELECT ?, ?, ?, ?, ?, ?" + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := mc.interpolateParams(q, args) + if err != nil { + b.Fatal(err) + } + } +} + +func benchmarkQueryContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + var got string + for pb.Next() { + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Fatalf("query = %q; want one", got) + } + } + }) +} + +func BenchmarkQueryContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +func benchmarkExecContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "DO 1")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := stmt.ExecContext(ctx); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkExecContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +// BenchmarkQueryRawBytes benchmarks fetching 100 blobs using sql.RawBytes. +// "size=" means size of each blobs. +func BenchmarkQueryRawBytes(b *testing.B) { + var sizes []int = []int{100, 1000, 2000, 4000, 8000, 12000, 16000, 32000, 64000, 256000} + db := initDB(b, + "DROP TABLE IF EXISTS bench_rawbytes", + "CREATE TABLE bench_rawbytes (id INT PRIMARY KEY, val LONGBLOB)", + ) + defer db.Close() + + blob := make([]byte, sizes[len(sizes)-1]) + for i := range blob { + blob[i] = 42 + } + for i := 0; i < 100; i++ { + _, err := db.Exec("INSERT INTO bench_rawbytes VALUES (?, ?)", i, blob) + if err != nil { + b.Fatal(err) + } + } + + for _, s := range sizes { + b.Run(fmt.Sprintf("size=%v", s), func(b *testing.B) { + db.SetMaxIdleConns(0) + db.SetMaxIdleConns(1) + b.ReportAllocs() + b.ResetTimer() + + for j := 0; j < b.N; j++ { + rows, err := db.Query("SELECT LEFT(val, ?) as v FROM bench_rawbytes", s) + if err != nil { + b.Fatal(err) + } + nrows := 0 + for rows.Next() { + var buf sql.RawBytes + err := rows.Scan(&buf) + if err != nil { + b.Fatal(err) + } + if len(buf) != s { + b.Fatalf("size mismatch: expected %v, got %v", s, len(buf)) + } + nrows++ + } + rows.Close() + if nrows != 100 { + b.Fatalf("numbers of rows mismatch: expected %v, got %v", 100, nrows) + } + } + }) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go index eb4748b..0774c5c 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -15,47 +15,69 @@ import ( ) const defaultBufSize = 4096 +const maxCachedBufSize = 256 * 1024 // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. +// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { - buf []byte + buf []byte // buf is a byte buffer who's length and capacity are equal. nc net.Conn idx int length int timeout time.Duration + dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer + flipcnt uint // flipccnt is the current buffer counter for double-buffering } +// newBuffer allocates and returns a new buffer. func newBuffer(nc net.Conn) buffer { - var b [defaultBufSize]byte + fg := make([]byte, defaultBufSize) return buffer{ - buf: b[:], - nc: nc, + buf: fg, + nc: nc, + dbuf: [2][]byte{fg, nil}, } } +// flip replaces the active buffer with the background buffer +// this is a delayed flip that simply increases the buffer counter; +// the actual flip will be performed the next time we call `buffer.fill` +func (b *buffer) flip() { + b.flipcnt += 1 +} + // fill reads into the buffer until at least _need_ bytes are in it func (b *buffer) fill(need int) error { n := b.length + // fill data into its double-buffering target: if we've called + // flip on this buffer, we'll be copying to the background buffer, + // and then filling it with network data; otherwise we'll just move + // the contents of the current buffer to the front before filling it + dest := b.dbuf[b.flipcnt&1] - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) - } - - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { + // grow buffer if necessary to fit the whole packet. + if need > len(dest) { // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf + dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) + + // if the allocated buffer is not too large, move it to backing storage + // to prevent extra allocations on applications that perform large reads + if len(dest) <= maxCachedBufSize { + b.dbuf[b.flipcnt&1] = dest + } } + // if we're filling the fg buffer, move the existing data to the start of it. + // if we're filling the bg buffer, copy over the data + if n > 0 { + copy(dest[:n], b.buf[b.idx:]) + } + + b.buf = dest b.idx = 0 for { @@ -105,43 +127,56 @@ func (b *buffer) readNext(need int) ([]byte, error) { return b.buf[offset:b.idx], nil } -// returns a buffer with the requested size. +// takeBuffer returns a buffer with the requested size. // If possible, a slice from the existing buffer is returned. // Otherwise a bigger buffer is made. // Only one buffer (total) can be used at a time. -func (b *buffer) takeBuffer(length int) []byte { +func (b *buffer) takeBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } // test (cheap) general case first - if length <= defaultBufSize || length <= cap(b.buf) { - return b.buf[:length] + if length <= cap(b.buf) { + return b.buf[:length], nil } if length < maxPacketSize { b.buf = make([]byte, length) - return b.buf + return b.buf, nil } - return make([]byte, length) + + // buffer is larger than we want to store. + return make([]byte, length), nil } -// shortcut which can be used if the requested buffer is guaranteed to be -// smaller than defaultBufSize +// takeSmallBuffer is shortcut which can be used if length is +// known to be smaller than defaultBufSize. // Only one buffer (total) can be used at a time. -func (b *buffer) takeSmallBuffer(length int) []byte { +func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } - return b.buf[:length] + return b.buf[:length], nil } // takeCompleteBuffer returns the complete existing buffer. // This can be used if the necessary buffer size is unknown. +// cap and len of the returned buffer will be equal. // Only one buffer (total) can be used at a time. -func (b *buffer) takeCompleteBuffer() []byte { +func (b *buffer) takeCompleteBuffer() ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } - return b.buf + return b.buf, nil +} + +// store stores buf, an updated buffer, if its suitable to do so. +func (b *buffer) store(buf []byte) error { + if b.length > 0 { + return ErrBusyBuffer + } else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) { + b.buf = buf[:cap(buf)] + } + return nil } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go index 136c9e4..8d2b556 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go @@ -8,183 +8,190 @@ package mysql -const defaultCollation = "utf8_general_ci" +const defaultCollation = "utf8mb4_general_ci" const binaryCollation = "binary" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. +// +// ucs2, utf16, and utf32 can't be used for connection charset. +// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset +// They are commented out to reduce this map. var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + //"ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + //"utf16_general_ci": 54, + //"utf16_bin": 55, + //"utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + //"utf32_general_ci": 60, + //"utf32_bin": 61, + //"utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "utf8_tolower_ci": 76, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + //"ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + //"utf16_unicode_ci": 101, + //"utf16_icelandic_ci": 102, + //"utf16_latvian_ci": 103, + //"utf16_romanian_ci": 104, + //"utf16_slovenian_ci": 105, + //"utf16_polish_ci": 106, + //"utf16_estonian_ci": 107, + //"utf16_spanish_ci": 108, + //"utf16_swedish_ci": 109, + //"utf16_turkish_ci": 110, + //"utf16_czech_ci": 111, + //"utf16_danish_ci": 112, + //"utf16_lithuanian_ci": 113, + //"utf16_slovak_ci": 114, + //"utf16_spanish2_ci": 115, + //"utf16_roman_ci": 116, + //"utf16_persian_ci": 117, + //"utf16_esperanto_ci": 118, + //"utf16_hungarian_ci": 119, + //"utf16_sinhala_ci": 120, + //"utf16_german2_ci": 121, + //"utf16_croatian_ci": 122, + //"utf16_unicode_520_ci": 123, + //"utf16_vietnamese_ci": 124, + //"ucs2_unicode_ci": 128, + //"ucs2_icelandic_ci": 129, + //"ucs2_latvian_ci": 130, + //"ucs2_romanian_ci": 131, + //"ucs2_slovenian_ci": 132, + //"ucs2_polish_ci": 133, + //"ucs2_estonian_ci": 134, + //"ucs2_spanish_ci": 135, + //"ucs2_swedish_ci": 136, + //"ucs2_turkish_ci": 137, + //"ucs2_czech_ci": 138, + //"ucs2_danish_ci": 139, + //"ucs2_lithuanian_ci": 140, + //"ucs2_slovak_ci": 141, + //"ucs2_spanish2_ci": 142, + //"ucs2_roman_ci": 143, + //"ucs2_persian_ci": 144, + //"ucs2_esperanto_ci": 145, + //"ucs2_hungarian_ci": 146, + //"ucs2_sinhala_ci": 147, + //"ucs2_german2_ci": 148, + //"ucs2_croatian_ci": 149, + //"ucs2_unicode_520_ci": 150, + //"ucs2_vietnamese_ci": 151, + //"ucs2_general_mysql500_ci": 159, + //"utf32_unicode_ci": 160, + //"utf32_icelandic_ci": 161, + //"utf32_latvian_ci": 162, + //"utf32_romanian_ci": 163, + //"utf32_slovenian_ci": 164, + //"utf32_polish_ci": 165, + //"utf32_estonian_ci": 166, + //"utf32_spanish_ci": 167, + //"utf32_swedish_ci": 168, + //"utf32_turkish_ci": 169, + //"utf32_czech_ci": 170, + //"utf32_danish_ci": 171, + //"utf32_lithuanian_ci": 172, + //"utf32_slovak_ci": 173, + //"utf32_spanish2_ci": 174, + //"utf32_roman_ci": 175, + //"utf32_persian_ci": 176, + //"utf32_esperanto_ci": 177, + //"utf32_hungarian_ci": 178, + //"utf32_sinhala_ci": 179, + //"utf32_german2_ci": 180, + //"utf32_croatian_ci": 181, + //"utf32_unicode_520_ci": 182, + //"utf32_vietnamese_ci": 183, "utf8_unicode_ci": 192, "utf8_icelandic_ci": 193, "utf8_latvian_ci": 194, @@ -234,18 +241,25 @@ var collations = map[string]byte{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, + "gb18030_unicode_520_ci": 250, + "utf8mb4_0900_ai_ci": 255, } // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. var unsafeCollations = map[string]bool{ - "big5_chinese_ci": true, - "sjis_japanese_ci": true, - "gbk_chinese_ci": true, - "big5_bin": true, - "gb2312_bin": true, - "gbk_bin": true, - "sjis_bin": true, - "cp932_japanese_ci": true, - "cp932_bin": true, + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, + "gb18030_chinese_ci": true, + "gb18030_bin": true, + "gb18030_unicode_520_ci": true, } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go new file mode 100644 index 0000000..cc47aa5 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -0,0 +1,53 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows,!appengine + +package mysql + +import ( + "errors" + "io" + "net" + "syscall" +) + +var errUnexpectedRead = errors.New("unexpected read from socket") + +func connCheck(c net.Conn) error { + var ( + n int + err error + buff [1]byte + ) + + sconn, ok := c.(syscall.Conn) + if !ok { + return nil + } + rc, err := sconn.SyscallConn() + if err != nil { + return err + } + rerr := rc.Read(func(fd uintptr) bool { + n, err = syscall.Read(int(fd), buff[:]) + return true + }) + switch { + case rerr != nil: + return rerr + case n == 0 && err == nil: + return io.EOF + case n > 0: + return errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + return nil + default: + return err + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go new file mode 100644 index 0000000..fd01f64 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -0,0 +1,17 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows appengine + +package mysql + +import "net" + +func connCheck(c net.Conn) error { + return nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go new file mode 100644 index 0000000..b7234b0 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go @@ -0,0 +1,38 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10,!windows + +package mysql + +import ( + "testing" + "time" +) + +func TestStaleConnectionChecks(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("SET @@SESSION.wait_timeout = 2") + + if err := dbt.db.Ping(); err != nil { + dbt.Fatal(err) + } + + // wait for MySQL to close our connection + time.Sleep(3 * time.Second) + + tx, err := dbt.db.Begin() + if err != nil { + dbt.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + dbt.Fatal(err) + } + }) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go index 911be20..565a548 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go @@ -19,19 +19,10 @@ import ( "time" ) -// a copy of context.Context for Go 1.7 and earlier -type mysqlContext interface { - Done() <-chan struct{} - Err() error - - // defined in context.Context, but not used in this driver: - // Deadline() (deadline time.Time, ok bool) - // Value(key interface{}) interface{} -} - type mysqlConn struct { buf buffer netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. affectedRows uint64 insertId uint64 cfg *Config @@ -42,10 +33,11 @@ type mysqlConn struct { status statusFlag sequence uint8 parseTime bool + reset bool // set when the Go SQL package calls ResetSession // for context support (Go 1.8+) watching bool - watcher chan<- mysqlContext + watcher chan<- context.Context closech chan struct{} finished chan<- struct{} canceled atomicError // set non-nil if conn is canceled @@ -192,10 +184,10 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - buf := mc.buf.takeCompleteBuffer() - if buf == nil { + buf, err := mc.buf.takeCompleteBuffer() + if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return "", ErrInvalidConn } buf = buf[:0] @@ -221,6 +213,9 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin switch v := arg.(type) { case int64: buf = strconv.AppendInt(buf, v, 10) + case uint64: + // Handle uint64 explicitly because our custom ConvertValue emits unsigned values + buf = strconv.AppendUint(buf, v, 10) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: @@ -475,7 +470,7 @@ func (mc *mysqlConn) Ping(ctx context.Context) (err error) { defer mc.finish() if err = mc.writeCommandPacket(comPing); err != nil { - return + return mc.markBadConn(err) } return mc.readResultOK() @@ -595,33 +590,32 @@ func (mc *mysqlConn) watchCancel(ctx context.Context) error { mc.cleanup() return nil } + // When ctx is already cancelled, don't watch it. + if err := ctx.Err(); err != nil { + return err + } + // When ctx is not cancellable, don't watch it. if ctx.Done() == nil { return nil } - - mc.watching = true - select { - default: - case <-ctx.Done(): - return ctx.Err() - } + // When watcher is not alive, can't watch it. if mc.watcher == nil { return nil } + mc.watching = true mc.watcher <- ctx - return nil } func (mc *mysqlConn) startWatcher() { - watcher := make(chan mysqlContext, 1) + watcher := make(chan context.Context, 1) mc.watcher = watcher finished := make(chan struct{}) mc.finished = finished go func() { for { - var ctx mysqlContext + var ctx context.Context select { case ctx = <-watcher: case <-mc.closech: @@ -650,5 +644,6 @@ func (mc *mysqlConn) ResetSession(ctx context.Context) error { if mc.closed.IsSet() { return driver.ErrBadConn } + mc.reset = true return nil } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go new file mode 100644 index 0000000..19c17ff --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go @@ -0,0 +1,175 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "errors" + "net" + "testing" +) + +func TestInterpolateParams(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42), "gopher"}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT 42+'gopher'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + +func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42)}) + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +// We don't support placeholder in string literal for now. +// https://github.com/go-sql-driver/mysql/pull/490 +func TestInterpolateParamsPlaceholderInString(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT 'abc?xyz',?", []driver.Value{int64(42)}) + // When InterpolateParams support string literal, this should return `"SELECT 'abc?xyz', 42` + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +func TestInterpolateParamsUint64(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?", []driver.Value{uint64(42)}) + if err != nil { + t.Errorf("Expected err=nil, got err=%#v, q=%#v", err, q) + } + if q != "SELECT 42" { + t.Errorf("Expected uint64 interpolation to work, got q=%#v", q) + } +} + +func TestCheckNamedValue(t *testing.T) { + value := driver.NamedValue{Value: ^uint64(0)} + x := &mysqlConn{} + err := x.CheckNamedValue(&value) + + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if value.Value != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", value.Value, value.Value) + } +} + +// TestCleanCancel tests passed context is cancelled at start. +// No packet should be sent. Connection should keep current status. +func TestCleanCancel(t *testing.T) { + mc := &mysqlConn{ + closech: make(chan struct{}), + } + mc.startWatcher() + defer mc.cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + for i := 0; i < 3; i++ { // Repeat same behavior + err := mc.Ping(ctx) + if err != context.Canceled { + t.Errorf("expected context.Canceled, got %#v", err) + } + + if mc.closed.IsSet() { + t.Error("expected mc is not closed, closed actually") + } + + if mc.watching { + t.Error("expected watching is false, but true") + } + } +} + +func TestPingMarkBadConnection(t *testing.T) { + nc := badConnection{err: errors.New("boom")} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + } + + err := ms.Ping(context.Background()) + + if err != driver.ErrBadConn { + t.Errorf("expected driver.ErrBadConn, got %#v", err) + } +} + +func TestPingErrInvalidConn(t *testing.T) { + nc := badConnection{err: errors.New("failed to write"), n: 10} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + closech: make(chan struct{}), + } + + err := ms.Ping(context.Background()) + + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %#v", err) + } +} + +type badConnection struct { + n int + err error + net.Conn +} + +func (bc badConnection) Write(b []byte) (n int, err error) { + return bc.n, bc.err +} + +func (bc badConnection) Close() error { + return nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go new file mode 100644 index 0000000..5aaaba4 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go @@ -0,0 +1,143 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "net" +) + +type connector struct { + cfg *Config // immutable private copy. +} + +// Connect implements driver.Connector interface. +// Connect returns a connection to the database. +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + cfg: c.cfg, + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + mc.netConn, err = dial(ctx, mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + } + + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Temporary() { + errLog.Print("net.Error from Dial()': ", nerr.Error()) + return nil, driver.ErrBadConn + } + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +// Driver implements driver.Connector interface. +// Driver returns &MySQLDriver{}. +func (c *connector) Driver() driver.Driver { + return &MySQLDriver{} +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go index ba12978..1f9decf 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go @@ -17,6 +17,7 @@ package mysql import ( + "context" "database/sql" "database/sql/driver" "net" @@ -29,135 +30,54 @@ type MySQLDriver struct{} // DialFunc is a function which can be used to establish the network connection. // Custom dial functions must be registered with RegisterDial +// +// Deprecated: users should register a DialContextFunc instead type DialFunc func(addr string) (net.Conn, error) +// DialContextFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDialContext +type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error) + var ( dialsLock sync.RWMutex - dials map[string]DialFunc + dials map[string]DialContextFunc ) -// RegisterDial registers a custom dial function. It can then be used by the +// RegisterDialContext registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { +// The current context for the connection and its address is passed to the dial function. +func RegisterDialContext(net string, dial DialContextFunc) { dialsLock.Lock() defer dialsLock.Unlock() if dials == nil { - dials = make(map[string]DialFunc) + dials = make(map[string]DialContextFunc) } dials[net] = dial } +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +// +// Deprecated: users should call RegisterDialContext instead +func RegisterDial(network string, dial DialFunc) { + RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) { + return dial(addr) + }) +} + // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how -// the DSN string is formated +// the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxAllowedPacket: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - closech: make(chan struct{}), - } - mc.cfg, err = ParseDSN(dsn) + cfg, err := ParseDSN(dsn) if err != nil { return nil, err } - mc.parseTime = mc.cfg.ParseTime - - // Connect to Server - dialsLock.RLock() - dial, ok := dials[mc.cfg.Net] - dialsLock.RUnlock() - if ok { - mc.netConn, err = dial(mc.cfg.Addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.Timeout} - mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) + c := &connector{ + cfg: cfg, } - if err != nil { - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - // Call startWatcher for context support (From Go 1.8) - mc.startWatcher() - - mc.buf = newBuffer(mc.netConn) - - // Set I/O timeouts - mc.buf.timeout = mc.cfg.ReadTimeout - mc.writeTimeout = mc.cfg.WriteTimeout - - // Reading Handshake Initialization Packet - authData, plugin, err := mc.readHandshakePacket() - if err != nil { - mc.cleanup() - return nil, err - } - if plugin == "" { - plugin = defaultAuthPlugin - } - - // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) - if err != nil { - // try the default auth plugin, if using the requested plugin failed - errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) - plugin = defaultAuthPlugin - authResp, addNUL, err = mc.auth(authData, plugin) - if err != nil { - mc.cleanup() - return nil, err - } - } - if err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin); err != nil { - mc.cleanup() - return nil, err - } - - // Handle response to auth packet, switch methods if possible - if err = mc.handleAuthResult(authData, plugin); err != nil { - // Authentication failed and MySQL has already closed the connection - // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). - // Do not send COM_QUIT, just cleanup and return the error. - mc.cleanup() - return nil, err - } - - if mc.cfg.MaxAllowedPacket > 0 { - mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket - } else { - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxAllowedPacket = stringToInt(maxap) - 1 - } - if mc.maxAllowedPacket < maxPacketSize { - mc.maxWriteSize = mc.maxAllowedPacket - } - - // Handle DSN Params - err = mc.handleParams() - if err != nil { - mc.Close() - return nil, err - } - - return mc, nil + return c.Connect(context.Background()) } func init() { diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go new file mode 100644 index 0000000..eb5a8fe --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go @@ -0,0 +1,37 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "database/sql/driver" +) + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go new file mode 100644 index 0000000..19a0e59 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go @@ -0,0 +1,137 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "net" + "testing" + "time" +) + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go new file mode 100644 index 0000000..3dee1ba --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go @@ -0,0 +1,2996 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "crypto/tls" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "net" + "net/url" + "os" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Ensure that all the driver interfaces are implemented +var ( + _ driver.Rows = &binaryRows{} + _ driver.Rows = &textRows{} +) + +var ( + user string + pass string + prot string + addr string + dbname string + dsn string + netAddr string + available bool +) + +var ( + tDate = time.Date(2012, 6, 14, 0, 0, 0, 0, time.UTC) + sDate = "2012-06-14" + tDateTime = time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC) + sDateTime = "2011-11-20 21:27:37" + tDate0 = time.Time{} + sDate0 = "0000-00-00" + sDateTime0 = "0000-00-00 00:00:00" +) + +// See https://github.com/go-sql-driver/mysql/wiki/Testing +func init() { + // get environment variables + env := func(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue + } + user = env("MYSQL_TEST_USER", "root") + pass = env("MYSQL_TEST_PASS", "") + prot = env("MYSQL_TEST_PROT", "tcp") + addr = env("MYSQL_TEST_ADDR", "localhost:3306") + dbname = env("MYSQL_TEST_DBNAME", "gotest") + netAddr = fmt.Sprintf("%s(%s)", prot, addr) + dsn = fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, pass, netAddr, dbname) + c, err := net.Dial(prot, addr) + if err == nil { + available = true + c.Close() + } +} + +type DBTest struct { + *testing.T + db *sql.DB +} + +type netErrorMock struct { + temporary bool + timeout bool +} + +func (e netErrorMock) Temporary() bool { + return e.temporary +} + +func (e netErrorMock) Timeout() bool { + return e.timeout +} + +func (e netErrorMock) Error() string { + return fmt.Sprintf("mock net error. Temporary: %v, Timeout %v", e.temporary, e.timeout) +} + +func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn += "&multiStatements=true" + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + } +} + +func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + db.Exec("DROP TABLE IF EXISTS test") + + dsn2 := dsn + "&interpolateParams=true" + var db2 *sql.DB + if _, err := ParseDSN(dsn2); err != errInvalidDSNUnsafeCollation { + db2, err = sql.Open("mysql", dsn2) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db2.Close() + } + + dsn3 := dsn + "&multiStatements=true" + var db3 *sql.DB + if _, err := ParseDSN(dsn3); err != errInvalidDSNUnsafeCollation { + db3, err = sql.Open("mysql", dsn3) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db3.Close() + } + + dbt := &DBTest{t, db} + dbt2 := &DBTest{t, db2} + dbt3 := &DBTest{t, db3} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + if db2 != nil { + test(dbt2) + dbt2.db.Exec("DROP TABLE IF EXISTS test") + } + if db3 != nil { + test(dbt3) + dbt3.db.Exec("DROP TABLE IF EXISTS test") + } + } +} + +func (dbt *DBTest) fail(method, query string, err error) { + if len(query) > 300 { + query = "[query too large to print]" + } + dbt.Fatalf("error on %s %s: %s", method, query, err.Error()) +} + +func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) { + res, err := dbt.db.Exec(query, args...) + if err != nil { + dbt.fail("exec", query, err) + } + return res +} + +func (dbt *DBTest) mustQuery(query string, args ...interface{}) (rows *sql.Rows) { + rows, err := dbt.db.Query(query, args...) + if err != nil { + dbt.fail("query", query, err) + } + return rows +} + +func maybeSkip(t *testing.T, err error, skipErrno uint16) { + mySQLErr, ok := err.(*MySQLError) + if !ok { + return + } + + if mySQLErr.Number == skipErrno { + t.Skipf("skipping test for error: %v", err) + } +} + +func TestEmptyQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // just a comment, no query + rows := dbt.mustQuery("--") + defer rows.Close() + // will hang before #255 + if rows.Next() { + dbt.Errorf("next on rows must be false") + } + }) +} + +func TestCRUD(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + + // Test for unexpected data + var out bool + rows := dbt.mustQuery("SELECT * FROM test") + if rows.Next() { + dbt.Error("unexpected data in empty table") + } + rows.Close() + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + id, err := res.LastInsertId() + if err != nil { + dbt.Fatalf("res.LastInsertId() returned error: %s", err.Error()) + } + if id != 0 { + dbt.Fatalf("expected InsertId 0, got %d", id) + } + + // Read + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if true != out { + dbt.Errorf("true != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Update + res = dbt.mustExec("UPDATE test SET value = ? WHERE value = ?", false, true) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check Update + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if false != out { + dbt.Errorf("false != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Delete + res = dbt.mustExec("DELETE FROM test WHERE value = ?", false) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check for unexpected rows + res = dbt.mustExec("DELETE FROM test") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 0 { + dbt.Fatalf("expected 0 affected row, got %d", count) + } + }) +} + +func TestMultiQuery(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE `test` (`id` int(11) NOT NULL, `value` int(11) NOT NULL) ") + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1, 1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Update + res = dbt.mustExec("UPDATE test SET value = 3 WHERE id = 1; UPDATE test SET value = 4 WHERE id = 1; UPDATE test SET value = 5 WHERE id = 1;") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Read + var out int + rows := dbt.mustQuery("SELECT value FROM test WHERE id=1;") + if rows.Next() { + rows.Scan(&out) + if 5 != out { + dbt.Errorf("5 != %d", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + }) +} + +func TestInt(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [5]string{"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT"} + in := int64(42) + var out int64 + var rows *sql.Rows + + // SIGNED + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // UNSIGNED ZEROFILL + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + " ZEROFILL)") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s ZEROFILL: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s ZEROFILL: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat32(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + in := float32(42.23) + var out float32 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %g != %g", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (42.23)") + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64Placeholder(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (id int, value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (1, 42.23)") + rows = dbt.mustQuery("SELECT value FROM test WHERE id = ?", 1) + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestString(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [6]string{"CHAR(255)", "VARCHAR(255)", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"} + in := "κόσμε üöäßñóùéàâÿœ'îë Árvíztűrő いろはにほへとちりぬるを イロハニホヘト דג סקרן чащах น่าฟังเอย" + var out string + var rows *sql.Rows + + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ") CHARACTER SET utf8") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %s != %s", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // BLOB + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + + id := 2 + in = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. " + + "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." + dbt.mustExec("INSERT INTO test VALUES (?, ?)", id, in) + + err := dbt.db.QueryRow("SELECT value FROM test WHERE id = ?", id).Scan(&out) + if err != nil { + dbt.Fatalf("Error on BLOB-Query: %s", err.Error()) + } else if out != in { + dbt.Errorf("BLOB: %s != %s", in, out) + } + }) +} + +func TestRawBytes(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + v1 := []byte("aaa") + v2 := []byte("bbb") + rows := dbt.mustQuery("SELECT ?, ?", v1, v2) + defer rows.Close() + if rows.Next() { + var o1, o2 sql.RawBytes + if err := rows.Scan(&o1, &o2); err != nil { + dbt.Errorf("Got error: %v", err) + } + if !bytes.Equal(v1, o1) { + dbt.Errorf("expected %v, got %v", v1, o1) + } + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + // https://github.com/go-sql-driver/mysql/issues/765 + // Appending to RawBytes shouldn't overwrite next RawBytes. + o1 = append(o1, "xyzzy"...) + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + } else { + dbt.Errorf("no data") + } + }) +} + +type testValuer struct { + value string +} + +func (tv testValuer) Value() (driver.Value, error) { + return tv.value, nil +} + +func TestValuer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuer{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE test (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + }) +} + +type testValuerWithValidation struct { + value string +} + +func (tv testValuerWithValidation) Value() (driver.Value, error) { + if len(tv.value) == 0 { + return nil, fmt.Errorf("Invalid string valuer. Value must not be empty") + } + + return tv.value, nil +} + +func TestValuerWithValidation(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuerWithValidation{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE testValuer (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO testValuer VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM testValuer") + defer rows.Close() + + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", testValuerWithValidation{""}); err == nil { + dbt.Errorf("Failed to check valuer error") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", nil); err != nil { + dbt.Errorf("Failed to check nil") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", map[string]bool{}); err == nil { + dbt.Errorf("Failed to check not valuer") + } + + dbt.mustExec("DROP TABLE IF EXISTS testValuer") + }) +} + +type timeTests struct { + dbtype string + tlayout string + tests []timeTest +} + +type timeTest struct { + s string // leading "!": do not use t as value in queries + t time.Time +} + +type timeMode byte + +func (t timeMode) String() string { + switch t { + case binaryString: + return "binary:string" + case binaryTime: + return "binary:time.Time" + case textString: + return "text:string" + } + panic("unsupported timeMode") +} + +func (t timeMode) Binary() bool { + switch t { + case binaryString, binaryTime: + return true + } + return false +} + +const ( + binaryString timeMode = iota + binaryTime + textString +) + +func (t timeTest) genQuery(dbtype string, mode timeMode) string { + var inner string + if mode.Binary() { + inner = "?" + } else { + inner = `"%s"` + } + return `SELECT cast(` + inner + ` as ` + dbtype + `)` +} + +func (t timeTest) run(dbt *DBTest, dbtype, tlayout string, mode timeMode) { + var rows *sql.Rows + query := t.genQuery(dbtype, mode) + switch mode { + case binaryString: + rows = dbt.mustQuery(query, t.s) + case binaryTime: + rows = dbt.mustQuery(query, t.t) + case textString: + query = fmt.Sprintf(query, t.s) + rows = dbt.mustQuery(query) + default: + panic("unsupported mode") + } + defer rows.Close() + var err error + if !rows.Next() { + err = rows.Err() + if err == nil { + err = fmt.Errorf("no data") + } + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + var dst interface{} + err = rows.Scan(&dst) + if err != nil { + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + switch val := dst.(type) { + case []uint8: + str := string(val) + if str == t.s { + return + } + if mode.Binary() && dbtype == "DATETIME" && len(str) == 26 && str[:19] == t.s { + // a fix mainly for TravisCI: + // accept full microsecond resolution in result for DATETIME columns + // where the binary protocol was used + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, str, + ) + case time.Time: + if val == t.t { + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, val.Format(tlayout), + ) + default: + fmt.Printf("%#v\n", []interface{}{dbtype, tlayout, mode, t.s, t.t}) + dbt.Errorf("%s [%s]: unhandled type %T (is '%v')", + dbtype, mode, + val, val, + ) + } +} + +func TestDateTime(t *testing.T) { + afterTime := func(t time.Time, d string) time.Time { + dur, err := time.ParseDuration(d) + if err != nil { + panic(err) + } + return t.Add(dur) + } + // NOTE: MySQL rounds DATETIME(x) up - but that's not included in the tests + format := "2006-01-02 15:04:05.999999" + t0 := time.Time{} + tstr0 := "0000-00-00 00:00:00.000000" + testcases := []timeTests{ + {"DATE", format[:10], []timeTest{ + {t: time.Date(2011, 11, 20, 0, 0, 0, 0, time.UTC)}, + {t: t0, s: tstr0[:10]}, + }}, + {"DATETIME", format[:19], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(0)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(1)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 100000000, time.UTC)}, + {t: t0, s: tstr0[:21]}, + }}, + {"DATETIME(6)", format, []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 123456000, time.UTC)}, + {t: t0, s: tstr0}, + }}, + {"TIME", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(0)", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(1)", format[11:21], []timeTest{ + {t: afterTime(t0, "12345600ms")}, + {s: "!-12:34:56.7"}, + {s: "!-838:59:58.9"}, + {s: "!838:59:58.9"}, + {t: t0, s: tstr0[11:21]}, + }}, + {"TIME(6)", format[11:], []timeTest{ + {t: afterTime(t0, "1234567890123000ns")}, + {s: "!-12:34:56.789012"}, + {s: "!-838:59:58.999999"}, + {s: "!838:59:58.999999"}, + {t: t0, s: tstr0[11:]}, + }}, + } + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + microsecsSupported := false + zeroDateSupported := false + var rows *sql.Rows + var err error + rows, err = dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`) + if err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + rows, err = dbt.db.Query(`SELECT cast("0000-00-00" as DATE) = "0000-00-00"`) + if err == nil { + rows.Scan(&zeroDateSupported) + rows.Close() + } + for _, setups := range testcases { + if t := setups.dbtype; !microsecsSupported && t[len(t)-1:] == ")" { + // skip fractional second tests if unsupported by server + continue + } + for _, setup := range setups.tests { + allowBinTime := true + if setup.s == "" { + // fill time string wherever Go can reliable produce it + setup.s = setup.t.Format(setups.tlayout) + } else if setup.s[0] == '!' { + // skip tests using setup.t as source in queries + allowBinTime = false + // fix setup.s - remove the "!" + setup.s = setup.s[1:] + } + if !zeroDateSupported && setup.s == tstr0[:len(setup.s)] { + // skip disallowed 0000-00-00 date + continue + } + setup.run(dbt, setups.dbtype, setups.tlayout, textString) + setup.run(dbt, setups.dbtype, setups.tlayout, binaryString) + if allowBinTime { + setup.run(dbt, setups.dbtype, setups.tlayout, binaryTime) + } + } + } + }) + } +} + +func TestTimestampMicros(t *testing.T) { + format := "2006-01-02 15:04:05.999999" + f0 := format[:19] + f1 := format[:21] + f6 := format[:26] + runTests(t, dsn, func(dbt *DBTest) { + // check if microseconds are supported. + // Do not use timestamp(x) for that check - before 5.5.6, x would mean display width + // and not precision. + // Se last paragraph at http://dev.mysql.com/doc/refman/5.6/en/fractional-seconds.html + microsecsSupported := false + if rows, err := dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`); err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + if !microsecsSupported { + // skip test + return + } + _, err := dbt.db.Exec(` + CREATE TABLE test ( + value0 TIMESTAMP NOT NULL DEFAULT '` + f0 + `', + value1 TIMESTAMP(1) NOT NULL DEFAULT '` + f1 + `', + value6 TIMESTAMP(6) NOT NULL DEFAULT '` + f6 + `' + )`, + ) + if err != nil { + dbt.Error(err) + } + defer dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("INSERT INTO test SET value0=?, value1=?, value6=?", f0, f1, f6) + var res0, res1, res6 string + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Errorf("test contained no selectable values") + } + err = rows.Scan(&res0, &res1, &res6) + if err != nil { + dbt.Error(err) + } + if res0 != f0 { + dbt.Errorf("expected %q, got %q", f0, res0) + } + if res1 != f1 { + dbt.Errorf("expected %q, got %q", f1, res1) + } + if res6 != f6 { + dbt.Errorf("expected %q, got %q", f6, res6) + } + }) +} + +func TestNULL(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + nullStmt, err := dbt.db.Prepare("SELECT NULL") + if err != nil { + dbt.Fatal(err) + } + defer nullStmt.Close() + + nonNullStmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + defer nonNullStmt.Close() + + // NullBool + var nb sql.NullBool + // Invalid + if err = nullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if nb.Valid { + dbt.Error("valid NullBool which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if !nb.Valid { + dbt.Error("invalid NullBool which should be valid") + } else if nb.Bool != true { + dbt.Errorf("Unexpected NullBool value: %t (should be true)", nb.Bool) + } + + // NullFloat64 + var nf sql.NullFloat64 + // Invalid + if err = nullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if nf.Valid { + dbt.Error("valid NullFloat64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if !nf.Valid { + dbt.Error("invalid NullFloat64 which should be valid") + } else if nf.Float64 != float64(1) { + dbt.Errorf("unexpected NullFloat64 value: %f (should be 1.0)", nf.Float64) + } + + // NullInt64 + var ni sql.NullInt64 + // Invalid + if err = nullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if ni.Valid { + dbt.Error("valid NullInt64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if !ni.Valid { + dbt.Error("invalid NullInt64 which should be valid") + } else if ni.Int64 != int64(1) { + dbt.Errorf("unexpected NullInt64 value: %d (should be 1)", ni.Int64) + } + + // NullString + var ns sql.NullString + // Invalid + if err = nullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if ns.Valid { + dbt.Error("valid NullString which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if !ns.Valid { + dbt.Error("invalid NullString which should be valid") + } else if ns.String != `1` { + dbt.Error("unexpected NullString value:" + ns.String + " (should be `1`)") + } + + // nil-bytes + var b []byte + // Read nil + if err = nullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil []byte which should be nil") + } + // Read non-nil + if err = nonNullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil []byte which should be non-nil") + } + // Insert nil + b = nil + success := false + if err = dbt.db.QueryRow("SELECT ? IS NULL", b).Scan(&success); err != nil { + dbt.Fatal(err) + } + if !success { + dbt.Error("inserting []byte(nil) as NULL failed") + } + // Check input==output with input==nil + b = nil + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil echo from nil input") + } + // Check input==output with input!=nil + b = []byte("") + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil echo from non-nil input") + } + + // Insert NULL + dbt.mustExec("CREATE TABLE test (dummmy1 int, value int, dummy2 int)") + + dbt.mustExec("INSERT INTO test VALUES (?, ?, ?)", 1, nil, 2) + + var out interface{} + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if out != nil { + dbt.Errorf("%v != nil", out) + } + } else { + dbt.Error("no data") + } + }) +} + +func TestUint64(t *testing.T) { + const ( + u0 = uint64(0) + uall = ^u0 + uhigh = uall >> 1 + utop = ^uhigh + s0 = int64(0) + sall = ^s0 + shigh = int64(uhigh) + stop = ^shigh + ) + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare(`SELECT ?, ?, ? ,?, ?, ?, ?, ?`) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + row := stmt.QueryRow( + u0, uhigh, utop, uall, + s0, shigh, stop, sall, + ) + + var ua, ub, uc, ud uint64 + var sa, sb, sc, sd int64 + + err = row.Scan(&ua, &ub, &uc, &ud, &sa, &sb, &sc, &sd) + if err != nil { + dbt.Fatal(err) + } + switch { + case ua != u0, + ub != uhigh, + uc != utop, + ud != uall, + sa != s0, + sb != shigh, + sc != stop, + sd != sall: + dbt.Fatal("unexpected result value") + } + }) +} + +func TestLongData(t *testing.T) { + runTests(t, dsn+"&maxAllowedPacket=0", func(dbt *DBTest) { + var maxAllowedPacketSize int + err := dbt.db.QueryRow("select @@max_allowed_packet").Scan(&maxAllowedPacketSize) + if err != nil { + dbt.Fatal(err) + } + maxAllowedPacketSize-- + + // don't get too ambitious + if maxAllowedPacketSize > 1<<25 { + maxAllowedPacketSize = 1 << 25 + } + + dbt.mustExec("CREATE TABLE test (value LONGBLOB)") + + in := strings.Repeat(`a`, maxAllowedPacketSize+1) + var out string + var rows *sql.Rows + + // Long text data + const nonDataQueryLen = 28 // length query w/o value + inS := in[:maxAllowedPacketSize-nonDataQueryLen] + dbt.mustExec("INSERT INTO test VALUES('" + inS + "')") + rows = dbt.mustQuery("SELECT value FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if inS != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(inS), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + dbt.Fatalf("LONGBLOB: no data") + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Long binary data + dbt.mustExec("INSERT INTO test VALUES(?)", in) + rows = dbt.mustQuery("SELECT value FROM test WHERE 1=?", 1) + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(in), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + if err = rows.Err(); err != nil { + dbt.Fatalf("LONGBLOB: no data (err: %s)", err.Error()) + } else { + dbt.Fatal("LONGBLOB: no data (err: )") + } + } + }) +} + +func TestLoadData(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + verifyLoadDataResult := func() { + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + dbt.Fatal(err.Error()) + } + + i := 0 + values := [4]string{ + "a string", + "a string containing a \t", + "a string containing a \n", + "a string containing both \t\n", + } + + var id int + var value string + + for rows.Next() { + i++ + err = rows.Scan(&id, &value) + if err != nil { + dbt.Fatal(err.Error()) + } + if i != id { + dbt.Fatalf("%d != %d", i, id) + } + if values[i-1] != value { + dbt.Fatalf("%q != %q", values[i-1], value) + } + } + err = rows.Err() + if err != nil { + dbt.Fatal(err.Error()) + } + + if i != 4 { + dbt.Fatalf("rows count mismatch. Got %d, want 4", i) + } + } + + dbt.db.Exec("DROP TABLE IF EXISTS test") + dbt.mustExec("CREATE TABLE test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") + + // Local File + file, err := ioutil.TempFile("", "gotest") + defer os.Remove(file.Name()) + if err != nil { + dbt.Fatal(err) + } + RegisterLocalFile(file.Name()) + + // Try first with empty file + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + var count int + err = dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&count) + if err != nil { + dbt.Fatal(err.Error()) + } + if count != 0 { + dbt.Fatalf("unexpected row count: got %d, want 0", count) + } + + // Then fille File with data and try to load it + file.WriteString("1\ta string\n2\ta string containing a \\t\n3\ta string containing a \\n\n4\ta string containing both \\t\\n\n") + file.Close() + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + verifyLoadDataResult() + + // Try with non-existing file + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent file didn't fail") + } else if err.Error() != "local file 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Reader + RegisterReaderHandler("test", func() io.Reader { + file, err = os.Open(file.Name()) + if err != nil { + dbt.Fatal(err) + } + return file + }) + dbt.mustExec("LOAD DATA LOCAL INFILE 'Reader::test' INTO TABLE test") + verifyLoadDataResult() + // negative test + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'Reader::doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent Reader didn't fail") + } else if err.Error() != "Reader 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + }) +} + +func TestFoundRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + }) + runTests(t, dsn+"&clientFoundRows=true", func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 matched rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 3 { + dbt.Fatalf("Expected 3 matched rows, got %d", count) + } + }) +} + +func TestTLS(t *testing.T) { + tlsTestReq := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + if err == ErrNoTLS { + dbt.Skip("server does not support TLS") + } else { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + rows := dbt.mustQuery("SHOW STATUS LIKE 'Ssl_cipher'") + defer rows.Close() + + var variable, value *sql.RawBytes + for rows.Next() { + if err := rows.Scan(&variable, &value); err != nil { + dbt.Fatal(err.Error()) + } + + if (*value == nil) || (len(*value) == 0) { + dbt.Fatalf("no Cipher") + } else { + dbt.Logf("Cipher: %s", *value) + } + } + } + tlsTestOpt := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + runTests(t, dsn+"&tls=preferred", tlsTestOpt) + runTests(t, dsn+"&tls=skip-verify", tlsTestReq) + + // Verify that registering / using a custom cfg works + RegisterTLSConfig("custom-skip-verify", &tls.Config{ + InsecureSkipVerify: true, + }) + runTests(t, dsn+"&tls=custom-skip-verify", tlsTestReq) +} + +func TestReuseClosedConnection(t *testing.T) { + // this test does not use sql.database, it uses the driver directly + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + md := &MySQLDriver{} + conn, err := md.Open(dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + stmt, err := conn.Prepare("DO 1") + if err != nil { + t.Fatalf("error preparing statement: %s", err.Error()) + } + _, err = stmt.Exec(nil) + if err != nil { + t.Fatalf("error executing statement: %s", err.Error()) + } + err = conn.Close() + if err != nil { + t.Fatalf("error closing connection: %s", err.Error()) + } + + defer func() { + if err := recover(); err != nil { + t.Errorf("panic after reusing a closed connection: %v", err) + } + }() + _, err = stmt.Exec(nil) + if err != nil && err != driver.ErrBadConn { + t.Errorf("unexpected error '%s', expected '%s'", + err.Error(), driver.ErrBadConn.Error()) + } +} + +func TestCharset(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mustSetCharset := func(charsetParam, expected string) { + runTests(t, dsn+"&"+charsetParam, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT @@character_set_connection") + defer rows.Close() + + if !rows.Next() { + dbt.Fatalf("error getting connection charset: %s", rows.Err()) + } + + var got string + rows.Scan(&got) + + if got != expected { + dbt.Fatalf("expected connection charset %s but got %s", expected, got) + } + }) + } + + // non utf8 test + mustSetCharset("charset=ascii", "ascii") + + // when the first charset is invalid, use the second + mustSetCharset("charset=none,utf8", "utf8") + + // when the first charset is valid, use it + mustSetCharset("charset=ascii,utf8", "ascii") + mustSetCharset("charset=utf8,ascii", "utf8") +} + +func TestFailingCharset(t *testing.T) { + runTests(t, dsn+"&charset=none", func(dbt *DBTest) { + // run query to really establish connection... + _, err := dbt.db.Exec("SELECT 1") + if err == nil { + dbt.db.Close() + t.Fatalf("connection must not succeed without a valid charset") + } + }) +} + +func TestCollation(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + defaultCollation := "utf8mb4_general_ci" + testCollations := []string{ + "", // do not set + defaultCollation, // driver default + "latin1_general_ci", + "binary", + "utf8_unicode_ci", + "cp1257_bin", + } + + for _, collation := range testCollations { + var expected, tdsn string + if collation != "" { + tdsn = dsn + "&collation=" + collation + expected = collation + } else { + tdsn = dsn + expected = defaultCollation + } + + runTests(t, tdsn, func(dbt *DBTest) { + var got string + if err := dbt.db.QueryRow("SELECT @@collation_connection").Scan(&got); err != nil { + dbt.Fatal(err) + } + + if got != expected { + dbt.Fatalf("expected connection collation %s but got %s", expected, got) + } + }) + } +} + +func TestColumnsWithAlias(t *testing.T) { + runTests(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1 AS A") + defer rows.Close() + cols, _ := rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A" { + t.Fatalf("expected column name \"A\", got \"%s\"", cols[0]) + } + + rows = dbt.mustQuery("SELECT * FROM (SELECT 1 AS one) AS A") + defer rows.Close() + cols, _ = rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A.one" { + t.Fatalf("expected column name \"A.one\", got \"%s\"", cols[0]) + } + }) +} + +func TestRawBytesResultExceedsBuffer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // defaultBufSize from buffer.go + expected := strings.Repeat("abc", defaultBufSize) + + rows := dbt.mustQuery("SELECT '" + expected + "'") + defer rows.Close() + if !rows.Next() { + dbt.Error("expected result, got none") + } + var result sql.RawBytes + rows.Scan(&result) + if expected != string(result) { + dbt.Error("result did not match expected value") + } + }) +} + +func TestTimezoneConversion(t *testing.T) { + zones := []string{"UTC", "US/Central", "US/Pacific", "Local"} + + // Regression test for timezone handling + tzTest := func(dbt *DBTest) { + // Create table + dbt.mustExec("CREATE TABLE test (ts TIMESTAMP)") + + // Insert local time into database (should be converted) + usCentral, _ := time.LoadLocation("US/Central") + reftime := time.Date(2014, 05, 30, 18, 03, 17, 0, time.UTC).In(usCentral) + dbt.mustExec("INSERT INTO test VALUE (?)", reftime) + + // Retrieve time from DB + rows := dbt.mustQuery("SELECT ts FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Fatal("did not get any rows out") + } + + var dbTime time.Time + err := rows.Scan(&dbTime) + if err != nil { + dbt.Fatal("Err", err) + } + + // Check that dates match + if reftime.Unix() != dbTime.Unix() { + dbt.Errorf("times do not match.\n") + dbt.Errorf(" Now(%v)=%v\n", usCentral, reftime) + dbt.Errorf(" Now(UTC)=%v\n", dbTime) + } + } + + for _, tz := range zones { + runTests(t, dsn+"&parseTime=true&loc="+url.QueryEscape(tz), tzTest) + } +} + +// Special cases + +func TestRowsClose(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows, err := dbt.db.Query("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + err = rows.Close() + if err != nil { + dbt.Fatal(err) + } + + if rows.Next() { + dbt.Fatal("unexpected row after rows.Close()") + } + + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + }) +} + +// dangling statements +// http://code.google.com/p/go/issues/detail?id=3865 +func TestCloseStmtBeforeRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + rows, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows.Close() + + err = stmt.Close() + if err != nil { + dbt.Fatal(err) + } + + if !rows.Next() { + dbt.Fatal("getting row failed") + } else { + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + + var out bool + err = rows.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + }) +} + +// It is valid to have multiple Rows for the same Stmt +// http://code.google.com/p/go/issues/detail?id=3734 +func TestStmtMultiRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1 UNION SELECT 0") + if err != nil { + dbt.Fatal(err) + } + + rows1, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows1.Close() + + rows2, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows2.Close() + + var out bool + + // 1 + if !rows1.Next() { + dbt.Fatal("first rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + if !rows2.Next() { + dbt.Fatal("first rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + // 2 + if !rows1.Next() { + dbt.Fatal("second rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows1.Next() { + dbt.Fatal("unexpected row on rows1") + } + err = rows1.Close() + if err != nil { + dbt.Fatal(err) + } + } + + if !rows2.Next() { + dbt.Fatal("second rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows2.Next() { + dbt.Fatal("unexpected row on rows2") + } + err = rows2.Close() + if err != nil { + dbt.Fatal(err) + } + } + }) +} + +// Regression test for +// * more than 32 NULL parameters (issue 209) +// * more parameters than fit into the buffer (issue 201) +// * parameters * 64 > max_allowed_packet (issue 734) +func TestPreparedManyCols(t *testing.T) { + numParams := 65535 + runTests(t, dsn, func(dbt *DBTest) { + query := "SELECT ?" + strings.Repeat(",?", numParams-1) + stmt, err := dbt.db.Prepare(query) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + + // create more parameters than fit into the buffer + // which will take nil-values + params := make([]interface{}, numParams) + rows, err := stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + + // Create 0byte string which we can't send via STMT_LONG_DATA. + for i := 0; i < numParams; i++ { + params[i] = "" + } + rows, err = stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + }) +} + +func TestConcurrent(t *testing.T) { + if enabled, _ := readBool(os.Getenv("MYSQL_TEST_CONCURRENT")); !enabled { + t.Skip("MYSQL_TEST_CONCURRENT env var not set") + } + + runTests(t, dsn, func(dbt *DBTest) { + var max int + err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + dbt.Logf("testing up to %d concurrent connections \r\n", max) + + var remaining, succeeded int32 = int32(max), 0 + + var wg sync.WaitGroup + wg.Add(max) + + var fatalError string + var once sync.Once + fatalf := func(s string, vals ...interface{}) { + once.Do(func() { + fatalError = fmt.Sprintf(s, vals...) + }) + } + + for i := 0; i < max; i++ { + go func(id int) { + defer wg.Done() + + tx, err := dbt.db.Begin() + atomic.AddInt32(&remaining, -1) + + if err != nil { + if err.Error() != "Error 1040: Too many connections" { + fatalf("error on conn %d: %s", id, err.Error()) + } + return + } + + // keep the connection busy until all connections are open + for remaining > 0 { + if _, err = tx.Exec("DO 1"); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + } + + if err = tx.Commit(); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + + // everything went fine with this connection + atomic.AddInt32(&succeeded, 1) + }(i) + } + + // wait until all conections are open + wg.Wait() + + if fatalError != "" { + dbt.Fatal(fatalError) + } + + dbt.Logf("reached %d concurrent connections\r\n", succeeded) + }) +} + +func testDialError(t *testing.T, dialErr error, expectErr error) { + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + return nil, dialErr + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + _, err = db.Exec("DO 1") + if err != expectErr { + t.Fatalf("was expecting %s. Got: %s", dialErr, err) + } +} + +func TestDialUnknownError(t *testing.T) { + testErr := fmt.Errorf("test") + testDialError(t, testErr, testErr) +} + +func TestDialNonRetryableNetErr(t *testing.T) { + testErr := netErrorMock{} + testDialError(t, testErr, testErr) +} + +func TestDialTemporaryNetErr(t *testing.T) { + testErr := netErrorMock{temporary: true} + testDialError(t, testErr, driver.ErrBadConn) +} + +// Tests custom dial functions +func TestCustomDial(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + // our custom dial function which justs wraps net.Dial here + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + if _, err = db.Exec("DO 1"); err != nil { + t.Fatalf("connection failed: %s", err.Error()) + } +} + +func TestSQLInjection(t *testing.T) { + createTest := func(arg string) func(dbt *DBTest) { + return func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (?)", 1) + + var v int + // NULL can't be equal to anything, the idea here is to inject query so it returns row + // This test verifies that escapeQuotes and escapeBackslash are working properly + err := dbt.db.QueryRow("SELECT v FROM test WHERE NULL = ?", arg).Scan(&v) + if err == sql.ErrNoRows { + return // success, sql injection failed + } else if err == nil { + dbt.Errorf("sql injection successful with arg: %s", arg) + } else { + dbt.Errorf("error running query with arg: %s; err: %s", arg, err.Error()) + } + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, createTest("1 OR 1=1")) + runTests(t, testdsn, createTest("' OR '1'='1")) + } +} + +// Test if inserted data is correctly retrieved after being escaped +func TestInsertRetrieveEscapedData(t *testing.T) { + testData := func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v VARCHAR(255))") + + // All sequences that are escaped by escapeQuotes and escapeBackslash + v := "foo \x00\n\r\x1a\"'\\" + dbt.mustExec("INSERT INTO test VALUES (?)", v) + + var out string + err := dbt.db.QueryRow("SELECT v FROM test").Scan(&out) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + if out != v { + dbt.Errorf("%q != %q", out, v) + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, testData) + } +} + +func TestUnixSocketAuthFail(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Save the current logger so we can restore it. + oldLogger := errLog + + // Set a new logger so we can capture its output. + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + newLogger := log.New(buffer, "prefix: ", 0) + SetLogger(newLogger) + + // Restore the logger. + defer SetLogger(oldLogger) + + // Make a new DSN that uses the MySQL socket file and a bad password, which + // we can make by simply appending any character to the real password. + badPass := pass + "x" + socket := "" + if prot == "unix" { + socket = addr + } else { + // Get socket file from MySQL. + err := dbt.db.QueryRow("SELECT @@socket").Scan(&socket) + if err != nil { + t.Fatalf("error on SELECT @@socket: %s", err.Error()) + } + } + t.Logf("socket: %s", socket) + badDSN := fmt.Sprintf("%s:%s@unix(%s)/%s?timeout=30s", user, badPass, socket, dbname) + db, err := sql.Open("mysql", badDSN) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + // Connect to MySQL for real. This will cause an auth failure. + err = db.Ping() + if err == nil { + t.Error("expected Ping() to return an error") + } + + // The driver should not log anything. + if actual := buffer.String(); actual != "" { + t.Errorf("expected no output, got %q", actual) + } + }) +} + +// See Issue #422 +func TestInterruptBySignal(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + DROP PROCEDURE IF EXISTS test_signal; + CREATE PROCEDURE test_signal(ret INT) + BEGIN + SELECT ret; + SIGNAL SQLSTATE + '45001' + SET + MESSAGE_TEXT = "an error", + MYSQL_ERRNO = 45001; + END + `) + defer dbt.mustExec("DROP PROCEDURE test_signal") + + var val int + + // text protocol + rows, err := dbt.db.Query("CALL test_signal(42)") + if err != nil { + dbt.Fatalf("error on text query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + + // binary protocol + rows, err = dbt.db.Query("CALL test_signal(?)", 42) + if err != nil { + dbt.Fatalf("error on binary query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + }) +} + +func TestColumnsReusesSlice(t *testing.T) { + rows := mysqlRows{ + rs: resultSet{ + columns: []mysqlField{ + { + tableName: "test", + name: "A", + }, + { + tableName: "test", + name: "B", + }, + }, + }, + } + + allocs := testing.AllocsPerRun(1, func() { + cols := rows.Columns() + + if len(cols) != 2 { + t.Fatalf("expected 2 columns, got %d", len(cols)) + } + }) + + if allocs != 0 { + t.Fatalf("expected 0 allocations, got %d", int(allocs)) + } + + if rows.rs.columnNames == nil { + t.Fatalf("expected columnNames to be set, got nil") + } +} + +func TestRejectReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read-only. We didn't set the `rejectReadOnly` + // option, so any writes after this should fail. + _, err := dbt.db.Exec("SET SESSION TRANSACTION READ ONLY") + // Error 1193: Unknown system variable 'TRANSACTION' => skip test, + // MySQL server version is too old + maybeSkip(t, err, 1193) + if _, err := dbt.db.Exec("DROP TABLE test"); err == nil { + t.Fatalf("writing to DB in read-only session without " + + "rejectReadOnly did not error") + } + // Set the session back to read-write so runTests() can properly clean + // up the table `test`. + dbt.mustExec("SET SESSION TRANSACTION READ WRITE") + }) + + // Enable the `rejectReadOnly` option. + runTests(t, dsn+"&rejectReadOnly=true", func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read only. Any writes after this should error on + // a driver.ErrBadConn, and cause `database/sql` to initiate a new + // connection. + dbt.mustExec("SET SESSION TRANSACTION READ ONLY") + // This would error, but `database/sql` should automatically retry on a + // new connection which is not read-only, and eventually succeed. + dbt.mustExec("DROP TABLE test") + }) +} + +func TestPing(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.fail("Ping", "Ping", err) + } + }) +} + +// See Issue #799 +func TestEmptyPassword(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn := fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, "", netAddr, dbname) + db, err := sql.Open("mysql", dsn) + if err == nil { + defer db.Close() + err = db.Ping() + } + + if pass == "" { + if err != nil { + t.Fatal(err.Error()) + } + } else { + if err == nil { + t.Fatal("expected authentication error") + } + if !strings.HasPrefix(err.Error(), "Error 1045") { + t.Fatal(err.Error()) + } + } +} + +// static interface implementation checks of mysqlConn +var ( + _ driver.ConnBeginTx = &mysqlConn{} + _ driver.ConnPrepareContext = &mysqlConn{} + _ driver.ExecerContext = &mysqlConn{} + _ driver.Pinger = &mysqlConn{} + _ driver.QueryerContext = &mysqlConn{} +) + +// static interface implementation checks of mysqlStmt +var ( + _ driver.StmtExecContext = &mysqlStmt{} + _ driver.StmtQueryContext = &mysqlStmt{} +) + +// Ensure that all the driver interfaces are implemented +var ( + // _ driver.RowsColumnTypeLength = &binaryRows{} + // _ driver.RowsColumnTypeLength = &textRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &binaryRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &textRows{} + _ driver.RowsColumnTypeNullable = &binaryRows{} + _ driver.RowsColumnTypeNullable = &textRows{} + _ driver.RowsColumnTypePrecisionScale = &binaryRows{} + _ driver.RowsColumnTypePrecisionScale = &textRows{} + _ driver.RowsColumnTypeScanType = &binaryRows{} + _ driver.RowsColumnTypeScanType = &textRows{} + _ driver.RowsNextResultSet = &binaryRows{} + _ driver.RowsNextResultSet = &textRows{} +) + +func TestMultiResultSet(t *testing.T) { + type result struct { + values [][]int + columns []string + } + + // checkRows is a helper test function to validate rows containing 3 result + // sets with specific values and columns. The basic query would look like this: + // + // SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + // SELECT 0 UNION SELECT 1; + // SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + // + // to distinguish test cases the first string argument is put in front of + // every error or fatal message. + checkRows := func(desc string, rows *sql.Rows, dbt *DBTest) { + expected := []result{ + { + values: [][]int{{1, 2}, {3, 4}}, + columns: []string{"col1", "col2"}, + }, + { + values: [][]int{{1, 2, 3}, {4, 5, 6}}, + columns: []string{"col1", "col2", "col3"}, + }, + } + + var res1 result + for rows.Next() { + var res [2]int + if err := rows.Scan(&res[0], &res[1]); err != nil { + dbt.Fatal(err) + } + res1.values = append(res1.values, res[:]) + } + + cols, err := rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res1.columns = cols + + if !reflect.DeepEqual(expected[0], res1) { + dbt.Error(desc, "want =", expected[0], "got =", res1) + } + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + // ignoring one result set + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + var res2 result + cols, err = rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res2.columns = cols + + for rows.Next() { + var res [3]int + if err := rows.Scan(&res[0], &res[1], &res[2]); err != nil { + dbt.Fatal(desc, err) + } + res2.values = append(res2.values, res[:]) + } + + if !reflect.DeepEqual(expected[1], res2) { + dbt.Error(desc, "want =", expected[1], "got =", res2) + } + + if rows.NextResultSet() { + dbt.Error(desc, "unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error(desc, err) + } + } + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery(`DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6;`) + defer rows.Close() + checkRows("query: ", rows, dbt) + }) + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + queries := []string{ + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + } + + defer dbt.mustExec("DROP PROCEDURE IF EXISTS test_mrss") + + for i, query := range queries { + dbt.mustExec(query) + + stmt, err := dbt.db.Prepare("CALL test_mrss()") + if err != nil { + dbt.Fatalf("%v (i=%d)", err, i) + } + defer stmt.Close() + + for j := 0; j < 2; j++ { + rows, err := stmt.Query() + if err != nil { + dbt.Fatalf("%v (i=%d) (j=%d)", err, i, j) + } + checkRows(fmt.Sprintf("prepared stmt query (i=%d) (j=%d): ", i, j), rows, dbt) + } + } + }) +} + +func TestMultiResultSetNoSelect(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("DO 1; DO 2;") + defer rows.Close() + + if rows.Next() { + dbt.Error("unexpected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +// tests if rows are set in a proper state if some results were ignored before +// calling rows.NextResultSet. +func TestSkipResults(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1, 2") + defer rows.Close() + + if !rows.Next() { + dbt.Error("expected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +func TestPingContext(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := dbt.db.PingContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (1)"); err == nil { + dbt.Error("expected error") + } else if err.Error() != "context canceled" { + dbt.Fatalf("unexpected error: %s", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (1)"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQueryRow(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (1), (2), (3)") + ctx, cancel := context.WithCancel(context.Background()) + + rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM test") + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + // the first row will be succeed. + var v int + if !rows.Next() { + dbt.Fatalf("unexpected end") + } + if err := rows.Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + + cancel() + // make sure the driver receives the cancel request. + time.Sleep(100 * time.Millisecond) + + if rows.Next() { + dbt.Errorf("expected end, but not") + } + if err := rows.Err(); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelPrepare(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := dbt.db.PrepareContext(ctx, "SELECT 1"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelStmtExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.ExecContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelStmtQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.QueryContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query has done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelBegin(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + tx, err := dbt.db.BeginTx(ctx, nil) + if err != nil { + dbt.Fatal(err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(100*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := tx.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Transaction is canceled, so expect an error. + switch err := tx.Commit(); err { + case sql.ErrTxDone: + // because the transaction has already been rollbacked. + // the database/sql package watches ctx + // and rollbacks when ctx is canceled. + case context.Canceled: + // the database/sql package rollbacks on another goroutine, + // so the transaction may not be rollbacked depending on goroutine scheduling. + default: + dbt.Errorf("expected sql.ErrTxDone or context.Canceled, got %v", err) + } + + // Context is canceled, so cannot begin a transaction. + if _, err := dbt.db.BeginTx(ctx, nil); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextBeginIsolationLevel(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx1, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelRepeatableRead, + }) + if err != nil { + dbt.Fatal(err) + } + + tx2, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelReadCommitted, + }) + if err != nil { + dbt.Fatal(err) + } + + _, err = tx1.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if err != nil { + dbt.Fatal(err) + } + + var v int + row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Because writer transaction wasn't commited yet, it should be available + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + err = tx1.Commit() + if err != nil { + dbt.Fatal(err) + } + + row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Data written by writer transaction is already commited, it should be selectable + if v != 1 { + dbt.Errorf("expected val to be 1, got %d", v) + } + tx2.Commit() + }) +} + +func TestContextBeginReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + ReadOnly: true, + }) + if _, ok := err.(*MySQLError); ok { + dbt.Skip("It seems that your MySQL does not support READ ONLY transactions") + return + } else if err != nil { + dbt.Fatal(err) + } + + // INSERT queries fail in a READ ONLY transaction. + _, err = tx.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if _, ok := err.(*MySQLError); !ok { + dbt.Errorf("expected MySQLError, got %v", err) + } + + // SELECT queries can be executed. + var v int + row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + if err := tx.Commit(); err != nil { + dbt.Fatal(err) + } + }) +} + +func TestRowsColumnTypes(t *testing.T) { + niNULL := sql.NullInt64{Int64: 0, Valid: false} + ni0 := sql.NullInt64{Int64: 0, Valid: true} + ni1 := sql.NullInt64{Int64: 1, Valid: true} + ni42 := sql.NullInt64{Int64: 42, Valid: true} + nfNULL := sql.NullFloat64{Float64: 0.0, Valid: false} + nf0 := sql.NullFloat64{Float64: 0.0, Valid: true} + nf1337 := sql.NullFloat64{Float64: 13.37, Valid: true} + nt0 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} + nt1 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} + nt2 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} + nt6 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} + nd1 := NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} + nd2 := NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} + ndNULL := NullTime{Time: time.Time{}, Valid: false} + rbNULL := sql.RawBytes(nil) + rb0 := sql.RawBytes("0") + rb42 := sql.RawBytes("42") + rbTest := sql.RawBytes("Test") + rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 + rbx0 := sql.RawBytes("\x00") + rbx42 := sql.RawBytes("\x42") + + var columns = []struct { + name string + fieldType string // type used when creating table schema + databaseTypeName string // actual type used by MySQL + scanType reflect.Type + nullable bool + precision int64 // 0 if not ok + scale int64 + valuesIn [3]string + valuesOut [3]interface{} + }{ + {"bit8null", "BIT(8)", "BIT", scanTypeRawBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]interface{}{rbx0, rbNULL, rbx42}}, + {"boolnull", "BOOL", "TINYINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "true", "0"}, [3]interface{}{niNULL, ni1, ni0}}, + {"bool", "BOOL NOT NULL", "TINYINT", scanTypeInt8, false, 0, 0, [3]string{"1", "0", "FALSE"}, [3]interface{}{int8(1), int8(0), int8(0)}}, + {"intnull", "INTEGER", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"smallint", "SMALLINT NOT NULL", "SMALLINT", scanTypeInt16, false, 0, 0, [3]string{"0", "-32768", "32767"}, [3]interface{}{int16(0), int16(-32768), int16(32767)}}, + {"smallintnull", "SMALLINT", "SMALLINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int3null", "INT(3)", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int7", "INT(7) NOT NULL", "INT", scanTypeInt32, false, 0, 0, [3]string{"0", "-1337", "42"}, [3]interface{}{int32(0), int32(-1337), int32(42)}}, + {"mediumintnull", "MEDIUMINT", "MEDIUMINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "42", "NULL"}, [3]interface{}{ni0, ni42, niNULL}}, + {"bigint", "BIGINT NOT NULL", "BIGINT", scanTypeInt64, false, 0, 0, [3]string{"0", "65535", "-42"}, [3]interface{}{int64(0), int64(65535), int64(-42)}}, + {"bigintnull", "BIGINT", "BIGINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "1", "42"}, [3]interface{}{niNULL, ni1, ni42}}, + {"tinyuint", "TINYINT UNSIGNED NOT NULL", "TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, + {"smalluint", "SMALLINT UNSIGNED NOT NULL", "SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, + {"biguint", "BIGINT UNSIGNED NOT NULL", "BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, + {"uint13", "INT(13) UNSIGNED NOT NULL", "INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, + {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float32(0), float32(42), float32(13.37)}}, + {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"double", "DOUBLE NOT NULL", "DOUBLE", scanTypeFloat64, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float64(0), float64(42), float64(13.37)}}, + {"doublenull", "DOUBLE", "DOUBLE", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), sql.RawBytes("13.370000"), sql.RawBytes("1234.123456")}}, + {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeRawBytes, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), rbNULL, sql.RawBytes("1234.123456")}}, + {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), sql.RawBytes("13.3700"), sql.RawBytes("1234.1235")}}, + {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, + {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, + {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, + {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, + {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, + {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, + {"date", "DATE", "DATE", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02'", "NULL", "'2006-03-04'"}, [3]interface{}{nd1, ndNULL, nd2}}, + {"year", "YEAR NOT NULL", "YEAR", scanTypeUint16, false, 0, 0, [3]string{"2006", "2000", "1994"}, [3]interface{}{uint16(2006), uint16(2000), uint16(1994)}}, + } + + schema := "" + values1 := "" + values2 := "" + values3 := "" + for _, column := range columns { + schema += fmt.Sprintf("`%s` %s, ", column.name, column.fieldType) + values1 += column.valuesIn[0] + ", " + values2 += column.valuesIn[1] + ", " + values3 += column.valuesIn[2] + ", " + } + schema = schema[:len(schema)-2] + values1 = values1[:len(values1)-2] + values2 = values2[:len(values2)-2] + values3 = values3[:len(values3)-2] + + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (" + schema + ")") + dbt.mustExec("INSERT INTO test VALUES (" + values1 + "), (" + values2 + "), (" + values3 + ")") + + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + t.Fatalf("Query: %v", err) + } + + tt, err := rows.ColumnTypes() + if err != nil { + t.Fatalf("ColumnTypes: %v", err) + } + + if len(tt) != len(columns) { + t.Fatalf("unexpected number of columns: expected %d, got %d", len(columns), len(tt)) + } + + types := make([]reflect.Type, len(tt)) + for i, tp := range tt { + column := columns[i] + + // Name + name := tp.Name() + if name != column.name { + t.Errorf("column name mismatch %s != %s", name, column.name) + continue + } + + // DatabaseTypeName + databaseTypeName := tp.DatabaseTypeName() + if databaseTypeName != column.databaseTypeName { + t.Errorf("databasetypename name mismatch for column %q: %s != %s", name, databaseTypeName, column.databaseTypeName) + continue + } + + // ScanType + scanType := tp.ScanType() + if scanType != column.scanType { + if scanType == nil { + t.Errorf("scantype is null for column %q", name) + } else { + t.Errorf("scantype mismatch for column %q: %s != %s", name, scanType.Name(), column.scanType.Name()) + } + continue + } + types[i] = scanType + + // Nullable + nullable, ok := tp.Nullable() + if !ok { + t.Errorf("nullable not ok %q", name) + continue + } + if nullable != column.nullable { + t.Errorf("nullable mismatch for column %q: %t != %t", name, nullable, column.nullable) + } + + // Length + // length, ok := tp.Length() + // if length != column.length { + // if !ok { + // t.Errorf("length not ok for column %q", name) + // } else { + // t.Errorf("length mismatch for column %q: %d != %d", name, length, column.length) + // } + // continue + // } + + // Precision and Scale + precision, scale, ok := tp.DecimalSize() + if precision != column.precision { + if !ok { + t.Errorf("precision not ok for column %q", name) + } else { + t.Errorf("precision mismatch for column %q: %d != %d", name, precision, column.precision) + } + continue + } + if scale != column.scale { + if !ok { + t.Errorf("scale not ok for column %q", name) + } else { + t.Errorf("scale mismatch for column %q: %d != %d", name, scale, column.scale) + } + continue + } + } + + values := make([]interface{}, len(tt)) + for i := range values { + values[i] = reflect.New(types[i]).Interface() + } + i := 0 + for rows.Next() { + err = rows.Scan(values...) + if err != nil { + t.Fatalf("failed to scan values in %v", err) + } + for j := range values { + value := reflect.ValueOf(values[j]).Elem().Interface() + if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { + if columns[j].scanType == scanTypeRawBytes { + t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) + } else { + t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) + } + } + } + i++ + } + if i != 3 { + t.Errorf("expected 3 rows, got %d", i) + } + + if err := rows.Close(); err != nil { + t.Errorf("error closing rows: %s", err) + } + }) + } +} + +func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (value VARCHAR(255))") + dbt.db.Exec("INSERT INTO test VALUES (?)", (*testValuer)(nil)) + // This test will panic on the INSERT if ConvertValue() does not check for typed nil before calling Value() + }) +} + +// TestRawBytesAreNotModified checks for a race condition that arises when a query context +// is canceled while a user is calling rows.Scan. This is a more stringent test than the one +// proposed in https://github.com/golang/go/issues/23519. Here we're explicitly using +// `sql.RawBytes` to check the contents of our internal buffers are not modified after an implicit +// call to `Rows.Close`, so Context cancellation should **not** invalidate the backing buffers. +func TestRawBytesAreNotModified(t *testing.T) { + const blob = "abcdefghijklmnop" + const contextRaceIterations = 20 + const blobSize = defaultBufSize * 3 / 4 // Second row overwrites first row. + const insertRows = 4 + + var sqlBlobs = [2]string{ + strings.Repeat(blob, blobSize/len(blob)), + strings.Repeat(strings.ToUpper(blob), blobSize/len(blob)), + } + + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + for i := 0; i < insertRows; i++ { + dbt.mustExec("INSERT INTO test VALUES (?, ?)", i+1, sqlBlobs[i&1]) + } + + for i := 0; i < contextRaceIterations; i++ { + func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rows, err := dbt.db.QueryContext(ctx, `SELECT id, value FROM test`) + if err != nil { + t.Fatal(err) + } + + var b int + var raw sql.RawBytes + for rows.Next() { + if err := rows.Scan(&b, &raw); err != nil { + t.Fatal(err) + } + + before := string(raw) + // Ensure cancelling the query does not corrupt the contents of `raw` + cancel() + time.Sleep(time.Microsecond * 100) + after := string(raw) + + if before != after { + t.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) + } + } + rows.Close() + }() + } + }) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go index be014ba..1d9b4ab 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -14,6 +14,7 @@ import ( "crypto/tls" "errors" "fmt" + "math/big" "net" "net/url" "sort" @@ -72,6 +73,26 @@ func NewConfig() *Config { } } +func (cfg *Config) Clone() *Config { + cp := *cfg + if cp.tls != nil { + cp.tls = cfg.tls.Clone() + } + if len(cp.Params) > 0 { + cp.Params = make(map[string]string, len(cfg.Params)) + for k, v := range cfg.Params { + cp.Params[k] = v + } + } + if cfg.pubKey != nil { + cp.pubKey = &rsa.PublicKey{ + N: new(big.Int).Set(cfg.pubKey.N), + E: cfg.pubKey.E, + } + } + return &cp +} + func (cfg *Config) normalize() error { if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { return errInvalidDSNUnsafeCollation @@ -92,17 +113,35 @@ func (cfg *Config) normalize() error { default: return errors.New("default addr for network '" + cfg.Net + "' unknown") } - } else if cfg.Net == "tcp" { cfg.Addr = ensureHavePort(cfg.Addr) } - if cfg.tls != nil { - if cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.Addr) - if err == nil { - cfg.tls.ServerName = host - } + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.tls = &tls.Config{} + case "skip-verify", "preferred": + cfg.tls = &tls.Config{InsecureSkipVerify: true} + default: + cfg.tls = getTLSConfigClone(cfg.TLSConfig) + if cfg.tls == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } + } + + if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + + if cfg.ServerPubKey != "" { + cfg.pubKey = getServerPubKey(cfg.ServerPubKey) + if cfg.pubKey == nil { + return errors.New("invalid value / unknown server pub key name: " + cfg.ServerPubKey) } } @@ -531,13 +570,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { if err != nil { return fmt.Errorf("invalid value for server pub key name: %v", err) } - - if pubKey := getServerPubKey(name); pubKey != nil { - cfg.ServerPubKey = name - cfg.pubKey = pubKey - } else { - return errors.New("invalid value / unknown server pub key name: " + name) - } + cfg.ServerPubKey = name // Strict mode case "strict": @@ -556,25 +589,17 @@ func parseDSNParams(cfg *Config, params string) (err error) { if isBool { if boolValue { cfg.TLSConfig = "true" - cfg.tls = &tls.Config{} } else { cfg.TLSConfig = "false" } - } else if vl := strings.ToLower(value); vl == "skip-verify" { + } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { cfg.TLSConfig = vl - cfg.tls = &tls.Config{InsecureSkipVerify: true} } else { name, err := url.QueryUnescape(value) if err != nil { return fmt.Errorf("invalid value for TLS config name: %v", err) } - - if tlsConfig := getTLSConfigClone(name); tlsConfig != nil { - cfg.TLSConfig = name - cfg.tls = tlsConfig - } else { - return errors.New("invalid value / unknown config name: " + name) - } + cfg.TLSConfig = name } // I/O write Timeout diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go new file mode 100644 index 0000000..50dc293 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go @@ -0,0 +1,415 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/tls" + "fmt" + "net/url" + "reflect" + "testing" + "time" +) + +var testDSNs = []struct { + in string + out *Config +}{{ + "username:password@protocol(address)/dbname?param=value", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, +}, { + "user@unix(/path/to/socket)/dbname?charset=utf8", + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, +}, { + "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, +}, { + "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, +}, { + "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "@/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:p@/ssword@/", + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "unix/?arg=%2Fsome%2Fpath.ext", + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(127.0.0.1)/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(de:ad:be:ef::ca:fe)/dbname", + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, +} + +func TestDSNParser(t *testing.T) { + for i, tst := range testDSNs { + cfg, err := ParseDSN(tst.in) + if err != nil { + t.Error(err.Error()) + } + + // pointer not static + cfg.tls = nil + + if !reflect.DeepEqual(cfg, tst.out) { + t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) + } + } +} + +func TestDSNParserInvalid(t *testing.T) { + var invalidDSNs = []string{ + "@net(addr/", // no closing brace + "@tcp(/", // no closing brace + "tcp(/", // no closing brace + "(/", // no closing brace + "net(addr)//", // unescaped + "User:pass@tcp(1.2.3.4:3306)", // no trailing slash + "net()/", // unknown default addr + //"/dbname?arg=/some/unescaped/path", + } + + for i, tst := range invalidDSNs { + if _, err := ParseDSN(tst); err == nil { + t.Errorf("invalid DSN #%d. (%s) didn't error!", i, tst) + } + } +} + +func TestDSNReformat(t *testing.T) { + for i, tst := range testDSNs { + dsn1 := tst.in + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Error(err.Error()) + continue + } + cfg1.tls = nil // pointer not static + res1 := fmt.Sprintf("%+v", cfg1) + + dsn2 := cfg1.FormatDSN() + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Error(err.Error()) + continue + } + cfg2.tls = nil // pointer not static + res2 := fmt.Sprintf("%+v", cfg2) + + if res1 != res2 { + t.Errorf("%d. %q does not match %q", i, res2, res1) + } + } +} + +func TestDSNServerPubKey(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + tst := baseDSN + "testKey" + cfg, err := ParseDSN(tst) + if err != nil { + t.Error(err.Error()) + } + + if cfg.ServerPubKey != "testKey" { + t.Errorf("unexpected cfg.ServerPubKey value: %v", cfg.ServerPubKey) + } + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } + + // Key is missing + tst = baseDSN + "invalid_name" + cfg, err = ParseDSN(tst) + if err == nil { + t.Errorf("invalid name in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } +} + +func TestDSNServerPubKeyQueryEscape(t *testing.T) { + const name = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + url.QueryEscape(name) + + RegisterServerPubKey(name, testPubKeyRSA) + defer DeregisterServerPubKey(name) + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } +} + +func TestDSNWithCustomTLS(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?tls=" + tlsCfg := tls.Config{} + + RegisterTLSConfig("utils_test", &tlsCfg) + defer DeregisterTLSConfig("utils_test") + + // Custom TLS is missing + tst := baseDSN + "invalid_tls" + cfg, err := ParseDSN(tst) + if err == nil { + t.Errorf("invalid custom TLS in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } + + tst = baseDSN + "utils_test" + + // Custom TLS with a server name + name := "foohost" + tlsCfg.ServerName = name + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, tst) + } + + // Custom TLS without a server name + name = "localhost" + tlsCfg.ServerName = "" + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct ServerName (%s) parsing DSN (%s).", name, tst) + } else if tlsCfg.ServerName != "" { + t.Errorf("tlsCfg was mutated ServerName (%s) should be empty parsing DSN (%s).", name, tst) + } +} + +func TestDSNTLSConfig(t *testing.T) { + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true" + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + dsn = "tcp(example.com)/?tls=true" + cfg, err = ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host without port)", expectedServerName, cfg.tls.ServerName) + } +} + +func TestDSNWithCustomTLSQueryEscape(t *testing.T) { + const configKey = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?tls=" + url.QueryEscape(configKey) + name := "foohost" + tlsCfg := tls.Config{ServerName: name} + + RegisterTLSConfig(configKey, &tlsCfg) + defer DeregisterTLSConfig(configKey) + + cfg, err := ParseDSN(dsn) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, dsn) + } +} + +func TestDSNUnsafeCollation(t *testing.T) { + _, err := ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=true") + if err != errInvalidDSNUnsafeCollation { + t.Errorf("expected %v, got %v", errInvalidDSNUnsafeCollation, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=false") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=ascii_bin&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=latin1_german1_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8mb4_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } +} + +func TestParamsAreSorted(t *testing.T) { + expected := "/dbname?interpolateParams=true&foobar=baz&quux=loo" + cfg := NewConfig() + cfg.DBName = "dbname" + cfg.InterpolateParams = true + cfg.Params = map[string]string{ + "quux": "loo", + "foobar": "baz", + } + actual := cfg.FormatDSN() + if actual != expected { + t.Errorf("generic Config.Params were not sorted: want %#v, got %#v", expected, actual) + } +} + +func TestCloneConfig(t *testing.T) { + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true&foobar=baz&serverPubKey=testKey" + cfg, err := ParseDSN(dsn) + if err != nil { + t.Fatal(err.Error()) + } + + cfg2 := cfg.Clone() + if cfg == cfg2 { + t.Errorf("Config.Clone did not create a separate config struct") + } + + if cfg2.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + cfg2.tls.ServerName = "example2.com" + if cfg.tls.ServerName == cfg2.tls.ServerName { + t.Errorf("changed cfg.tls.Server name should not propagate to original Config") + } + + if _, ok := cfg2.Params["foobar"]; !ok { + t.Errorf("cloned Config is missing custom params") + } + + delete(cfg2.Params, "foobar") + + if _, ok := cfg.Params["foobar"]; !ok { + t.Errorf("custom params in cloned Config should not propagate to original Config") + } + + if !reflect.DeepEqual(cfg.pubKey, cfg2.pubKey) { + t.Errorf("public key in Config should be identical") + } +} + +func TestNormalizeTLSConfig(t *testing.T) { + tt := []struct { + tlsConfig string + want *tls.Config + }{ + {"", nil}, + {"false", nil}, + {"true", &tls.Config{ServerName: "myserver"}}, + {"skip-verify", &tls.Config{InsecureSkipVerify: true}}, + {"preferred", &tls.Config{InsecureSkipVerify: true}}, + {"test_tls_config", &tls.Config{ServerName: "myServerName"}}, + } + + RegisterTLSConfig("test_tls_config", &tls.Config{ServerName: "myServerName"}) + defer func() { DeregisterTLSConfig("test_tls_config") }() + + for _, tc := range tt { + t.Run(tc.tlsConfig, func(t *testing.T) { + cfg := &Config{ + Addr: "myserver:3306", + TLSConfig: tc.tlsConfig, + } + + cfg.normalize() + + if cfg.tls == nil { + if tc.want != nil { + t.Fatal("wanted a tls config but got nil instead") + } + return + } + + if cfg.tls.ServerName != tc.want.ServerName { + t.Errorf("tls.ServerName doesn't match (want: '%s', got: '%s')", + tc.want.ServerName, cfg.tls.ServerName) + } + if cfg.tls.InsecureSkipVerify != tc.want.InsecureSkipVerify { + t.Errorf("tls.InsecureSkipVerify doesn't match (want: %T, got :%T)", + tc.want.InsecureSkipVerify, cfg.tls.InsecureSkipVerify) + } + }) + } +} + +func BenchmarkParseDSN(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + for _, tst := range testDSNs { + if _, err := ParseDSN(tst.in); err != nil { + b.Error(err.Error()) + } + } + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go new file mode 100644 index 0000000..96f9126 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go @@ -0,0 +1,42 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "log" + "testing" +) + +func TestErrorsSetLogger(t *testing.T) { + previous := errLog + defer func() { + errLog = previous + }() + + // set up logger + const expected = "prefix: test\n" + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + logger := log.New(buffer, "prefix: ", 0) + + // print + SetLogger(logger) + errLog.Print("test") + + // check result + if actual := buffer.String(); actual != expected { + t.Errorf("expected %q, got %q", expected, actual) + } +} + +func TestErrorsStrictIgnoreNotes(t *testing.T) { + runTests(t, dsn+"&sql_notes=false", func(dbt *DBTest) { + dbt.mustExec("DROP TABLE IF EXISTS does_not_exist") + }) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go index 170aaa0..30b3352 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go @@ -51,7 +51,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { mc.sequence++ // packets with length 0 terminate a previous packet which is a - // multiple of (2^24)−1 bytes long + // multiple of (2^24)-1 bytes long if pktLen == 0 { // there was no previous packet if prevData == nil { @@ -96,6 +96,35 @@ func (mc *mysqlConn) writePacket(data []byte) error { return ErrPktTooLarge } + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.reset { + mc.reset = false + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + // If this connection has a ReadTimeout which we've been setting on + // reads, reset it to its default value before we attempt a non-blocking + // read, otherwise the scheduler will just time us out before we can read + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Time{}) + } + if err == nil { + err = connCheck(conn) + } + if err != nil { + errLog.Print("closing bad idle connection: ", err) + mc.Close() + return driver.ErrBadConn + } + } + for { var size int if pktLen >= maxPacketSize { @@ -194,7 +223,11 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro return nil, "", ErrOldProtocol } if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - return nil, "", ErrNoTLS + if mc.cfg.TLSConfig == "preferred" { + mc.cfg.tls = nil + } else { + return nil, "", ErrNoTLS + } } pos += 2 @@ -243,7 +276,7 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro // Client Authentication Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse -func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, plugin string) error { +func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error { // Adjust client flags based on server support clientFlags := clientProtocol41 | clientSecureConn | @@ -269,7 +302,8 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // encode length of the auth plugin data var authRespLEIBuf [9]byte - authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(authResp))) + authRespLen := len(authResp) + authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen)) if len(authRespLEI) > 1 { // if the length can not be written in 1 byte, it must be written as a // length encoded integer @@ -277,9 +311,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, } pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1 - if addNUL { - pktLen++ - } // To specify a db name if n := len(mc.cfg.DBName); n > 0 { @@ -288,10 +319,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, } // Calculate packet length and get buffer with that size - data := mc.buf.takeSmallBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -330,6 +361,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, if err := tlsConn.Handshake(); err != nil { return err } + mc.rawConn = mc.netConn mc.netConn = tlsConn mc.buf.nc = tlsConn } @@ -350,10 +382,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // Auth Data [length encoded integer] pos += copy(data[pos:], authRespLEI) pos += copy(data[pos:], authResp) - if addNUL { - data[pos] = 0x00 - pos++ - } // Databasename [null terminated string] if len(mc.cfg.DBName) > 0 { @@ -364,30 +392,24 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, pos += copy(data[pos:], plugin) data[pos] = 0x00 + pos++ // Send Auth packet - return mc.writePacket(data) + return mc.writePacket(data[:pos]) } // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte, addNUL bool) error { +func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { pktLen := 4 + len(authData) - if addNUL { - pktLen++ - } - data := mc.buf.takeSmallBuffer(pktLen) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } // Add the auth data [EOF] copy(data[4:], authData) - if addNUL { - data[pktLen-1] = 0x00 - } - return mc.writePacket(data) } @@ -399,10 +421,10 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -418,10 +440,10 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { mc.sequence = 0 pktLen := 1 + len(arg) - data := mc.buf.takeBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -439,10 +461,10 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1 + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -479,7 +501,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { return data[1:], "", err case iEOF: - if len(data) < 1 { + if len(data) == 1 { // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest return nil, "mysql_old_password", nil } @@ -895,7 +917,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { const minPktLen = 4 + 1 + 4 + 1 + 4 mc := stmt.mc - // Determine threshould dynamically to avoid packet size shortage. + // Determine threshold dynamically to avoid packet size shortage. longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1) if longDataSize < 64 { longDataSize = 64 @@ -905,15 +927,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { mc.sequence = 0 var data []byte + var err error if len(args) == 0 { - data = mc.buf.takeBuffer(minPktLen) + data, err = mc.buf.takeBuffer(minPktLen) } else { - data = mc.buf.takeCompleteBuffer() + data, err = mc.buf.takeCompleteBuffer() + // In this case the len(data) == cap(data) which is used to optimise the flow below. } - if data == nil { + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -939,7 +963,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { pos := minPktLen var nullMask []byte - if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { + if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= cap(data) { // buffer has to be extended but we don't know by how much so // we depend on append after all data with known sizes fit. // We stop at that because we deal with a lot of columns here @@ -948,10 +972,11 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { copy(tmp[:pos], data[:pos]) data = tmp nullMask = data[pos : pos+maskLen] + // No need to clean nullMask as make ensures that. pos += maskLen } else { nullMask = data[pos : pos+maskLen] - for i := 0; i < maskLen; i++ { + for i := range nullMask { nullMask[i] = 0 } pos += maskLen @@ -996,6 +1021,22 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { ) } + case uint64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x80 // type is unsigned + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + case float64: paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 @@ -1088,7 +1129,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In that case we must build the data packet with the new values buffer if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) - mc.buf.buf = data + if err = mc.buf.store(data); err != nil { + errLog.Print(err) + return errBadConnNoWrite + } } pos += len(paramValues) diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go new file mode 100644 index 0000000..b61e4db --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go @@ -0,0 +1,336 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "errors" + "net" + "testing" + "time" +) + +var ( + errConnClosed = errors.New("connection is closed") + errConnTooManyReads = errors.New("too many reads") + errConnTooManyWrites = errors.New("too many writes") +) + +// struct to mock a net.Conn for testing purposes +type mockConn struct { + laddr net.Addr + raddr net.Addr + data []byte + written []byte + queuedReplies [][]byte + closed bool + read int + reads int + writes int + maxReads int + maxWrites int +} + +func (m *mockConn) Read(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.reads++ + if m.maxReads > 0 && m.reads > m.maxReads { + return 0, errConnTooManyReads + } + + n = copy(b, m.data) + m.read += n + m.data = m.data[n:] + return +} +func (m *mockConn) Write(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.writes++ + if m.maxWrites > 0 && m.writes > m.maxWrites { + return 0, errConnTooManyWrites + } + + n = len(b) + m.written = append(m.written, b...) + + if n > 0 && len(m.queuedReplies) > 0 { + m.data = m.queuedReplies[0] + m.queuedReplies = m.queuedReplies[1:] + } + return +} +func (m *mockConn) Close() error { + m.closed = true + return nil +} +func (m *mockConn) LocalAddr() net.Addr { + return m.laddr +} +func (m *mockConn) RemoteAddr() net.Addr { + return m.raddr +} +func (m *mockConn) SetDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetReadDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetWriteDeadline(t time.Time) error { + return nil +} + +// make sure mockConn implements the net.Conn interface +var _ net.Conn = new(mockConn) + +func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: NewConfig(), + netConn: conn, + closech: make(chan struct{}), + maxAllowedPacket: defaultMaxAllowedPacket, + sequence: sequence, + } + return conn, mc +} + +func TestReadPacketSingleByte(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 1 { + t.Fatalf("unexpected packet length: expected %d, got %d", 1, len(packet)) + } + if packet[0] != 0xff { + t.Fatalf("unexpected packet content: expected %x, got %x", 0xff, packet[0]) + } +} + +func TestReadPacketWrongSequenceID(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + // too low sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + mc.sequence = 1 + _, err := mc.readPacket() + if err != ErrPktSync { + t.Errorf("expected ErrPktSync, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // too high sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x42, 0xff} + _, err = mc.readPacket() + if err != ErrPktSyncMul { + t.Errorf("expected ErrPktSyncMul, got %v", err) + } +} + +func TestReadPacketSplit(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + data := make([]byte, maxPacketSize*2+4*3) + const pkt2ofs = maxPacketSize + 4 + const pkt3ofs = 2 * (maxPacketSize + 4) + + // case 1: payload has length maxPacketSize + data = data[:pkt2ofs+4] + + // 1st packet has maxPacketSize length and sequence id 0 + // ff ff ff 00 ... + data[0] = 0xff + data[1] = 0xff + data[2] = 0xff + + // mark the payload start and end of 1st packet so that we can check if the + // content was correctly appended + data[4] = 0x11 + data[maxPacketSize+3] = 0x22 + + // 2nd packet has payload length 0 and squence id 1 + // 00 00 00 01 + data[pkt2ofs+3] = 0x01 + + conn.data = data + conn.maxReads = 3 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize-1] != 0x22 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x22, packet[maxPacketSize-1]) + } + + // case 2: payload has length which is a multiple of maxPacketSize + data = data[:cap(data)] + + // 2nd packet now has maxPacketSize length + data[pkt2ofs] = 0xff + data[pkt2ofs+1] = 0xff + data[pkt2ofs+2] = 0xff + + // mark the payload start and end of the 2nd packet + data[pkt2ofs+4] = 0x33 + data[pkt2ofs+maxPacketSize+3] = 0x44 + + // 3rd packet has payload length 0 and squence id 2 + // 00 00 00 02 + data[pkt3ofs+3] = 0x02 + + conn.data = data + conn.reads = 0 + conn.maxReads = 5 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 2*maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", 2*maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[2*maxPacketSize-1] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[2*maxPacketSize-1]) + } + + // case 3: payload has a length larger maxPacketSize, which is not an exact + // multiple of it + data = data[:pkt2ofs+4+42] + data[pkt2ofs] = 0x2a + data[pkt2ofs+1] = 0x00 + data[pkt2ofs+2] = 0x00 + data[pkt2ofs+4+41] = 0x44 + + conn.data = data + conn.reads = 0 + conn.maxReads = 4 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize+42 { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize+42, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize+41] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[maxPacketSize+41]) + } +} + +func TestReadPacketFail(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + closech: make(chan struct{}), + } + + // illegal empty (stand-alone) packet + conn.data = []byte{0x00, 0x00, 0x00, 0x00} + conn.maxReads = 1 + _, err := mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read header + conn.closed = true + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.closed = false + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read body + conn.maxReads = 1 + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } +} + +// https://github.com/go-sql-driver/mysql/pull/801 +// not-NUL terminated plugin_name in init packet +func TestRegression801(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: new(Config), + sequence: 42, + closech: make(chan struct{}), + } + + conn.data = []byte{72, 0, 0, 42, 10, 53, 46, 53, 46, 56, 0, 165, 0, 0, 0, + 60, 70, 63, 58, 68, 104, 34, 97, 0, 223, 247, 33, 2, 0, 15, 128, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 120, 114, 47, 85, 75, 109, 99, 51, 77, + 50, 64, 0, 109, 121, 115, 113, 108, 95, 110, 97, 116, 105, 118, 101, 95, + 112, 97, 115, 115, 119, 111, 114, 100} + conn.maxReads = 1 + + authData, pluginName, err := mc.readHandshakePacket() + if err != nil { + t.Fatalf("got error: %v", err) + } + + if pluginName != "mysql_native_password" { + t.Errorf("expected plugin name 'mysql_native_password', got '%s'", pluginName) + } + + expectedAuthData := []byte{60, 70, 63, 58, 68, 104, 34, 97, 98, 120, 114, + 47, 85, 75, 109, 99, 51, 77, 50, 64} + if !bytes.Equal(authData, expectedAuthData) { + t.Errorf("expected authData '%v', got '%v'", expectedAuthData, authData) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go index d3b1e28..888bdb5 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go @@ -111,6 +111,13 @@ func (rows *mysqlRows) Close() (err error) { return err } + // flip the buffer for this connection if we need to drain it. + // note that for a successful query (i.e. one where rows.next() + // has been called until it returns false), `rows.mc` will be nil + // by the time the user calls `(*Rows).Close`, so we won't reach this + // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 + mc.buf.flip() + // Remove unread packets from stream if !rows.rs.done { err = mc.readUntilEOF() diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go index ce7fe4c..f7e3709 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "reflect" - "strconv" ) type mysqlStmt struct { @@ -164,14 +163,8 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint(), nil case reflect.Float32, reflect.Float64: return rv.Float(), nil case reflect.Bool: diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go new file mode 100644 index 0000000..4b9914f --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go @@ -0,0 +1,126 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "testing" +) + +func TestConvertDerivedString(t *testing.T) { + type derived string + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Derived string type not convertible", err) + } + + if output != "value" { + t.Fatalf("Derived string type not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedByteSlice(t *testing.T) { + type derived []uint8 + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Byte slice not convertible", err) + } + + if bytes.Compare(output.([]byte), []byte("value")) != 0 { + t.Fatalf("Byte slice not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedUnsupportedSlice(t *testing.T) { + type derived []int + + _, err := converter{}.ConvertValue(derived{1}) + if err == nil || err.Error() != "unsupported type mysql.derived, a slice of int" { + t.Fatal("Unexpected error", err) + } +} + +func TestConvertDerivedBool(t *testing.T) { + type derived bool + + output, err := converter{}.ConvertValue(derived(true)) + if err != nil { + t.Fatal("Derived bool type not convertible", err) + } + + if output != true { + t.Fatalf("Derived bool type not converted, got %#v %T", output, output) + } +} + +func TestConvertPointer(t *testing.T) { + str := "value" + + output, err := converter{}.ConvertValue(&str) + if err != nil { + t.Fatal("Pointer type not convertible", err) + } + + if output != "value" { + t.Fatalf("Pointer type not converted, got %#v %T", output, output) + } +} + +func TestConvertSignedIntegers(t *testing.T) { + values := []interface{}{ + int8(-42), + int16(-42), + int32(-42), + int64(-42), + int(-42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != int64(-42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } +} + +func TestConvertUnsignedIntegers(t *testing.T) { + values := []interface{}{ + uint8(42), + uint16(42), + uint32(42), + uint64(42), + uint(42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != uint64(42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } + + output, err := converter{}.ConvertValue(^uint64(0)) + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if output != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", output, output) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go index cb3650b..cfa10e9 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go @@ -56,7 +56,7 @@ var ( // db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") // func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { return fmt.Errorf("key '%s' is reserved", key) } @@ -684,7 +684,7 @@ type atomicBool struct { value uint32 } -// IsSet returns wether the current boolean value is true +// IsSet returns whether the current boolean value is true func (ab *atomicBool) IsSet() bool { return atomic.LoadUint32(&ab.value) > 0 } @@ -698,7 +698,7 @@ func (ab *atomicBool) Set(value bool) { } } -// TrySet sets the value of the bool and returns wether the value changed +// TrySet sets the value of the bool and returns whether the value changed func (ab *atomicBool) TrySet(value bool) bool { if value { return atomic.SwapUint32(&ab.value, 1) == 0 diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go new file mode 100644 index 0000000..8951a7a --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go @@ -0,0 +1,334 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "database/sql" + "database/sql/driver" + "encoding/binary" + "testing" + "time" +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} + +func TestLengthEncodedInteger(t *testing.T) { + var integerTests = []struct { + num uint64 + encoded []byte + }{ + {0x0000000000000000, []byte{0x00}}, + {0x0000000000000012, []byte{0x12}}, + {0x00000000000000fa, []byte{0xfa}}, + {0x0000000000000100, []byte{0xfc, 0x00, 0x01}}, + {0x0000000000001234, []byte{0xfc, 0x34, 0x12}}, + {0x000000000000ffff, []byte{0xfc, 0xff, 0xff}}, + {0x0000000000010000, []byte{0xfd, 0x00, 0x00, 0x01}}, + {0x0000000000123456, []byte{0xfd, 0x56, 0x34, 0x12}}, + {0x0000000000ffffff, []byte{0xfd, 0xff, 0xff, 0xff}}, + {0x0000000001000000, []byte{0xfe, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}}, + {0x123456789abcdef0, []byte{0xfe, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12}}, + {0xffffffffffffffff, []byte{0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + } + + for _, tst := range integerTests { + num, isNull, numLen := readLengthEncodedInteger(tst.encoded) + if isNull { + t.Errorf("%x: expected %d, got NULL", tst.encoded, tst.num) + } + if num != tst.num { + t.Errorf("%x: expected %d, got %d", tst.encoded, tst.num, num) + } + if numLen != len(tst.encoded) { + t.Errorf("%x: expected size %d, got %d", tst.encoded, len(tst.encoded), numLen) + } + encoded := appendLengthEncodedInteger(nil, num) + if !bytes.Equal(encoded, tst.encoded) { + t.Errorf("%v: expected %x, got %x", num, tst.encoded, encoded) + } + } +} + +func TestFormatBinaryDateTime(t *testing.T) { + rawDate := [11]byte{} + binary.LittleEndian.PutUint16(rawDate[:2], 1978) // years + rawDate[2] = 12 // months + rawDate[3] = 30 // days + rawDate[4] = 15 // hours + rawDate[5] = 46 // minutes + rawDate[6] = 23 // seconds + binary.LittleEndian.PutUint32(rawDate[7:], 987654) // microseconds + expect := func(expected string, inlen, outlen uint8) { + actual, _ := formatBinaryDateTime(rawDate[:inlen], outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for length in %d, out %d", + expected, actual, inlen, outlen, + ) + } + } + expect("0000-00-00", 0, 10) + expect("0000-00-00 00:00:00", 0, 19) + expect("1978-12-30", 4, 10) + expect("1978-12-30 15:46:23", 7, 19) + expect("1978-12-30 15:46:23.987654", 11, 26) +} + +func TestFormatBinaryTime(t *testing.T) { + expect := func(expected string, src []byte, outlen uint8) { + actual, _ := formatBinaryTime(src, outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for src=%q and outlen=%d", + expected, actual, src, outlen) + } + } + + // binary format: + // sign (0: positive, 1: negative), days(4), hours, minutes, seconds, micro(4) + + // Zeros + expect("00:00:00", []byte{}, 8) + expect("00:00:00.0", []byte{}, 10) + expect("00:00:00.000000", []byte{}, 15) + + // Without micro(4) + expect("12:34:56", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("-12:34:56", []byte{1, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 11) + expect("24:34:56", []byte{0, 1, 0, 0, 0, 0, 34, 56}, 8) + expect("-99:34:56", []byte{1, 4, 0, 0, 0, 3, 34, 56}, 8) + expect("103079215103:34:56", []byte{0, 255, 255, 255, 255, 23, 34, 56}, 8) + + // With micro(4) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 11) + expect("12:34:56.000099", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 15) +} + +func TestEscapeBackslash(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesBackslash([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringBackslash([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\\0bar", "foo\x00bar") + expect("foo\\nbar", "foo\nbar") + expect("foo\\rbar", "foo\rbar") + expect("foo\\Zbar", "foo\x1abar") + expect("foo\\\"bar", "foo\"bar") + expect("foo\\\\bar", "foo\\bar") + expect("foo\\'bar", "foo'bar") +} + +func TestEscapeQuotes(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesQuotes([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringQuotes([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\x00bar", "foo\x00bar") // not affected + expect("foo\nbar", "foo\nbar") // not affected + expect("foo\rbar", "foo\rbar") // not affected + expect("foo\x1abar", "foo\x1abar") // not affected + expect("foo''bar", "foo'bar") // affected + expect("foo\"bar", "foo\"bar") // not affected +} + +func TestAtomicBool(t *testing.T) { + var ab atomicBool + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(true) + if ab.value != 1 { + t.Fatal("Set(true) did not set value to 1") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(false) + if ab.value != 0 { + t.Fatal("Set(false) did not set value to 0") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(false) + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + if ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to fail") + } + if !ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to succeed") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + if ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to fail") + } + if !ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to succeed") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab._noCopy.Lock() // we've "tested" it ¯\_(ツ)_/¯ +} + +func TestAtomicError(t *testing.T) { + var ae atomicError + if ae.Value() != nil { + t.Fatal("Expected value to be nil") + } + + ae.Set(ErrMalformPkt) + if v := ae.Value(); v != ErrMalformPkt { + if v == nil { + t.Fatal("Value is still nil") + } + t.Fatal("Error did not match") + } + ae.Set(ErrPktSync) + if ae.Value() == ErrMalformPkt { + t.Fatal("Error still matches old error") + } + if v := ae.Value(); v != ErrPktSync { + t.Fatal("Error did not match") + } +} + +func TestIsolationLevelMapping(t *testing.T) { + data := []struct { + level driver.IsolationLevel + expected string + }{ + { + level: driver.IsolationLevel(sql.LevelReadCommitted), + expected: "READ COMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelRepeatableRead), + expected: "REPEATABLE READ", + }, + { + level: driver.IsolationLevel(sql.LevelReadUncommitted), + expected: "READ UNCOMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelSerializable), + expected: "SERIALIZABLE", + }, + } + + for i, td := range data { + if actual, err := mapIsolationLevel(td.level); actual != td.expected || err != nil { + t.Fatal(i, td.expected, actual, err) + } + } + + // check unsupported mapping + expectedErr := "mysql: unsupported isolation level: 7" + actual, err := mapIsolationLevel(driver.IsolationLevel(sql.LevelLinearizable)) + if actual != "" || err == nil { + t.Fatal("Expected error on unsupported isolation level") + } + if err.Error() != expectedErr { + t.Fatalf("Expected error to be %q, got %q", expectedErr, err) + } +} From 84fe34b22c9a80a265ffbea043ef1918ceed7eef Mon Sep 17 00:00:00 2001 From: Zach Moazeni Date: Mon, 10 Jun 2019 09:24:03 -0400 Subject: [PATCH 032/103] Bump default-retries to 3 There are some legitimate retries that can occur during testing. Namely `logic.ExpectProcess()` (in `applier.go`). We'll look for a process that does exist, but timing-wise doesn't have the `state` or `info` columns populated. Without this, the test will fail abruptly. --- localtests/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localtests/test.sh b/localtests/test.sh index 17b60e9..1078965 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -152,7 +152,7 @@ test_single() { --serve-socket-file=/tmp/gh-ost.test.sock \ --initially-drop-socket-file \ --test-on-replica \ - --default-retries=1 \ + --default-retries=3 \ --chunk-size=10 \ --verbose \ --debug \ From 43ea21c3bb1b6b9b5e50038278e7bebbe5d0f096 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 14:54:00 +0300 Subject: [PATCH 033/103] Initial GitHub CI/CD GitHub CI/CD is [a thing](https://github.com/features/actions)! We will attempt: - CI: build + unit test - replica tests: running `gh-ost` tests on multiple MySQL versions --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ec8978d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: CI + +on: [pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + + - name: Set up Go 1.12 + uses: actions/setup-go@v1 + with: + version: 1.12 + id: go + + - name: Build + run: script/cibuild From 6cfa76d1399238fb6350125e1b20613247cb50c2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 14:58:37 +0300 Subject: [PATCH 034/103] adding replica-tests --- .github/workflows/replica-tests.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/replica-tests.yml diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml new file mode 100644 index 0000000..5fbff05 --- /dev/null +++ b/.github/workflows/replica-tests.yml @@ -0,0 +1,20 @@ +name: CI + +on: [pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + + - name: Set up Go 1.12 + uses: actions/setup-go@v1 + with: + version: 1.12 + id: go + + - name: Build + run: script/cibuild-gh-ost-replica-tests From 18612b661ac30e3ff45237ba104af251308a0a8f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 14:59:22 +0300 Subject: [PATCH 035/103] name of test --- .github/workflows/replica-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index 5fbff05..ac2535b 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -1,4 +1,4 @@ -name: CI +name: Replica tests on: [pull_request] From e618682ff9355093c6b7ce7625aec3057e352891 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:04:04 +0300 Subject: [PATCH 036/103] go1.12 --- .github/workflows/replica-tests.yml | 4 ++-- script/ensure-go-installed | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index ac2535b..d775131 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -1,4 +1,4 @@ -name: Replica tests +name: migration tests on: [pull_request] @@ -16,5 +16,5 @@ jobs: version: 1.12 id: go - - name: Build + - name: migration tests run: script/cibuild-gh-ost-replica-tests diff --git a/script/ensure-go-installed b/script/ensure-go-installed index a511e0e..c9c2ae8 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,20 +1,20 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.9.2 -SUPPORTED_GO_VERSIONS='go1.[89]' +PREFERRED_GO_VERSION=go1.12.6 +SUPPORTED_GO_VERSIONS='go1.1[234]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=8b4f6ae6deae1150d2e341d02c247fd18a99af387516540eeb84702ffd76d3a1 +GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815 GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz -GO_PKG_LINUX_SHA=de874549d9a8d8d8062be05808509c09a88a248e77ec14eb77453530829ac02b +GO_PKG_LINUX_SHA=dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" cd $ROOTDIR # If Go isn't installed globally, setup environment variables for local install. if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" ]; then - GODIR="$ROOTDIR/.vendor/go19" + GODIR="$ROOTDIR/.vendor/golocal" if [ $(uname -s) = "Darwin" ]; then export GOROOT="$GODIR/usr/local/go" @@ -32,12 +32,12 @@ if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" cd "$GODIR"; if [ $(uname -s) = "Darwin" ]; then - curl -L -O https://storage.googleapis.com/golang/$GO_PKG_DARWIN + curl -L -O https://dl.google.com/go/$GO_PKG_DARWIN shasum -a256 $GO_PKG_DARWIN | grep $GO_PKG_DARWIN_SHA xar -xf $GO_PKG_DARWIN cpio -i < com.googlecode.go.pkg/Payload else - curl -L -O https://storage.googleapis.com/golang/$GO_PKG_LINUX + curl -L -O https://dl.google.com/go/$GO_PKG_LINUX shasum -a256 $GO_PKG_LINUX | grep $GO_PKG_LINUX_SHA tar xf $GO_PKG_LINUX fi From b881bff1e8f3801ed00a727bb9bafe42c7299c46 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:06:03 +0300 Subject: [PATCH 037/103] fixed linter issue --- go/logic/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/server.go b/go/logic/server.go index 774c4ab..4c1f581 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -144,7 +144,7 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr switch command { case "help": { - fmt.Fprintln(writer, `available commands: + fmt.Fprint(writer, `available commands: status # Print a detailed status message sup # Print a short status message coordinates # Print the currently inspected coordinates From 106ca5df3caa3a9c972dbc69cf966eb583e72406 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:12:14 +0300 Subject: [PATCH 038/103] debug info --- script/cibuild-gh-ost-replica-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index ab8c689..c312aa9 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -44,9 +44,11 @@ test_mysql_version() { sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master + cat gh-ost-ci-env/bin/gh-ost-test-mysql-master sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica + cat gh-ost-ci-env/bin/gh-ost-test-mysql-replica export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" From 1305ff6e010e40631fb887c3e317acedb7c09af5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:17:48 +0300 Subject: [PATCH 039/103] debug info --- localtests/test.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/localtests/test.sh b/localtests/test.sh index 1078965..0a913ae 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -50,6 +50,22 @@ verify_master_and_replica() { original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" echo "sql_mode on master is ${original_sql_mode}" + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then echo "Cannot verify gh-ost-test-mysql-replica" exit 1 From 90ab2368dd247da73d1cc58c726fc7fc089e86ed Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:21:04 +0300 Subject: [PATCH 040/103] graceful sleep; remove extra debug info --- localtests/test.sh | 17 ++--------------- script/cibuild-gh-ost-replica-tests | 2 -- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/localtests/test.sh b/localtests/test.sh index 0a913ae..d4b3f17 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -50,21 +50,8 @@ verify_master_and_replica() { original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" echo "sql_mode on master is ${original_sql_mode}" - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 + echo "Gracefully sleeping for 3 seconds while replica is setting up..." + sleep 3 if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then echo "Cannot verify gh-ost-test-mysql-replica" diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c312aa9..ab8c689 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -44,11 +44,9 @@ test_mysql_version() { sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master - cat gh-ost-ci-env/bin/gh-ost-test-mysql-master sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica - cat gh-ost-ci-env/bin/gh-ost-test-mysql-replica export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" From 17e4fe80175c1d00b2665017d07a3571619814d5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:26:29 +0300 Subject: [PATCH 041/103] TravisCI updated to go1.12 ; but we are likely to remove TravisCI anyhow --- .travis.yml | 3 +-- README.md | 2 +- build.sh | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6b68f3..1944f00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,7 @@ language: go go: - - "1.9" - - "1.10" + - "1.12.x" os: - linux diff --git a/README.md b/README.md index 008c65b..cd6f592 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d [Download latest release here](https://github.com/github/gh-ost/releases/latest) -`gh-ost` is a Go project; it is built with Go `1.9` and above. To build on your own, use either: +`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either: - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary. - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost` diff --git a/build.sh b/build.sh index df9ac7d..8b6925d 100755 --- a/build.sh +++ b/build.sh @@ -20,9 +20,9 @@ function build { - if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then - echo "go version is too low. Must use 1.9 or above" - exit 1 + if ! go version | egrep -q 'go(1\.1[234])' ; then + echo "go version must be 1.12 or above" + ok=1 fi echo "Building ${osname} binary" From e1c8df744788721ef7b1def8c75b4981f686f0bf Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:29:28 +0300 Subject: [PATCH 042/103] travis: declaring mysql service --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1944f00..c1c1891 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ go: os: - linux +services: + - mysql + env: - MYSQL_USER=root - CURRENT_CI_ENV=travis From 143abf05b0f85080cc1e27a6c5f2ed5f1538cef3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 12 Aug 2019 08:03:17 +0300 Subject: [PATCH 043/103] fail when go is < 1.12 --- build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 8b6925d..b7f3a54 100755 --- a/build.sh +++ b/build.sh @@ -18,11 +18,9 @@ function build { GOOS=$3 GOARCH=$4 - - if ! go version | egrep -q 'go(1\.1[234])' ; then echo "go version must be 1.12 or above" - ok=1 + exit 1 fi echo "Building ${osname} binary" From 3362c5d32254bd78e0bfcf305c77cff46e62983c Mon Sep 17 00:00:00 2001 From: Gillian Gunson Date: Mon, 12 Aug 2019 12:40:31 -0700 Subject: [PATCH 044/103] Changing panic message --- go/logic/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/server.go b/go/logic/server.go index 774c4ab..e1919ab 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -341,7 +341,7 @@ help # This message err := fmt.Errorf("User commanded 'panic' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) return NoPrintStatusRule, err } - err := fmt.Errorf("User commanded 'panic'. I will now panic, without cleanup. PANIC!") + err := fmt.Errorf("User commanded 'panic'. The migration will be aborted without cleanup. Please drop the gh-ost tables before trying again.") this.migrationContext.PanicAbort <- err return NoPrintStatusRule, err } From 6cc1a1a34e27a301fb755ff21646acc9f3f7cdf0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 15 Aug 2019 11:10:48 +0300 Subject: [PATCH 045/103] CI: adapting to internal builds system --- Dockerfile.test | 11 ++++ docker-compose.yml | 7 +++ script/bootstrap | 1 + script/build-deploy-tarball | 35 ++++++++++++ script/cibuild | 16 +----- script/cibuild-gh-ost-build-deploy-tarball | 66 +++++++++++++--------- script/test | 17 ++++++ 7 files changed, 110 insertions(+), 43 deletions(-) create mode 100644 Dockerfile.test create mode 100644 docker-compose.yml create mode 100644 script/build-deploy-tarball create mode 100644 script/test diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..6abc4a0 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,11 @@ +FROM golang:1.12.1 +LABEL maintainer="github@github.com" + +RUN apt-get update +RUN apt-get install -y lsb-release +RUN rm -rf /var/lib/apt/lists/* + +COPY . /go/src/github.com/github/gh-ost +WORKDIR /go/src/github.com/github/gh-ost + +CMD ["script/test"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8a2bd2e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: "3.5" +services: + app: + image: app + build: + context: . + dockerfile: Dockerfile.test diff --git a/script/bootstrap b/script/bootstrap index 6ac885b..573313a 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -4,6 +4,7 @@ set -e # Make sure we have the version of Go we want to depend on, either from the # system or one we grab ourselves. +# If executing from within Dockerfile then this assumption is inherently true, since we use a `golang` docker image. . script/ensure-go-installed # Since we want to be able to build this outside of GOPATH, we set it diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball new file mode 100644 index 0000000..d36b6c9 --- /dev/null +++ b/script/build-deploy-tarball @@ -0,0 +1,35 @@ +#!/bin/sh + +set -e + +script/build + +# Get a fresh directory and make sure to delete it afterwards +build_dir=tmp/build +rm -rf $build_dir +mkdir -p $build_dir +trap "rm -rf $build_dir" EXIT + +commit_sha=$(git rev-parse HEAD) + +if [ $(uname -s) = "Darwin" ]; then + build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" +else + build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" +fi + +tarball=$build_dir/${commit_sha}-${build_arch}.tar + +# Create the tarball +tar cvf $tarball --mode="ugo=rx" bin/ + +# Compress it and copy it to the directory for the CI to upload it +gzip $tarball +mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost +cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ + +### HACK HACK HACK ### +# blame @carlosmn and @mattr- +# Allow builds on stretch to also be used for jessie +jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-stretch-/-jessie-/) +cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" diff --git a/script/cibuild b/script/cibuild index 7e757b5..e609b7a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,17 +1,3 @@ #!/bin/bash -set -e - -. script/bootstrap - -echo "Verifying code is formatted via 'gofmt -s -w go/'" -gofmt -s -w go/ -git diff --exit-code --quiet - -echo "Building" -script/build - -cd .gopath/src/github.com/github/gh-ost - -echo "Running unit tests" -go test ./go/... +script/test diff --git a/script/cibuild-gh-ost-build-deploy-tarball b/script/cibuild-gh-ost-build-deploy-tarball index 692b42b..a852ad6 100755 --- a/script/cibuild-gh-ost-build-deploy-tarball +++ b/script/cibuild-gh-ost-build-deploy-tarball @@ -1,37 +1,47 @@ -#!/bin/sh +#!/bin/bash -set -e +output_fold() { + # Exit early if no label provided + if [ -z "$1" ]; then + echo "output_fold(): requires a label argument." + return + fi -script/cibuild + exit_value=0 # exit_value is used to record exit status of the given command + label=$1 # human-readable label describing what's being folded up + shift 1 # having retrieved the output_fold()-specific arguments, strip them off $@ -# Get a fresh directory and make sure to delete it afterwards -build_dir=tmp/build -rm -rf $build_dir -mkdir -p $build_dir -trap "rm -rf $build_dir" EXIT + # Only echo the tags when in CI_MODE + if [ "$CI_MODE" ]; then + echo "%%%FOLD {$label}%%%" + fi -commit_sha=$(git rev-parse HEAD) + # run the remaining arguments. If the command exits non-0, the `||` will + # prevent the `-e` flag from seeing the failure exit code, and we'll see + # the second echo execute + "$@" || exit_value=$? -if [ $(uname -s) = "Darwin" ]; then - build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" -else - build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" -fi + # Only echo the tags when in CI_MODE + if [ "$CI_MODE" ]; then + echo "%%%END FOLD%%%" + fi -tarball=$build_dir/${commit_sha}-${build_arch}.tar + # preserve the exit code from the subcommand. + return $exit_value +} -# Create the tarball -tar cvf $tarball --mode="ugo=rx" bin/ +function cleanup() { + echo + echo "%%%FOLD {Shutting down services...}%%%" + docker-compose down + echo "%%%END FOLD%%%" +} -# Compress it and copy it to the directory for the CI to upload it -gzip $tarball -mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost -cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ +trap cleanup EXIT -### HACK HACK HACK ### -# Blame @carlosmn. In the good way. -# We don't have any jessie machines for building, but a pure-Go binary depends -# on a version of libc and ld which are widely available, so we can copy the -# tarball over with jessie in its name so we can deploy it on jessie machines. -jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-precise-/-jessie-/) -cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" +export CI_MODE=true + +output_fold "Bootstrapping container..." docker-compose build +output_fold "Running tests..." docker-compose run --rm app + +docker-compose run -e BUILD_ARTIFACT_DIR=$BUILD_ARTIFACT_DIR -v $BUILD_ARTIFACT_DIR:$BUILD_ARTIFACT_DIR app script/build-deploy-tarball diff --git a/script/test b/script/test new file mode 100644 index 0000000..7e757b5 --- /dev/null +++ b/script/test @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +. script/bootstrap + +echo "Verifying code is formatted via 'gofmt -s -w go/'" +gofmt -s -w go/ +git diff --exit-code --quiet + +echo "Building" +script/build + +cd .gopath/src/github.com/github/gh-ost + +echo "Running unit tests" +go test ./go/... From 36b0adb1b8a06aa490f00518348e5c4dcf37ce42 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 15 Aug 2019 11:14:10 +0300 Subject: [PATCH 046/103] executable scripts --- script/build-deploy-tarball | 0 script/test | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 script/build-deploy-tarball mode change 100644 => 100755 script/test diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball old mode 100644 new mode 100755 diff --git a/script/test b/script/test old mode 100644 new mode 100755 From 78698899880b45c6e93a715b190317b55d07eabc Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 6 Oct 2019 17:08:35 +0300 Subject: [PATCH 047/103] context, status and hooks: progressPct and CurrentLag --- go/base/context.go | 5 +++++ go/logic/hooks.go | 3 +++ go/logic/migrator.go | 7 +++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index f12cd43..84e6264 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -174,6 +174,7 @@ type MigrationContext struct { pointOfInterestTime time.Time pointOfInterestTimeMutex *sync.Mutex CurrentLag int64 + CurrentProgress uint64 // math.Float64bits([f=0..100]) ThrottleHTTPStatusCode int64 controlReplicasLagResult mysql.ReplicationLagResult TotalRowsCopied int64 @@ -428,6 +429,10 @@ func (this *MigrationContext) MarkRowCopyEndTime() { this.RowCopyEndTime = time.Now() } +func (this *MigrationContext) GetCurrentLagDuration() time.Duration { + return time.Duration(atomic.LoadInt64(&this.CurrentLag)) +} + // GetTotalRowsCopied returns the accurate number of rows being copied (affected) // This is not exactly the same as the rows being iterated via chunks, but potentially close enough func (this *MigrationContext) GetTotalRowsCopied() int64 { diff --git a/go/logic/hooks.go b/go/logic/hooks.go index be11130..b56b0b4 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -8,6 +8,7 @@ package logic import ( "fmt" + "math" "os" "os/exec" "path/filepath" @@ -63,6 +64,8 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.GetApplierHostname())) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) + env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", math.Float64frombits(atomic.LoadUint64(&this.migrationContext.CurrentProgress)))) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 86de5ac..985bef1 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -895,6 +895,8 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { } else { progressPct = 100.0 * float64(totalRowsCopied) / float64(rowsEstimate) } + // we take the opportunity to update migration context with progressPct + atomic.StoreUint64(&this.migrationContext.CurrentProgress, math.Float64bits(progressPct)) // Before status, let's see if we should print a nice reminder for what exactly we're doing here. shouldPrintMigrationStatusHint := (elapsedSeconds%600 == 0) if rule == ForcePrintStatusAndHintRule { @@ -911,7 +913,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { eta := "N/A" if progressPct >= 100.0 { eta = "due" - } else if progressPct >= 1.0 { + } else if progressPct >= 0.1 { elapsedRowCopySeconds := this.migrationContext.ElapsedRowCopyTime().Seconds() totalExpectedSeconds := elapsedRowCopySeconds * float64(rowsEstimate) / float64(totalRowsCopied) etaSeconds = totalExpectedSeconds - elapsedRowCopySeconds @@ -958,12 +960,13 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+v, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied), len(this.applyEventsQueue), cap(this.applyEventsQueue), base.PrettifyDurationOutput(elapsedTime), base.PrettifyDurationOutput(this.migrationContext.ElapsedRowCopyTime()), currentBinlogCoordinates, + this.migrationContext.GetCurrentLagDuration().Seconds(), state, eta, ) From 8893b2207b4bbadedf80f428dc43de28810dc4b4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 6 Oct 2019 17:12:20 +0300 Subject: [PATCH 048/103] documenting new hook env variables --- doc/hooks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/hooks.md b/doc/hooks.md index 93f500f..4c49c85 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -65,6 +65,8 @@ The following variables are available on all hooks: - `GH_OST_ELAPSED_COPY_SECONDS` - row-copy time (excluding startup, row-count and postpone time) - `GH_OST_ESTIMATED_ROWS` - estimated total rows in table - `GH_OST_COPIED_ROWS` - number of rows copied by `gh-ost` +- `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server +- `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration - `GH_OST_MIGRATED_HOST` - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` From 271c7274a3dbac3f82c8939ccfbbc22ec8bcd964 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 7 Oct 2019 06:59:56 +0300 Subject: [PATCH 049/103] refactor progressPct into migrationContext --- go/base/context.go | 13 ++++++++++++- go/logic/hooks.go | 3 +-- go/logic/migrator.go | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 84e6264..5ebf092 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -7,6 +7,7 @@ package base import ( "fmt" + "math" "os" "regexp" "strings" @@ -174,7 +175,7 @@ type MigrationContext struct { pointOfInterestTime time.Time pointOfInterestTimeMutex *sync.Mutex CurrentLag int64 - CurrentProgress uint64 // math.Float64bits([f=0..100]) + currentProgress uint64 ThrottleHTTPStatusCode int64 controlReplicasLagResult mysql.ReplicationLagResult TotalRowsCopied int64 @@ -433,6 +434,16 @@ func (this *MigrationContext) GetCurrentLagDuration() time.Duration { return time.Duration(atomic.LoadInt64(&this.CurrentLag)) } +func (this *MigrationContext) GetProgressPct() float64 { + return math.Float64frombits(atomic.LoadUint64(&this.currentProgress)) +} + +func (this *MigrationContext) SetProgressPct(progressPct float64) { + atomic.StoreUint64(&this.currentProgress, math.Float64bits(progressPct)) +} + +// math.Float64bits([f=0..100]) + // GetTotalRowsCopied returns the accurate number of rows being copied (affected) // This is not exactly the same as the rows being iterated via chunks, but potentially close enough func (this *MigrationContext) GetTotalRowsCopied() int64 { diff --git a/go/logic/hooks.go b/go/logic/hooks.go index b56b0b4..fa5011e 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -8,7 +8,6 @@ package logic import ( "fmt" - "math" "os" "os/exec" "path/filepath" @@ -65,7 +64,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) - env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", math.Float64frombits(atomic.LoadUint64(&this.migrationContext.CurrentProgress)))) + env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 985bef1..1446a65 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -896,7 +896,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { progressPct = 100.0 * float64(totalRowsCopied) / float64(rowsEstimate) } // we take the opportunity to update migration context with progressPct - atomic.StoreUint64(&this.migrationContext.CurrentProgress, math.Float64bits(progressPct)) + this.migrationContext.SetProgressPct(progressPct) // Before status, let's see if we should print a nice reminder for what exactly we're doing here. shouldPrintMigrationStatusHint := (elapsedSeconds%600 == 0) if rule == ForcePrintStatusAndHintRule { From 21e7ec6b7c005d713eae354f76f6791a9f539e50 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 7 Oct 2019 07:06:11 +0300 Subject: [PATCH 050/103] lag: %+vs notation, e.g. 1s --- go/logic/migrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 1446a65..b0bec68 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -960,7 +960,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+v, State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+vs, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied), len(this.applyEventsQueue), cap(this.applyEventsQueue), From f4676bf463e39a983803c55fe8d790f0ce858951 Mon Sep 17 00:00:00 2001 From: Abeyu M Date: Mon, 7 Oct 2019 11:10:36 -0400 Subject: [PATCH 051/103] implement a logging interface --- go/base/context.go | 21 ++++++ go/base/default_logger.go | 73 +++++++++++++++++++ go/base/utils.go | 3 +- go/cmd/gh-ost/main.go | 64 ++++++++--------- go/logic/applier.go | 129 +++++++++++++++++----------------- go/logic/inspect.go | 71 +++++++++---------- go/logic/migrator.go | 144 +++++++++++++++++++------------------- go/logic/server.go | 13 ++-- go/logic/streamer.go | 9 ++- go/logic/throttler.go | 13 ++-- 10 files changed, 313 insertions(+), 227 deletions(-) create mode 100644 go/base/default_logger.go diff --git a/go/base/context.go b/go/base/context.go index f12cd43..a4318b9 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -7,6 +7,7 @@ package base import ( "fmt" + "github.com/outbrain/golib/log" "os" "regexp" "strings" @@ -214,6 +215,25 @@ type MigrationContext struct { ForceTmpTableName string recentBinlogCoordinates mysql.BinlogCoordinates + + Log Logger +} + +type Logger interface { + Debug(args ...interface{}) + Debugf(format string, args ...interface{}) + Info(args ...interface{}) + Infof(format string, args ...interface{}) + Warning(args ...interface{}) error + Warningf(format string, args ...interface{}) error + Error(args ...interface{}) error + Errorf(format string, args ...interface{}) error + Errore(err error) error + Fatal(args ...interface{}) error + Fatalf(format string, args ...interface{}) error + Fatale(err error) error + SetLevel(level log.LogLevel) + SetPrintStackTrace(printStackTraceFlag bool) } type ContextConfig struct { @@ -248,6 +268,7 @@ func NewMigrationContext() *MigrationContext { pointOfInterestTimeMutex: &sync.Mutex{}, ColumnRenameMap: make(map[string]string), PanicAbort: make(chan error), + Log: NewDefaultLogger(), } } diff --git a/go/base/default_logger.go b/go/base/default_logger.go new file mode 100644 index 0000000..be6b1f2 --- /dev/null +++ b/go/base/default_logger.go @@ -0,0 +1,73 @@ +package base + +import ( + "github.com/outbrain/golib/log" +) + +type simpleLogger struct{} + +func NewDefaultLogger() *simpleLogger { + return &simpleLogger{} +} + +func (*simpleLogger) Debug(args ...interface{}) { + log.Debug(args[0].(string), args[1:]) + return +} + +func (*simpleLogger) Debugf(format string, args ...interface{}) { + log.Debugf(format, args...) + return +} + +func (*simpleLogger) Info(args ...interface{}) { + log.Info(args[0].(string), args[1:]) + return +} + +func (*simpleLogger) Infof(format string, args ...interface{}) { + log.Infof(format, args...) + return +} + +func (*simpleLogger) Warning(args ...interface{}) error { + return log.Warning(args[0].(string), args[1:]) +} + +func (*simpleLogger) Warningf(format string, args ...interface{}) error { + return log.Warningf(format, args...) +} + +func (*simpleLogger) Error(args ...interface{}) error { + return log.Error(args[0].(string), args[1:]) +} + +func (*simpleLogger) Errorf(format string, args ...interface{}) error { + return log.Errorf(format, args...) +} + +func (*simpleLogger) Errore(err error) error { + return log.Errore(err) +} + +func (*simpleLogger) Fatal(args ...interface{}) error { + return log.Fatal(args[0].(string), args[1:]) +} + +func (*simpleLogger) Fatalf(format string, args ...interface{}) error { + return log.Fatalf(format, args...) +} + +func (*simpleLogger) Fatale(err error) error { + return log.Fatale(err) +} + +func (*simpleLogger) SetLevel(level log.LogLevel) { + log.SetLevel(level) + return +} + +func (*simpleLogger) SetPrintStackTrace(printStackTraceFlag bool) { + log.SetPrintStackTrace(printStackTraceFlag) + return +} diff --git a/go/base/utils.go b/go/base/utils.go index 99536f8..1476a21 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -14,7 +14,6 @@ import ( gosql "database/sql" "github.com/github/gh-ost/go/mysql" - "github.com/outbrain/golib/log" ) var ( @@ -86,7 +85,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, } if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) { - log.Infof("connection validated on %+v", connectionConfig.Key) + migrationContext.Log.Infof("connection validated on %+v", connectionConfig.Key) return version, nil } else if extraPort == 0 { return "", fmt.Errorf("Unexpected database port reported: %+v", port) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..6773c08 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -31,7 +31,7 @@ func acceptSignals(migrationContext *base.MigrationContext) { for sig := range c { switch sig { case syscall.SIGHUP: - log.Infof("Received SIGHUP. Reloading configuration") + migrationContext.Log.Infof("Received SIGHUP. Reloading configuration") if err := migrationContext.ReadConfigFile(); err != nil { log.Errore(err) } else { @@ -156,69 +156,69 @@ func main() { return } - log.SetLevel(log.ERROR) + migrationContext.Log.SetLevel(log.ERROR) if *verbose { - log.SetLevel(log.INFO) + migrationContext.Log.SetLevel(log.INFO) } if *debug { - log.SetLevel(log.DEBUG) + migrationContext.Log.SetLevel(log.DEBUG) } if *stack { - log.SetPrintStackTrace(*stack) + migrationContext.Log.SetPrintStackTrace(*stack) } if *quiet { // Override!! - log.SetLevel(log.ERROR) + migrationContext.Log.SetLevel(log.ERROR) } if migrationContext.DatabaseName == "" { - log.Fatalf("--database must be provided and database name must not be empty") + migrationContext.Log.Fatalf("--database must be provided and database name must not be empty") } if migrationContext.OriginalTableName == "" { - log.Fatalf("--table must be provided and table name must not be empty") + migrationContext.Log.Fatalf("--table must be provided and table name must not be empty") } if migrationContext.AlterStatement == "" { - log.Fatalf("--alter must be provided and statement must not be empty") + migrationContext.Log.Fatalf("--alter must be provided and statement must not be empty") } migrationContext.Noop = !(*executeFlag) if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica { - log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") } if migrationContext.AllowedRunningOnMaster && migrationContext.MigrateOnReplica { - log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive") } if migrationContext.MigrateOnReplica && migrationContext.TestOnReplica { - log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive") } if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR { - log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") + migrationContext.Log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") } if migrationContext.TestOnReplicaSkipReplicaStop { if !migrationContext.TestOnReplica { - log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") + migrationContext.Log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") } - log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") + migrationContext.Log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") } if migrationContext.CliMasterUser != "" && migrationContext.AssumeMasterHostname == "" { - log.Fatalf("--master-user requires --assume-master-host") + migrationContext.Log.Fatalf("--master-user requires --assume-master-host") } if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" { - log.Fatalf("--master-password requires --assume-master-host") + migrationContext.Log.Fatalf("--master-password requires --assume-master-host") } if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-ca requires --ssl") + migrationContext.Log.Fatalf("--ssl-ca requires --ssl") } if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-cert requires --ssl") + migrationContext.Log.Fatalf("--ssl-cert requires --ssl") } if migrationContext.TLSKey != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-key requires --ssl") + migrationContext.Log.Fatalf("--ssl-key requires --ssl") } if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS { - log.Fatalf("--ssl-allow-insecure requires --ssl") + migrationContext.Log.Fatalf("--ssl-allow-insecure requires --ssl") } if *replicationLagQuery != "" { - log.Warningf("--replication-lag-query is deprecated") + migrationContext.Log.Warningf("--replication-lag-query is deprecated") } switch *cutOver { @@ -227,19 +227,19 @@ func main() { case "two-step": migrationContext.CutOverType = base.CutOverTwoStep default: - log.Fatalf("Unknown cut-over: %s", *cutOver) + migrationContext.Log.Fatalf("Unknown cut-over: %s", *cutOver) } if err := migrationContext.ReadConfigFile(); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadThrottleControlReplicaKeys(*throttleControlReplicas); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadMaxLoad(*maxLoad); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadCriticalLoad(*criticalLoad); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if migrationContext.ServeSocketFile == "" { migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName) @@ -248,7 +248,7 @@ func main() { fmt.Println("Password:") bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) if err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } migrationContext.CliPassword = string(bytePassword) } @@ -262,13 +262,13 @@ func main() { migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() if err := migrationContext.SetupTLS(); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { - log.Errore(err) + migrationContext.Log.Errore(err) } if err := migrationContext.SetExponentialBackoffMaxInterval(*exponentialBackoffMaxInterval); err != nil { - log.Errore(err) + migrationContext.Log.Errore(err) } log.Infof("starting gh-ost %+v", AppVersion) @@ -278,7 +278,7 @@ func main() { err := migrator.Migrate() if err != nil { migrator.ExecOnFailureHook() - log.Fatale(err) + migrationContext.Log.Fatale(err) } fmt.Fprintf(os.Stdout, "# Done\n") } diff --git a/go/logic/applier.go b/go/logic/applier.go index 5fb795b..926023f 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -16,7 +16,6 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -99,7 +98,7 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.readTableColumns(); err != nil { return err } - log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion) + this.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion) return nil } @@ -110,13 +109,13 @@ func (this *Applier) validateAndReadTimeZone() error { return err } - log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone) + this.migrationContext.Log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone) return nil } // readTableColumns reads table columns on applier func (this *Applier) readTableColumns() (err error) { - log.Infof("Examining table structure on applier") + this.migrationContext.Log.Infof("Examining table structure on applier") this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) if err != nil { return err @@ -157,7 +156,7 @@ func (this *Applier) ValidateOrDropExistingTables() error { } } if len(this.migrationContext.GetOldTableName()) > mysql.MaxTableNameLength { - log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength) + this.migrationContext.Log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength) } if this.tableExists(this.migrationContext.GetOldTableName()) { @@ -175,14 +174,14 @@ func (this *Applier) CreateGhostTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Creating ghost table %s.%s", + this.migrationContext.Log.Infof("Creating ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Ghost table created") + this.migrationContext.Log.Infof("Ghost table created") return nil } @@ -193,15 +192,15 @@ func (this *Applier) AlterGhost() error { sql.EscapeName(this.migrationContext.GetGhostTableName()), this.migrationContext.AlterStatement, ) - log.Infof("Altering ghost table %s.%s", + this.migrationContext.Log.Infof("Altering ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) - log.Debugf("ALTER statement: %s", query) + this.migrationContext.Log.Debugf("ALTER statement: %s", query) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Ghost table altered") + this.migrationContext.Log.Infof("Ghost table altered") return nil } @@ -222,14 +221,14 @@ func (this *Applier) CreateChangelogTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetChangelogTableName()), ) - log.Infof("Creating changelog table %s.%s", + this.migrationContext.Log.Infof("Creating changelog table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetChangelogTableName()), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Changelog table created") + this.migrationContext.Log.Infof("Changelog table created") return nil } @@ -239,14 +238,14 @@ func (this *Applier) dropTable(tableName string) error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) - log.Infof("Dropping table %s.%s", + this.migrationContext.Log.Infof("Dropping table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Table dropped") + this.migrationContext.Log.Infof("Table dropped") return nil } @@ -313,7 +312,7 @@ func (this *Applier) InitiateHeartbeat() { if _, err := this.WriteChangelog("heartbeat", time.Now().Format(time.RFC3339Nano)); err != nil { numSuccessiveFailures++ if numSuccessiveFailures > this.migrationContext.MaxRetries() { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } else { numSuccessiveFailures = 0 @@ -348,14 +347,14 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) { } var result int64 if err := this.db.QueryRow(throttleQuery).Scan(&result); err != nil { - return 0, log.Errore(err) + return 0, this.migrationContext.Log.Errore(err) } return result, nil } // ReadMigrationMinValues returns the minimum values to be iterated on rowcopy func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { - log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) + this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) if err != nil { return err @@ -370,13 +369,13 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { return err } } - log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) + this.migrationContext.Log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) return err } // ReadMigrationMaxValues returns the maximum values to be iterated on rowcopy func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { - log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) + this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) if err != nil { return err @@ -391,7 +390,7 @@ func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { return err } } - log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) + this.migrationContext.Log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) return err } @@ -449,7 +448,7 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo return hasFurtherRange, nil } } - log.Debugf("Iteration complete: no further range to iterate") + this.migrationContext.Log.Debugf("Iteration complete: no further range to iterate") return hasFurtherRange, nil } @@ -507,7 +506,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } rowsAffected, _ = sqlResult.RowsAffected() duration = time.Since(startTime) - log.Debugf( + this.migrationContext.Log.Debugf( "Issued INSERT on range: [%s]..[%s]; iteration: %d; chunk-size: %d", this.migrationContext.MigrationIterationRangeMinValues, this.migrationContext.MigrationIterationRangeMaxValues, @@ -522,7 +521,7 @@ func (this *Applier) LockOriginalTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Locking %s.%s", + this.migrationContext.Log.Infof("Locking %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) @@ -530,18 +529,18 @@ func (this *Applier) LockOriginalTable() error { if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err } - log.Infof("Table locked") + this.migrationContext.Log.Infof("Table locked") return nil } // UnlockTables makes tea. No wait, it unlocks tables. func (this *Applier) UnlockTables() error { query := `unlock /* gh-ost */ tables` - log.Infof("Unlocking tables") + this.migrationContext.Log.Infof("Unlocking tables") if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err } - log.Infof("Tables unlocked") + this.migrationContext.Log.Infof("Tables unlocked") return nil } @@ -555,7 +554,7 @@ func (this *Applier) SwapTablesQuickAndBumpy() error { sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - log.Infof("Renaming original table") + this.migrationContext.Log.Infof("Renaming original table") this.migrationContext.RenameTablesStartTime = time.Now() if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err @@ -565,13 +564,13 @@ func (this *Applier) SwapTablesQuickAndBumpy() error { sql.EscapeName(this.migrationContext.GetGhostTableName()), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming ghost table") + this.migrationContext.Log.Infof("Renaming ghost table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } this.migrationContext.RenameTablesEndTime = time.Now() - log.Infof("Tables renamed") + this.migrationContext.Log.Infof("Tables renamed") return nil } @@ -590,7 +589,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming back both tables") + this.migrationContext.Log.Infof("Renaming back both tables") if _, err := sqlutils.ExecNoPrepare(this.db, query); err == nil { return nil } @@ -601,7 +600,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) - log.Infof("Renaming back to ghost table") + this.migrationContext.Log.Infof("Renaming back to ghost table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { renameError = err } @@ -611,11 +610,11 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming back to original table") + this.migrationContext.Log.Infof("Renaming back to original table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { renameError = err } - return log.Errore(renameError) + return this.migrationContext.Log.Errore(renameError) } // StopSlaveIOThread is applicable with --test-on-replica; it stops the IO thread, duh. @@ -623,44 +622,44 @@ func (this *Applier) RenameTablesRollback() (renameError error) { // and have them written to the binary log, so that we can then read them via streamer. func (this *Applier) StopSlaveIOThread() error { query := `stop /* gh-ost */ slave io_thread` - log.Infof("Stopping replication IO thread") + this.migrationContext.Log.Infof("Stopping replication IO thread") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Replication IO thread stopped") + this.migrationContext.Log.Infof("Replication IO thread stopped") return nil } // StartSlaveIOThread is applicable with --test-on-replica func (this *Applier) StartSlaveIOThread() error { query := `start /* gh-ost */ slave io_thread` - log.Infof("Starting replication IO thread") + this.migrationContext.Log.Infof("Starting replication IO thread") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Replication IO thread started") + this.migrationContext.Log.Infof("Replication IO thread started") return nil } // StartSlaveSQLThread is applicable with --test-on-replica func (this *Applier) StopSlaveSQLThread() error { query := `stop /* gh-ost */ slave sql_thread` - log.Infof("Verifying SQL thread is stopped") + this.migrationContext.Log.Infof("Verifying SQL thread is stopped") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("SQL thread stopped") + this.migrationContext.Log.Infof("SQL thread stopped") return nil } // StartSlaveSQLThread is applicable with --test-on-replica func (this *Applier) StartSlaveSQLThread() error { query := `start /* gh-ost */ slave sql_thread` - log.Infof("Verifying SQL thread is running") + this.migrationContext.Log.Infof("Verifying SQL thread is running") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("SQL thread started") + this.migrationContext.Log.Infof("SQL thread started") return nil } @@ -677,7 +676,7 @@ func (this *Applier) StopReplication() error { if err != nil { return err } - log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates) + this.migrationContext.Log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates) return nil } @@ -689,7 +688,7 @@ func (this *Applier) StartReplication() error { if err := this.StartSlaveSQLThread(); err != nil { return err } - log.Infof("Replication started") + this.migrationContext.Log.Infof("Replication started") return nil } @@ -703,7 +702,7 @@ func (this *Applier) ExpectUsedLock(sessionId int64) error { var result int64 query := `select is_used_lock(?)` lockName := this.GetSessionLockName(sessionId) - log.Infof("Checking session lock: %s", lockName) + this.migrationContext.Log.Infof("Checking session lock: %s", lockName) if err := this.db.QueryRow(query, lockName).Scan(&result); err != nil || result != sessionId { return fmt.Errorf("Session lock %s expected to be found but wasn't", lockName) } @@ -738,7 +737,7 @@ func (this *Applier) ExpectProcess(sessionId int64, stateHint, infoHint string) // DropAtomicCutOverSentryTableIfExists checks if the "old" table name // happens to be a cut-over magic table; if so, it drops it. func (this *Applier) DropAtomicCutOverSentryTableIfExists() error { - log.Infof("Looking for magic cut-over table") + this.migrationContext.Log.Infof("Looking for magic cut-over table") tableName := this.migrationContext.GetOldTableName() rowMap := this.showTableStatus(tableName) if rowMap == nil { @@ -748,7 +747,7 @@ func (this *Applier) DropAtomicCutOverSentryTableIfExists() error { if rowMap["Comment"].String != atomicCutOverMagicHint { return fmt.Errorf("Expected magic comment on %s, did not find it", tableName) } - log.Infof("Dropping magic cut-over table") + this.migrationContext.Log.Infof("Dropping magic cut-over table") return this.dropTable(tableName) } @@ -768,14 +767,14 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error { this.migrationContext.TableEngine, atomicCutOverMagicHint, ) - log.Infof("Creating magic cut-over table %s.%s", + this.migrationContext.Log.Infof("Creating magic cut-over table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Magic cut-over table created") + this.migrationContext.Log.Infof("Magic cut-over table created") return nil } @@ -804,7 +803,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke lockResult := 0 query := `select get_lock(?, 0)` lockName := this.GetSessionLockName(sessionId) - log.Infof("Grabbing voluntary lock: %s", lockName) + this.migrationContext.Log.Infof("Grabbing voluntary lock: %s", lockName) if err := tx.QueryRow(query, lockName).Scan(&lockResult); err != nil || lockResult != 1 { err := fmt.Errorf("Unable to acquire lock %s", lockName) tableLocked <- err @@ -812,7 +811,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke } tableLockTimeoutSeconds := this.migrationContext.CutOverLockTimeoutSeconds * 2 - log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds) + this.migrationContext.Log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds) query = fmt.Sprintf(`set session lock_wait_timeout:=%d`, tableLockTimeoutSeconds) if _, err := tx.Exec(query); err != nil { tableLocked <- err @@ -830,7 +829,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - log.Infof("Locking %s.%s, %s.%s", + this.migrationContext.Log.Infof("Locking %s.%s, %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.DatabaseName), @@ -841,7 +840,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke tableLocked <- err return err } - log.Infof("Tables locked") + this.migrationContext.Log.Infof("Tables locked") tableLocked <- nil // No error. // From this point on, we are committed to UNLOCK TABLES. No matter what happens, @@ -850,22 +849,22 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke // The cut-over phase will proceed to apply remaining backlog onto ghost table, // and issue RENAME. We wait here until told to proceed. <-okToUnlockTable - log.Infof("Will now proceed to drop magic table and unlock tables") + this.migrationContext.Log.Infof("Will now proceed to drop magic table and unlock tables") // The magic table is here because we locked it. And we are the only ones allowed to drop it. // And in fact, we will: - log.Infof("Dropping magic cut-over table") + this.migrationContext.Log.Infof("Dropping magic cut-over table") query = fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) if _, err := tx.Exec(query); err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) // We DO NOT return here because we must `UNLOCK TABLES`! } // Tables still locked - log.Infof("Releasing lock from %s.%s, %s.%s", + this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.DatabaseName), @@ -874,9 +873,9 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke query = `unlock tables` if _, err := tx.Exec(query); err != nil { tableUnlocked <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } - log.Infof("Tables unlocked") + this.migrationContext.Log.Infof("Tables unlocked") tableUnlocked <- nil return nil } @@ -898,7 +897,7 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed } sessionIdChan <- sessionId - log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds) + this.migrationContext.Log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds) query := fmt.Sprintf(`set session lock_wait_timeout:=%d`, this.migrationContext.CutOverLockTimeoutSeconds) if _, err := tx.Exec(query); err != nil { return err @@ -914,13 +913,13 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Issuing and expecting this to block: %s", query) + this.migrationContext.Log.Infof("Issuing and expecting this to block: %s", query) if _, err := tx.Exec(query); err != nil { tablesRenamed <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } tablesRenamed <- nil - log.Infof("Tables renamed") + this.migrationContext.Log.Infof("Tables renamed") return nil } @@ -1026,19 +1025,19 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) }() if err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // no error atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, int64(len(dmlEvents))) if this.migrationContext.CountTableRows { atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, totalDelta) } - log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents)) + this.migrationContext.Log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents)) return nil } func (this *Applier) Teardown() { - log.Debugf("Tearing down...") + this.migrationContext.Log.Debugf("Tearing down...") this.db.Close() this.singletonDB.Close() atomic.StoreInt64(&this.finishedMigrating, 1) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31184b0..bc10830 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -17,7 +17,6 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -69,7 +68,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.applyBinlogFormat(); err != nil { return err } - log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion) + this.migrationContext.Log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion) return nil } @@ -137,14 +136,14 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { switch column.Type { case sql.FloatColumnType: { - log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) + this.migrationContext.Log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) uniqueKeyIsValid = false } case sql.JSONColumnType: { // Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code // will remain in place to potentially handle the future case where JSON is supported in indexes. - log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) + this.migrationContext.Log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) uniqueKeyIsValid = false } } @@ -157,17 +156,17 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { if this.migrationContext.UniqueKey == nil { return fmt.Errorf("No shared unique key can be found after ALTER! Bailing out") } - log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name) + this.migrationContext.Log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name) if this.migrationContext.UniqueKey.HasNullable { if this.migrationContext.NullableUniqueKeyAllowed { - log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey) + this.migrationContext.Log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey) } else { return fmt.Errorf("Chosen key (%s) has nullable columns. Bailing out. To force this operation to continue, supply --allow-nullable-unique-key flag. Only do so if you are certain there are no actual NULL values in this key. As long as there aren't, migration should be fine. NULL values in columns of this key will corrupt migration's data", this.migrationContext.UniqueKey) } } this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) - log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) + this.migrationContext.Log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) // By fact that a non-empty unique key exists we also know the shared columns are non-empty // This additional step looks at which columns are unsigned. We could have merged this within @@ -250,19 +249,19 @@ func (this *Inspector) validateGrants() error { this.migrationContext.HasSuperPrivilege = foundSuper if foundAll { - log.Infof("User has ALL privileges") + this.migrationContext.Log.Infof("User has ALL privileges") return nil } if foundSuper && foundReplicationSlave && foundDBAll { - log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) return nil } if foundReplicationClient && foundReplicationSlave && foundDBAll { - log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) return nil } - log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll) - return log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll) + return this.migrationContext.Log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) } // restartReplication is required so that we are _certain_ the binlog format and @@ -270,7 +269,7 @@ func (this *Inspector) validateGrants() error { // It is entirely possible, for example, that the replication is using 'STATEMENT' // binlog format even as the variable says 'ROW' func (this *Inspector) restartReplication() error { - log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) masterKey, _ := mysql.GetMasterKeyFromSlaveStatus(this.connectionConfig) if masterKey == nil { @@ -289,7 +288,7 @@ func (this *Inspector) restartReplication() error { } time.Sleep(startSlavePostWaitMilliseconds) - log.Debugf("Replication restarted") + this.migrationContext.Log.Debugf("Replication restarted") return nil } @@ -309,7 +308,7 @@ func (this *Inspector) applyBinlogFormat() error { if err := this.restartReplication(); err != nil { return err } - log.Debugf("'ROW' binlog format applied") + this.migrationContext.Log.Debugf("'ROW' binlog format applied") return nil } // We already have RBR, no explicit switch @@ -347,7 +346,7 @@ func (this *Inspector) validateBinlogs() error { if countReplicas > 0 { return fmt.Errorf("%s:%d has %s binlog_format, but I'm too scared to change it to ROW because it has replicas. Bailing out", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) } - log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) + this.migrationContext.Log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) } query = `select @@global.binlog_row_image` if err := this.db.QueryRow(query).Scan(&this.migrationContext.OriginalBinlogRowImage); err != nil { @@ -359,7 +358,7 @@ func (this *Inspector) validateBinlogs() error { return fmt.Errorf("%s:%d has '%s' binlog_row_image, and only 'FULL' is supported. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogRowImage) } - log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -372,12 +371,12 @@ func (this *Inspector) validateLogSlaveUpdates() error { } if logSlaveUpdates { - log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } if this.migrationContext.IsTungsten { - log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -386,7 +385,7 @@ func (this *Inspector) validateLogSlaveUpdates() error { } if this.migrationContext.InspectorIsAlsoApplier() { - log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -413,17 +412,17 @@ func (this *Inspector) validateTable() error { return err } if !tableFound { - return log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine) - log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate) + this.migrationContext.Log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine) + this.migrationContext.Log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate) return nil } // validateTableForeignKeys makes sure no foreign keys exist on the migrated table func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error { if this.migrationContext.SkipForeignKeyChecks { - log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys") + this.migrationContext.Log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys") return nil } query := ` @@ -457,16 +456,16 @@ func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) erro return err } if numParentForeignKeys > 0 { - return log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } if numChildForeignKeys > 0 { if allowChildForeignKeys { - log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") + this.migrationContext.Log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") return nil } - return log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Debugf("Validated no foreign keys exist on table") + this.migrationContext.Log.Debugf("Validated no foreign keys exist on table") return nil } @@ -492,9 +491,9 @@ func (this *Inspector) validateTableTriggers() error { return err } if numTriggers > 0 { - return log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Debugf("Validated no triggers exist on table") + this.migrationContext.Log.Debugf("Validated no triggers exist on table") return nil } @@ -514,9 +513,9 @@ func (this *Inspector) estimateTableRowsViaExplain() error { return err } if !outputFound { - return log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate) + this.migrationContext.Log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate) return nil } @@ -525,7 +524,7 @@ func (this *Inspector) CountTableRows() error { atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 1) defer atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 0) - log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") + this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 @@ -535,7 +534,7 @@ func (this *Inspector) CountTableRows() error { atomic.StoreInt64(&this.migrationContext.RowsEstimate, rowsEstimate) this.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate - log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) + this.migrationContext.Log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) return nil } @@ -663,7 +662,7 @@ func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](* if err != nil { return uniqueKeys, err } - log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys) + this.migrationContext.Log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys) return uniqueKeys, nil } @@ -753,7 +752,7 @@ func (this *Inspector) readChangelogState(hint string) (string, error) { } func (this *Inspector) getMasterConnectionConfig() (applierConfig *mysql.ConnectionConfig, err error) { - log.Infof("Recursively searching for replication master") + this.migrationContext.Log.Infof("Recursively searching for replication master") visitedKeys := mysql.NewInstanceKeyMap() return mysql.GetMasterConnectionConfigSafe(this.connectionConfig, visitedKeys, this.migrationContext.AllowedMasterMaster) } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 86de5ac..4002de8 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -18,8 +18,6 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - - "github.com/outbrain/golib/log" ) type ChangelogState string @@ -216,7 +214,7 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er } changelogStateString := dmlEvent.NewColumnValues.StringColumn(3) changelogState := ReadChangelogState(changelogStateString) - log.Infof("Intercepted changelog state %s", changelogState) + this.migrationContext.Log.Infof("Intercepted changelog state %s", changelogState) switch changelogState { case GhostTableMigrated: { @@ -242,14 +240,14 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er return fmt.Errorf("Unknown changelog state: %+v", changelogState) } } - log.Infof("Handled changelog state %s", changelogState) + this.migrationContext.Log.Infof("Handled changelog state %s", changelogState) return nil } // listenOnPanicAbort aborts on abort request func (this *Migrator) listenOnPanicAbort() { err := <-this.migrationContext.PanicAbort - log.Fatale(err) + this.migrationContext.Log.Fatale(err) } // validateStatement validates the `alter` statement meets criteria. @@ -265,7 +263,7 @@ func (this *Migrator) validateStatement() (err error) { if !this.migrationContext.ApproveRenamedColumns { return fmt.Errorf("gh-ost believes the ALTER statement renames columns, as follows: %v; as precaution, you are asked to confirm gh-ost is correct, and provide with `--approve-renamed-columns`, and we're all happy. Or you can skip renamed columns via `--skip-renamed-columns`, in which case column data may be lost", this.parser.GetNonTrivialRenames()) } - log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames()) + this.migrationContext.Log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames()) } this.migrationContext.DroppedColumnsMap = this.parser.DroppedColumnsMap() return nil @@ -277,7 +275,7 @@ func (this *Migrator) countTableRows() (err error) { return nil } if this.migrationContext.Noop { - log.Debugf("Noop operation; not really counting table rows") + this.migrationContext.Log.Debugf("Noop operation; not really counting table rows") return nil } @@ -292,7 +290,7 @@ func (this *Migrator) countTableRows() (err error) { } if this.migrationContext.ConcurrentCountTableRows { - log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on") + this.migrationContext.Log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on") go countRowsFunc() // and we ignore errors, because this turns to be a background job return nil @@ -304,9 +302,9 @@ func (this *Migrator) createFlagFiles() (err error) { if this.migrationContext.PostponeCutOverFlagFile != "" { if !base.FileExists(this.migrationContext.PostponeCutOverFlagFile) { if err := base.TouchFile(this.migrationContext.PostponeCutOverFlagFile); err != nil { - return log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error()) + return this.migrationContext.Log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error()) } - log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile) + this.migrationContext.Log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile) } } return nil @@ -314,7 +312,7 @@ func (this *Migrator) createFlagFiles() (err error) { // Migrate executes the complete migration logic. This is *the* major gh-ost function. func (this *Migrator) Migrate() (err error) { - log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) this.migrationContext.StartTime = time.Now() if this.migrationContext.Hostname, err = os.Hostname(); err != nil { return err @@ -353,9 +351,9 @@ func (this *Migrator) Migrate() (err error) { } initialLag, _ := this.inspector.getReplicationLag() - log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) + this.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) <-this.ghostTableMigrated - log.Debugf("ghost table migrated") + this.migrationContext.Log.Debugf("ghost table migrated") // Yay! We now know the Ghost and Changelog tables are good to examine! // When running on replica, this means the replica has those tables. When running // on master this is always true, of course, and yet it also implies this knowledge @@ -393,9 +391,9 @@ func (this *Migrator) Migrate() (err error) { this.migrationContext.MarkRowCopyStartTime() go this.initiateStatus() - log.Debugf("Operating until row copy is complete") + this.migrationContext.Log.Debugf("Operating until row copy is complete") this.consumeRowCopyComplete() - log.Infof("Row copy complete") + this.migrationContext.Log.Infof("Row copy complete") if err := this.hooksExecutor.onRowCopyComplete(); err != nil { return err } @@ -421,7 +419,7 @@ func (this *Migrator) Migrate() (err error) { if err := this.hooksExecutor.onSuccess(); err != nil { return err } - log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -447,14 +445,14 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) { // and swap the tables. // The difference is that we will later swap the tables back. if err := this.hooksExecutor.onStartReplication(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if this.migrationContext.TestOnReplicaSkipReplicaStop { - log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.") + this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.") } else { - log.Debugf("testing on replica. Starting replication IO thread after cut-over failure") + this.migrationContext.Log.Debugf("testing on replica. Starting replication IO thread after cut-over failure") if err := this.retryOperation(this.applier.StartReplication); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } } @@ -465,16 +463,16 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) { // type (on replica? atomic? safe?) func (this *Migrator) cutOver() (err error) { if this.migrationContext.Noop { - log.Debugf("Noop operation; not really swapping tables") + this.migrationContext.Log.Debugf("Noop operation; not really swapping tables") return nil } this.migrationContext.MarkPointOfInterest() this.throttler.throttle(func() { - log.Debugf("throttling before swapping tables") + this.migrationContext.Log.Debugf("throttling before swapping tables") }) this.migrationContext.MarkPointOfInterest() - log.Debugf("checking for cut-over postpone") + this.migrationContext.Log.Debugf("checking for cut-over postpone") this.sleepWhileTrue( func() (bool, error) { if this.migrationContext.PostponeCutOverFlagFile == "" { @@ -499,7 +497,7 @@ func (this *Migrator) cutOver() (err error) { ) atomic.StoreInt64(&this.migrationContext.IsPostponingCutOver, 0) this.migrationContext.MarkPointOfInterest() - log.Debugf("checking for cut-over postpone: complete") + this.migrationContext.Log.Debugf("checking for cut-over postpone: complete") if this.migrationContext.TestOnReplica { // With `--test-on-replica` we stop replication thread, and then proceed to use @@ -510,9 +508,9 @@ func (this *Migrator) cutOver() (err error) { return err } if this.migrationContext.TestOnReplicaSkipReplicaStop { - log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") + this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") } else { - log.Debugf("testing on replica. Stopping replication IO thread") + this.migrationContext.Log.Debugf("testing on replica. Stopping replication IO thread") if err := this.retryOperation(this.applier.StopReplication); err != nil { return err } @@ -530,7 +528,7 @@ func (this *Migrator) cutOver() (err error) { this.handleCutOverResult(err) return err } - return log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType) + return this.migrationContext.Log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType) } // Inject the "AllEventsUpToLockProcessed" state hint, wait for it to appear in the binary logs, @@ -542,32 +540,32 @@ func (this *Migrator) waitForEventsUpToLock() (err error) { waitForEventsUpToLockStartTime := time.Now() allEventsUpToLockProcessedChallenge := fmt.Sprintf("%s:%d", string(AllEventsUpToLockProcessed), waitForEventsUpToLockStartTime.UnixNano()) - log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge) + this.migrationContext.Log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge) if _, err := this.applier.WriteChangelogState(allEventsUpToLockProcessedChallenge); err != nil { return err } - log.Infof("Waiting for events up to lock") + this.migrationContext.Log.Infof("Waiting for events up to lock") atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 1) for found := false; !found; { select { case <-timeout.C: { - return log.Errorf("Timeout while waiting for events up to lock") + return this.migrationContext.Log.Errorf("Timeout while waiting for events up to lock") } case state := <-this.allEventsUpToLockProcessed: { if state == allEventsUpToLockProcessedChallenge { - log.Infof("Waiting for events up to lock: got %s", state) + this.migrationContext.Log.Infof("Waiting for events up to lock: got %s", state) found = true } else { - log.Infof("Waiting for events up to lock: skipping %s", state) + this.migrationContext.Log.Infof("Waiting for events up to lock: skipping %s", state) } } } } waitForEventsUpToLockDuration := time.Since(waitForEventsUpToLockStartTime) - log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration) + this.migrationContext.Log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration) this.printStatus(ForcePrintStatusAndHintRule) return nil @@ -598,7 +596,7 @@ func (this *Migrator) cutOverTwoStep() (err error) { lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime) renameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.RenameTablesStartTime) - log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -620,18 +618,18 @@ func (this *Migrator) atomicCutOver() (err error) { tableUnlocked := make(chan error, 2) go func() { if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } }() if err := <-tableLocked; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } lockOriginalSessionId := <-lockOriginalSessionIdChan - log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId) + this.migrationContext.Log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId) // At this point we know the original table is locked. // We know any newly incoming DML on original table is blocked. if err := this.waitForEventsUpToLock(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // Step 2 @@ -649,7 +647,7 @@ func (this *Migrator) atomicCutOver() (err error) { } }() renameSessionId := <-renameSessionIdChan - log.Infof("Session renaming tables is %+v", renameSessionId) + this.migrationContext.Log.Infof("Session renaming tables is %+v", renameSessionId) waitForRename := func() error { if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 1 { @@ -666,13 +664,13 @@ func (this *Migrator) atomicCutOver() (err error) { return err } if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 0 { - log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)") + this.migrationContext.Log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)") } if err := this.applier.ExpectUsedLock(lockOriginalSessionId); err != nil { // Abort operation. Just make sure to drop the magic table. - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } - log.Infof("Connection holding lock on original table still exists") + this.migrationContext.Log.Infof("Connection holding lock on original table still exists") // Now that we've found the RENAME blocking, AND the locking connection still alive, // we know it is safe to proceed to release the lock @@ -681,16 +679,16 @@ func (this *Migrator) atomicCutOver() (err error) { // BAM! magic table dropped, original table lock is released // -> RENAME released -> queries on original are unblocked. if err := <-tableUnlocked; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if err := <-tablesRenamed; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } this.migrationContext.RenameTablesEndTime = time.Now() // ooh nice! We're actually truly and thankfully done lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime) - log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -736,7 +734,7 @@ func (this *Migrator) initiateInspector() (err error) { if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil { return err } - log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + this.migrationContext.Log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } else { // Forced master host. key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname) @@ -750,14 +748,14 @@ func (this *Migrator) initiateInspector() (err error) { if this.migrationContext.CliMasterPassword != "" { this.migrationContext.ApplierConnectionConfig.Password = this.migrationContext.CliMasterPassword } - log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + this.migrationContext.Log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } // validate configs if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica { if this.migrationContext.InspectorIsAlsoApplier() { return fmt.Errorf("Instructed to --test-on-replica or --migrate-on-replica, but the server we connect to doesn't seem to be a replica") } - log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", + this.migrationContext.Log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", *this.migrationContext.ApplierConnectionConfig.ImpliedKey, *this.migrationContext.InspectorConnectionConfig.ImpliedKey, ) this.migrationContext.ApplierConnectionConfig = this.migrationContext.InspectorConnectionConfig.Duplicate() @@ -995,12 +993,12 @@ func (this *Migrator) initiateStreaming() error { ) go func() { - log.Debugf("Beginning streaming") + this.migrationContext.Log.Debugf("Beginning streaming") err := this.eventsStreamer.StreamEvents(this.canStopStreaming) if err != nil { this.migrationContext.PanicAbort <- err } - log.Debugf("Done streaming") + this.migrationContext.Log.Debugf("Done streaming") }() go func() { @@ -1035,11 +1033,11 @@ func (this *Migrator) initiateThrottler() error { this.throttler = NewThrottler(this.migrationContext, this.applier, this.inspector) go this.throttler.initiateThrottlerCollection(this.firstThrottlingCollected) - log.Infof("Waiting for first throttle metrics to be collected") + this.migrationContext.Log.Infof("Waiting for first throttle metrics to be collected") <-this.firstThrottlingCollected // replication lag <-this.firstThrottlingCollected // HTTP status <-this.firstThrottlingCollected // other, general metrics - log.Infof("First throttle metrics collected") + this.migrationContext.Log.Infof("First throttle metrics collected") go this.throttler.initiateThrottlerChecks() return nil @@ -1054,16 +1052,16 @@ func (this *Migrator) initiateApplier() error { return err } if err := this.applier.CreateChangelogTable(); err != nil { - log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") + this.migrationContext.Log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") return err } if err := this.applier.CreateGhostTable(); err != nil { - log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") + this.migrationContext.Log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") return err } if err := this.applier.AlterGhost(); err != nil { - log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out") + this.migrationContext.Log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out") return err } @@ -1077,14 +1075,14 @@ func (this *Migrator) initiateApplier() error { func (this *Migrator) iterateChunks() error { terminateRowIteration := func(err error) error { this.rowCopyComplete <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if this.migrationContext.Noop { - log.Debugf("Noop operation; not really copying data") + this.migrationContext.Log.Debugf("Noop operation; not really copying data") return terminateRowIteration(nil) } if this.migrationContext.MigrationRangeMinValues == nil { - log.Debugf("No rows found in table. Rowcopy will be implicitly empty") + this.migrationContext.Log.Debugf("No rows found in table. Rowcopy will be implicitly empty") return terminateRowIteration(nil) } @@ -1152,7 +1150,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error { if eventStruct.writeFunc != nil { if err := this.retryOperation(*eventStruct.writeFunc); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } return nil @@ -1186,13 +1184,13 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { return this.applier.ApplyDMLEventQueries(dmlEvents) } if err := this.retryOperation(applyEventFunc); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if nonDmlStructToApply != nil { // We pulled DML events from the queue, and then we hit a non-DML event. Wait! // We need to handle it! if err := handleNonDMLEventStruct(nonDmlStructToApply); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } } @@ -1204,7 +1202,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { // Both event backlog and rowcopy events are polled; the backlog events have precedence. func (this *Migrator) executeWriteFuncs() error { if this.migrationContext.Noop { - log.Debugf("Noop operation; not really executing write funcs") + this.migrationContext.Log.Debugf("Noop operation; not really executing write funcs") return nil } for { @@ -1231,7 +1229,7 @@ func (this *Migrator) executeWriteFuncs() error { copyRowsStartTime := time.Now() // Retries are handled within the copyRowsFunc if err := copyRowsFunc(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if niceRatio := this.migrationContext.GetNiceRatio(); niceRatio > 0 { copyRowsDuration := time.Since(copyRowsStartTime) @@ -1244,7 +1242,7 @@ func (this *Migrator) executeWriteFuncs() error { { // Hmmmmm... nothing in the queue; no events, but also no row copy. // This is possible upon load. Let's just sleep it over. - log.Debugf("Getting nothing in the write queue. Sleeping...") + this.migrationContext.Log.Debugf("Getting nothing in the write queue. Sleeping...") time.Sleep(time.Second) } } @@ -1260,14 +1258,14 @@ func (this *Migrator) finalCleanup() error { if this.migrationContext.Noop { if createTableStatement, err := this.inspector.showCreateTable(this.migrationContext.GetGhostTableName()); err == nil { - log.Infof("New table structure follows") + this.migrationContext.Log.Infof("New table structure follows") fmt.Println(createTableStatement) } else { - log.Errore(err) + this.migrationContext.Log.Errore(err) } } if err := this.eventsStreamer.Close(); err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } if err := this.retryOperation(this.applier.DropChangelogTable); err != nil { @@ -1279,8 +1277,8 @@ func (this *Migrator) finalCleanup() error { } } else { if !this.migrationContext.Noop { - log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:") - log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName())) + this.migrationContext.Log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:") + this.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName())) } } if this.migrationContext.Noop { @@ -1296,22 +1294,22 @@ func (this *Migrator) teardown() { atomic.StoreInt64(&this.finishedMigrating, 1) if this.inspector != nil { - log.Infof("Tearing down inspector") + this.migrationContext.Log.Infof("Tearing down inspector") this.inspector.Teardown() } if this.applier != nil { - log.Infof("Tearing down applier") + this.migrationContext.Log.Infof("Tearing down applier") this.applier.Teardown() } if this.eventsStreamer != nil { - log.Infof("Tearing down streamer") + this.migrationContext.Log.Infof("Tearing down streamer") this.eventsStreamer.Teardown() } if this.throttler != nil { - log.Infof("Tearing down throttler") + this.migrationContext.Log.Infof("Tearing down throttler") this.throttler.Teardown() } } diff --git a/go/logic/server.go b/go/logic/server.go index 7af80b5..1606884 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -16,7 +16,6 @@ import ( "sync/atomic" "github.com/github/gh-ost/go/base" - "github.com/outbrain/golib/log" ) type printStatusFunc func(PrintStatusRule, io.Writer) @@ -49,12 +48,12 @@ func (this *Server) BindSocketFile() (err error) { if err != nil { return err } - log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile) + this.migrationContext.Log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile) return nil } func (this *Server) RemoveSocketFile() (err error) { - log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile) + this.migrationContext.Log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile) return os.Remove(this.migrationContext.ServeSocketFile) } @@ -66,7 +65,7 @@ func (this *Server) BindTCPPort() (err error) { if err != nil { return err } - log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort) + this.migrationContext.Log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort) return nil } @@ -76,7 +75,7 @@ func (this *Server) Serve() (err error) { for { conn, err := this.unixListener.Accept() if err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } go this.handleConnection(conn) } @@ -88,7 +87,7 @@ func (this *Server) Serve() (err error) { for { conn, err := this.tcpListener.Accept() if err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } go this.handleConnection(conn) } @@ -118,7 +117,7 @@ func (this *Server) onServerCommand(command string, writer *bufio.Writer) (err e } else { fmt.Fprintf(writer, "%s\n", err.Error()) } - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // applyServerCommand parses and executes commands by user diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 25d9f53..5f11fd0 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -16,7 +16,6 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -160,7 +159,7 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { if !foundMasterStatus { return fmt.Errorf("Got no results from SHOW MASTER STATUS. Bailing out") } - log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates) + this.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates) return nil } @@ -186,7 +185,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { return nil } - log.Infof("StreamEvents encountered unexpected error: %+v", err) + this.migrationContext.Log.Infof("StreamEvents encountered unexpected error: %+v", err) this.migrationContext.MarkPointOfInterest() time.Sleep(ReconnectStreamerSleepSeconds * time.Second) @@ -202,7 +201,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { // Reposition at same binlog file. lastAppliedRowsEventHint = this.binlogReader.LastAppliedRowsEventHint - log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint) + this.migrationContext.Log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint) if err := this.initBinlogReader(this.GetReconnectBinlogCoordinates()); err != nil { return err } @@ -213,7 +212,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { func (this *EventsStreamer) Close() (err error) { err = this.binlogReader.Close() - log.Infof("Closed streamer connection. err=%+v", err) + this.migrationContext.Log.Infof("Closed streamer connection. err=%+v", err) return err } diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 6f9f4bb..1fe413c 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -15,7 +15,6 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" ) var ( @@ -120,7 +119,7 @@ func parseChangelogHeartbeat(heartbeatValue string) (lag time.Duration, err erro // parseChangelogHeartbeat parses a string timestamp and deduces replication lag func (this *Throttler) parseChangelogHeartbeat(heartbeatValue string) (err error) { if lag, err := parseChangelogHeartbeat(heartbeatValue); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) return nil @@ -142,13 +141,13 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo // This means we will always get a good heartbeat value. // When running on replica, we should instead check the `SHOW SLAVE STATUS` output. if lag, err := mysql.GetReplicationLagFromSlaveStatus(this.inspector.informationSchemaDb); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) } } else { if heartbeatValue, err := this.inspector.readChangelogState("heartbeat"); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { this.parseChangelogHeartbeat(heartbeatValue) } @@ -330,7 +329,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error { hibernateDuration := time.Duration(this.migrationContext.CriticalLoadHibernateSeconds) * time.Second hibernateUntilTime := time.Now().Add(hibernateDuration) atomic.StoreInt64(&this.migrationContext.HibernateUntil, hibernateUntilTime.UnixNano()) - log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime) + this.migrationContext.Log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime) go func() { time.Sleep(hibernateDuration) this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(true, "leaving hibernation", base.LeavingHibernationThrottleReasonHint)) @@ -343,7 +342,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error { this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold) } if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 { - log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds) + this.migrationContext.Log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds) go func() { timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds)) <-timer.C @@ -461,6 +460,6 @@ func (this *Throttler) throttle(onThrottled func()) { } func (this *Throttler) Teardown() { - log.Debugf("Tearing down...") + this.migrationContext.Log.Debugf("Tearing down...") atomic.StoreInt64(&this.finishedMigrating, 1) } From 3ca5f89fa52aa76d4ef0725ddad1eb85e13dc5f7 Mon Sep 17 00:00:00 2001 From: Abeyu M Date: Tue, 8 Oct 2019 13:49:15 -0400 Subject: [PATCH 052/103] add migrationcontext to gomysql_reader --- go/binlog/gomysql_reader.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 41d1ead..bc80cb5 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -13,13 +13,13 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" gomysql "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "golang.org/x/net/context" ) type GoMySQLReader struct { + migrationContext *base.MigrationContext connectionConfig *mysql.ConnectionConfig binlogSyncer *replication.BinlogSyncer binlogStreamer *replication.BinlogStreamer @@ -30,6 +30,7 @@ type GoMySQLReader struct { func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *GoMySQLReader, err error) { binlogReader = &GoMySQLReader{ + migrationContext: migrationContext, connectionConfig: migrationContext.InspectorConnectionConfig, currentCoordinates: mysql.BinlogCoordinates{}, currentCoordinatesMutex: &sync.Mutex{}, @@ -57,11 +58,11 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go // ConnectBinlogStreamer func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordinates) (err error) { if coordinates.IsEmpty() { - return log.Errorf("Empty coordinates at ConnectBinlogStreamer()") + return this.migrationContext.Log.Errorf("Empty coordinates at ConnectBinlogStreamer()") } this.currentCoordinates = coordinates - log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) + this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) // Start sync with specified binlog file and position this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{this.currentCoordinates.LogFile, uint32(this.currentCoordinates.LogPos)}) @@ -78,7 +79,7 @@ func (this *GoMySQLReader) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinate // StreamEvents func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error { if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) { - log.Debugf("Skipping handled query at %+v", this.currentCoordinates) + this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates) return nil } @@ -147,14 +148,14 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates.LogFile = string(rotateEvent.NextLogName) }() - log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName) + this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName) } else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok { if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil { return err } } } - log.Debugf("done streaming events") + this.migrationContext.Log.Debugf("done streaming events") return nil } From d0ce7c014ee7fd4f1692574527506c0f8ce9b258 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 23 Oct 2019 10:18:56 +0300 Subject: [PATCH 053/103] truncate lag digits --- go/logic/migrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index b0bec68..b1a238f 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -960,7 +960,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+vs, State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied), len(this.applyEventsQueue), cap(this.applyEventsQueue), From 991cdc51f0f882a917d3b559945595f485a1d697 Mon Sep 17 00:00:00 2001 From: Mohit Maroliya <35539313+mohitm15@users.noreply.github.com> Date: Thu, 31 Oct 2019 19:28:54 +0530 Subject: [PATCH 054/103] Update command-line-flags.md --- doc/command-line-flags.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 61ffdbc..ee0051c 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -18,7 +18,7 @@ If, for some reason, you do not wish `gh-ost` to connect to a replica, you may c ### approve-renamed-columns -When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try an associate the old column name with new column name. Otherwise the new structure may also look like some column was dropped and another was added. +When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise the new structure may also look like some column was dropped and another was added. `gh-ost` will print out what it thinks the _rename_ implied, but will not issue the migration unless you provide with `--approve-renamed-columns`. @@ -161,7 +161,7 @@ List of metrics and threshold values; topping the threshold of any will cause th ### migrate-on-replica -Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but other issue no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. +Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but other issue will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. ### postpone-cut-over-flag-file From 859800079bfb73a3e86ee778d51bfcdc1095b739 Mon Sep 17 00:00:00 2001 From: Mohit Maroliya <35539313+mohitm15@users.noreply.github.com> Date: Thu, 31 Oct 2019 20:13:54 +0530 Subject: [PATCH 055/103] Update doc/command-line-flags.md Thank you for the suggestion @shlomi-noach Co-Authored-By: Shlomi Noach --- doc/command-line-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index ee0051c..629e9f9 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -161,7 +161,7 @@ List of metrics and threshold values; topping the threshold of any will cause th ### migrate-on-replica -Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but other issue will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. +Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but otherwise will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. ### postpone-cut-over-flag-file From 19215b268834c8d909f866f7561a89ebad0900b6 Mon Sep 17 00:00:00 2001 From: Tobias Johansson Date: Tue, 10 Dec 2019 13:57:18 +0100 Subject: [PATCH 056/103] Update rds docs --- doc/rds.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/rds.md b/doc/rds.md index 37d1d80..da59abb 100644 --- a/doc/rds.md +++ b/doc/rds.md @@ -26,6 +26,14 @@ If you use `pt-table-checksum` as a part of your data integrity checks, you migh This tool requires binlog_format=STATEMENT, but the current binlog_format is set to ROW and an error occurred while attempting to change it. If running MySQL 5.1.29 or newer, setting binlog_format requires the SUPER privilege. You will need to manually set binlog_format to 'STATEMENT' before running this tool. ``` +#### Binlog filtering + +In Aurora, the [binlog filtering feature][aws_replication_docs_bin_log_filtering] is enabled by default. This becomes an issue when gh-ost tries to do the cut-over, because gh-ost waits for an entry in the binlog to proceed but this entry will never end up in the binlog because it gets filtered out by the binlog filtering feature. +You need to turn this feature off during the migration process. +Set the `aurora_enable_repl_bin_log_filtering` parameter to 0 in the Parameter Group for your cluster. +When the migration is done, set it back to 1 (default). + + #### Preflight checklist Before trying to run any `gh-ost` migrations you will want to confirm the following: @@ -35,6 +43,7 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow - [ ] Executing `SHOW SLAVE STATUS\G` on your replica cluster displays the correct master host, binlog position, etc. - [ ] Database backup retention is greater than 1 day to enable binlogs - [ ] You have setup [`hooks`][ghost_hooks] to issue RDS procedures for stopping and starting replication. (see [github/gh-ost#163][ghost_rds_issue_tracking] for examples) +- [ ] The parameter `aurora_enable_repl_bin_log_filtering` is set to 0 [new_issue]: https://github.com/github/gh-ost/issues/new [assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr @@ -43,3 +52,4 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow [percona_toolkit_patch]: https://github.com/jacobbednarz/percona-toolkit/commit/0271ba6a094da446a5e5bb8d99b5c26f1777f2b9 [ghost_hooks]: https://github.com/github/gh-ost/blob/master/doc/hooks.md [ghost_rds_issue_tracking]: https://github.com/github/gh-ost/issues/163 +[aws_replication_docs_bin_log_filtering]: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Replication.html#AuroraMySQL.Replication.Performance \ No newline at end of file From 2eb141010bbc63da9aed593304026affa8b6606f Mon Sep 17 00:00:00 2001 From: Yaser Azfar Date: Fri, 27 Dec 2019 16:54:43 +1300 Subject: [PATCH 057/103] adds error checking for an err variable that was left unchecked --- go/sql/builder.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/sql/builder.go b/go/sql/builder.go index 2c5a7ae..4b019bc 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -492,6 +492,9 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol } setClause, err := BuildSetPreparedClause(mappedSharedColumns) + if err != nil { + return "", sharedArgs, uniqueKeyArgs, err + } equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names()) result = fmt.Sprintf(` From 9af130d93127fb2b3f8109ff764c27bef9905bd1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:46:35 +0200 Subject: [PATCH 058/103] Replication tests on 8.0 --- script/cibuild-gh-ost-replica-tests | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index ab8c689..447e5e0 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -61,6 +61,9 @@ test_mysql_version() { echo "Building..." . script/build # Test all versions: +find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do + echo "found MySQL version: $mysql_version" +done find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do test_mysql_version "$mysql_version" done From 14674274dc9858ef8fa027db4fd366aacd613c0d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:48:55 +0200 Subject: [PATCH 059/103] Experiment with https://github.com/github/gh-ost-ci-env/pull/8 --- script/cibuild-gh-ost-replica-tests | 1 + 1 file changed, 1 insertion(+) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 447e5e0..942e95a 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -16,6 +16,7 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git + git checkout real-tar-gz fi test_mysql_version() { From 919647cbbc32a7ec5514d64396cbb9349694d35a Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:50:34 +0200 Subject: [PATCH 060/103] removing travis --- .travis.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c1c1891..0000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -# http://docs.travis-ci.com/user/languages/go/ -language: go - -go: - - "1.12.x" - -os: - - linux - -services: - - mysql - -env: -- MYSQL_USER=root -- CURRENT_CI_ENV=travis - -addons: - apt: - packages: - - git - - numactl - - libaio1 - -before_install: - - mysql -e 'CREATE DATABASE IF NOT EXISTS test;' - -install: true - -script: - - script/cibuild - -notifications: - email: false From 3528d675502f2c1cefd4cee1f6caa77042faf21d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:51:40 +0200 Subject: [PATCH 061/103] fetch the correct branch. Oh and removing travis --- script/cibuild-gh-ost-replica-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 942e95a..c0b53d9 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -6,7 +6,7 @@ whoami # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git real-tar-gz | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) @@ -16,6 +16,7 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git + git fetch origin real-tar-gz git checkout real-tar-gz fi From 9201b9bc04b41dbfadd9da7406dd3549f3fe4ad7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:54:07 +0200 Subject: [PATCH 062/103] fetch and checkout from git path --- script/cibuild-gh-ost-replica-tests | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c0b53d9..813fdd8 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -16,8 +16,11 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git - git fetch origin real-tar-gz - git checkout real-tar-gz + ( + cd gh-ost-ci-env + git fetch origin real-tar-gz + git checkout real-tar-gz + ) fi test_mysql_version() { From b0f487ea613445c6d5a444b9176b3e93e333d7ea Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:07:14 +0200 Subject: [PATCH 063/103] create user --- script/cibuild-gh-ost-replica-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 813fdd8..c0c925c 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -55,7 +55,8 @@ test_mysql_version() { export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" - gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%' identified by 'gh-ost'" + gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" + gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" echo "### Running gh-ost tests for $mysql_version" ./localtests/test.sh -b bin/gh-ost From a6d0d5ca26fa9291721afc73f61792933e9565d7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:12:29 +0200 Subject: [PATCH 064/103] not null for unique key test --- localtests/swap-uk-uk/create.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localtests/swap-uk-uk/create.sql b/localtests/swap-uk-uk/create.sql index 5fcbf32..30c1542 100644 --- a/localtests/swap-uk-uk/create.sql +++ b/localtests/swap-uk-uk/create.sql @@ -1,8 +1,8 @@ drop table if exists gh_ost_test; create table gh_ost_test ( - id bigint, + id bigint not null, i int not null, - ts timestamp(6), + ts timestamp(6) not null, unique key id_uidx(id), unique key its_uidx(i, ts) ) ; From 4f827a271e5094a3b59c195dd9e4af54a1fb857d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:42:36 +0200 Subject: [PATCH 065/103] 8.0 is on master now --- script/cibuild-gh-ost-replica-tests | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c0c925c..8b518e1 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -6,7 +6,7 @@ whoami # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git real-tar-gz | cut -f1) +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) @@ -16,11 +16,6 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git - ( - cd gh-ost-ci-env - git fetch origin real-tar-gz - git checkout real-tar-gz - ) fi test_mysql_version() { From c30e5862f7b46170c1a8d444b7439e02bd16a752 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:43:08 +0200 Subject: [PATCH 066/103] 8.0 is on master now --- script/cibuild-gh-ost-replica-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 8b518e1..3de9e05 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -6,7 +6,7 @@ whoami # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git | cut -f1) +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) From 05b4ac08b22991c9f6e31a6f04acfe79a3034321 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 11:43:19 +0200 Subject: [PATCH 067/103] Release v1.0.49 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 56d0dad..f133985 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.48 +1.0.49 From 8bdf2d2dbd698a71c40e20d3885f92aca85a6805 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 16:11:19 +0200 Subject: [PATCH 068/103] Adding script/dock: simplifying testing/packaging --- script/dock | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 script/dock diff --git a/script/dock b/script/dock new file mode 100755 index 0000000..d17ae5d --- /dev/null +++ b/script/dock @@ -0,0 +1,30 @@ +#!/bin/bash + +# Usage: +# dock [arg] +# dock test: build gh-ost & run unit and integration tests +# docker alpine: build and run gh-ost on alpine linux +# docker pkg [target-path]: build gh-ost release packages and copy to target path (default path: /tmp/gh-ost-release) + +command="$1" + +case "$command" in + "test") + docker_target="gh-ost-test" + docker build . -f Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest" + ;; + "alpine") + docker_target="gh-ost-alpine" + docker build . -f Dockerfile -t "${docker_target}" && docker run --rm -it -p 3000:3000 "${docker_target}:latest" + ;; + "pkg") + packages_path="${2:-/tmp/gh-ost-release}" + docker_target="gh-ost-packaging" + docker build . -f Dockerfile.packaging -t "${docker_target}" && docker run --rm -it -v "${packages_path}:/tmp/pkg" "${docker_target}:latest" bash -c 'find /tmp/gh-ost-release/ -maxdepth 1 -type f | xargs cp -t /tmp/pkg' + echo "packages generated on ${packages_path}:" + ls -l "${packages_path}" + ;; + *) + >&2 echo "Usage: dock dock [arg]" + exit 1 +esac From 40953fe82f9db405cbd45f04ee4f473b2f0511ee Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 16:11:34 +0200 Subject: [PATCH 069/103] Adding script/dock: simplifying testing/packaging --- build.sh | 4 ++-- script/dock | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index b7f3a54..46db9c2 100755 --- a/build.sh +++ b/build.sh @@ -61,11 +61,11 @@ main() { mkdir -p ${buildpath} rm -rf ${buildpath:?}/* - build macOS osx darwin amd64 build GNU/Linux linux linux amd64 + # build macOS osx darwin amd64 echo "Binaries found in:" - ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz + find $buildpath/gh-ost* -type f -maxdepth 1 } main "$@" diff --git a/script/dock b/script/dock index d17ae5d..486061d 100755 --- a/script/dock +++ b/script/dock @@ -1,9 +1,8 @@ #!/bin/bash # Usage: -# dock [arg] +# dock [arg] # dock test: build gh-ost & run unit and integration tests -# docker alpine: build and run gh-ost on alpine linux # docker pkg [target-path]: build gh-ost release packages and copy to target path (default path: /tmp/gh-ost-release) command="$1" @@ -13,10 +12,6 @@ case "$command" in docker_target="gh-ost-test" docker build . -f Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest" ;; - "alpine") - docker_target="gh-ost-alpine" - docker build . -f Dockerfile -t "${docker_target}" && docker run --rm -it -p 3000:3000 "${docker_target}:latest" - ;; "pkg") packages_path="${2:-/tmp/gh-ost-release}" docker_target="gh-ost-packaging" From f6ea377a4b372e2cf11645dbffd326d5a9f57a38 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 16:11:50 +0200 Subject: [PATCH 070/103] Dockerfile.packaging --- Dockerfile.packaging | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Dockerfile.packaging diff --git a/Dockerfile.packaging b/Dockerfile.packaging new file mode 100644 index 0000000..6aa2ace --- /dev/null +++ b/Dockerfile.packaging @@ -0,0 +1,36 @@ +# Requires Docker 17.09 or later (multi stage builds) +# +# Orchestrator will look for a configuration file at /etc/orchestrator.conf.json. +# It will listen on port 3000. +# If not present a minimal configuration will be generated using the following environment variables: +# +# Default variables which can be used are: +# +# ORC_TOPOLOGY_USER (default: orchestrator): username used by orchestrator to login to MySQL when polling/discovering +# ORC_TOPOLOGY_PASSWORD (default: orchestrator): password needed to login to MySQL when polling/discovering +# ORC_DB_HOST (default: orchestrator): orchestrator backend MySQL host +# ORC_DB_PORT (default: 3306): port used by orchestrator backend MySQL server +# ORC_DB_NAME (default: orchestrator): database named used by orchestrator backend MySQL server +# ORC_USER (default: orc_server_user): username used to login to orchestrator backend MySQL server +# ORC_PASSWORD (default: orc_server_password): password used to login to orchestrator backend MySQL server + +FROM golang:1.12.6 + +RUN apt-get update +RUN apt-get install -y ruby ruby-dev rubygems build-essential +RUN gem install --no-ri --no-rdoc fpm +ENV GOPATH=/tmp/go + +RUN apt-get install -y curl +RUN apt-get install -y rsync +RUN apt-get install -y gcc +RUN apt-get install -y g++ +RUN apt-get install -y bash +RUN apt-get install -y git +RUN apt-get install -y tar +RUN apt-get install -y rpm + +RUN mkdir -p $GOPATH/src/github.com/github/gh-ost +WORKDIR $GOPATH/src/github.com/github/gh-ost +COPY . . +RUN bash build.sh From 85a285a9e4ecfd63f817c04f998b572a2fd18e5d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 10 Feb 2020 08:59:44 +0200 Subject: [PATCH 071/103] cleanup --- Dockerfile.packaging | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 6aa2ace..214c70c 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,18 +1,4 @@ -# Requires Docker 17.09 or later (multi stage builds) # -# Orchestrator will look for a configuration file at /etc/orchestrator.conf.json. -# It will listen on port 3000. -# If not present a minimal configuration will be generated using the following environment variables: -# -# Default variables which can be used are: -# -# ORC_TOPOLOGY_USER (default: orchestrator): username used by orchestrator to login to MySQL when polling/discovering -# ORC_TOPOLOGY_PASSWORD (default: orchestrator): password needed to login to MySQL when polling/discovering -# ORC_DB_HOST (default: orchestrator): orchestrator backend MySQL host -# ORC_DB_PORT (default: 3306): port used by orchestrator backend MySQL server -# ORC_DB_NAME (default: orchestrator): database named used by orchestrator backend MySQL server -# ORC_USER (default: orc_server_user): username used to login to orchestrator backend MySQL server -# ORC_PASSWORD (default: orc_server_password): password used to login to orchestrator backend MySQL server FROM golang:1.12.6 From 08f171790b01810f83e5972a0d0c1f4cdbc74276 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 13 Feb 2020 13:45:39 +0200 Subject: [PATCH 072/103] Update go-sql-driver to latest --- .../mysql/{ => .github}/CONTRIBUTING.md | 0 .../go-sql-driver/mysql/.travis.yml | 3 +- vendor/github.com/go-sql-driver/mysql/AUTHORS | 5 + .../go-sql-driver/mysql/CHANGELOG.md | 39 ++++ .../github.com/go-sql-driver/mysql/README.md | 26 ++- .../go-sql-driver/mysql/appengine.go | 25 --- .../go-sql-driver/mysql/conncheck.go | 43 ++--- .../go-sql-driver/mysql/conncheck_dummy.go | 4 +- .../go-sql-driver/mysql/conncheck_test.go | 2 +- .../go-sql-driver/mysql/connection.go | 13 +- .../go-sql-driver/mysql/connection_test.go | 28 +++ .../go-sql-driver/mysql/connector.go | 13 +- .../go-sql-driver/mysql/connector_test.go | 30 +++ .../github.com/go-sql-driver/mysql/driver.go | 22 +++ .../go-sql-driver/mysql/driver_go110.go | 37 ---- .../go-sql-driver/mysql/driver_go110_test.go | 137 -------------- .../go-sql-driver/mysql/driver_test.go | 171 +++++++++++++++++- vendor/github.com/go-sql-driver/mysql/dsn.go | 166 +++++------------ .../go-sql-driver/mysql/dsn_test.go | 36 ++-- vendor/github.com/go-sql-driver/mysql/go.mod | 3 + .../go-sql-driver/mysql/nulltime.go | 50 +++++ .../go-sql-driver/mysql/nulltime_go113.go | 31 ++++ .../go-sql-driver/mysql/nulltime_legacy.go | 34 ++++ .../go-sql-driver/mysql/nulltime_test.go | 62 +++++++ .../github.com/go-sql-driver/mysql/packets.go | 2 +- .../github.com/go-sql-driver/mysql/utils.go | 54 ------ .../go-sql-driver/mysql/utils_test.go | 41 ----- 27 files changed, 601 insertions(+), 476 deletions(-) rename vendor/github.com/go-sql-driver/mysql/{ => .github}/CONTRIBUTING.md (100%) delete mode 100644 vendor/github.com/go-sql-driver/mysql/appengine.go create mode 100644 vendor/github.com/go-sql-driver/mysql/connector_test.go delete mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110.go delete mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/go.mod create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_go113.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_test.go diff --git a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/go-sql-driver/mysql/.github/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md rename to vendor/github.com/go-sql-driver/mysql/.github/CONTRIBUTING.md diff --git a/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/go-sql-driver/mysql/.travis.yml index eae311b..56fcf25 100644 --- a/vendor/github.com/go-sql-driver/mysql/.travis.yml +++ b/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -1,10 +1,10 @@ sudo: false language: go go: - - 1.9.x - 1.10.x - 1.11.x - 1.12.x + - 1.13.x - master before_install: @@ -105,6 +105,7 @@ matrix: homebrew: packages: - mysql + update: true go: 1.12.x before_install: - go get golang.org/x/tools/cmd/cover diff --git a/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/go-sql-driver/mysql/AUTHORS index bfe74c4..0896ba1 100644 --- a/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -13,6 +13,7 @@ Aaron Hopkins Achille Roussel +Alex Snast Alexey Palazhchenko Andrew Reid Arne Hormann @@ -44,6 +45,7 @@ James Harr Jeff Hodges Jeffrey Charles Jerome Meyer +Jiajia Zhong Jian Zhen Joshua Prunier Julien Lefevre @@ -62,6 +64,7 @@ Lucas Liu Luke Scott Maciej Zimnoch Michael Woolnough +Nathanial Murphy Nicola Peduzzi Olivier Mengué oscarzhao @@ -81,6 +84,7 @@ Steven Hartland Thomas Wodarek Tim Ruffles Tom Jenkinson +Vladimir Kovpak Xiangyu Hu Xiaobing Jiang Xiuming Chen @@ -90,6 +94,7 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +DigitalOcean Inc. Facebook Inc. GitHub Inc. Google Inc. diff --git a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md index 2d87d74..9cb97b3 100644 --- a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md +++ b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -1,3 +1,42 @@ +## Version 1.5 (2020-01-07) + +Changes: + + - Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017) + - Improve buffer handling (#890) + - Document potentially insecure TLS configs (#901) + - Use a double-buffering scheme to prevent data races (#943) + - Pass uint64 values without converting them to string (#838, #955) + - Update collations and make utf8mb4 default (#877, #1054) + - Make NullTime compatible with sql.NullTime in Go 1.13+ (#995) + - Removed CloudSQL support (#993, #1007) + - Add Go Module support (#1003) + +New Features: + + - Implement support of optional TLS (#900) + - Check connection liveness (#934, #964, #997, #1048, #1051, #1052) + - Implement Connector Interface (#941, #958, #1020, #1035) + +Bugfixes: + + - Mark connections as bad on error during ping (#875) + - Mark connections as bad on error during dial (#867) + - Fix connection leak caused by rapid context cancellation (#1024) + - Mark connections as bad on error during Conn.Prepare (#1030) + + +## Version 1.4.1 (2018-11-14) + +Bugfixes: + + - Fix TIME format for binary columns (#818) + - Fix handling of empty auth plugin names (#835) + - Fix caching_sha2_password with empty password (#826) + - Fix canceled context broke mysqlConn (#862) + - Fix OldAuthSwitchRequest support (#870) + - Fix Auth Response packet for cleartext password (#887) + ## Version 1.4 (2018-06-03) Changes: diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md index c6adf1d..d2627a4 100644 --- a/vendor/github.com/go-sql-driver/mysql/README.md +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.9 or higher. We aim to support the 3 latest versions of Go. + * Go 1.10 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- @@ -166,6 +166,17 @@ Sets the charset used for client-server interaction (`"SET NAMES "`). If Usage of the `charset` parameter is discouraged because it issues additional queries to the server. Unless you need the fallback behavior, please use `collation` instead. +##### `checkConnLiveness` + +``` +Type: bool +Valid Values: true, false +Default: true +``` + +On supported platforms connections retrieved from the connection pool are checked for liveness before using them. If the check fails, the respective connection is marked as bad and the query retried with another connection. +`checkConnLiveness=false` disables this liveness check of connections. + ##### `collation` ``` @@ -396,14 +407,9 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine (First Generation MySQL Server): +Google Cloud SQL on App Engine: ``` -user@cloudsql(project-id:instance-name)/dbname -``` - -Google Cloud SQL on App Engine (Second Generation MySQL Server): -``` -user@cloudsql(project-id:regionname:instance-name)/dbname +user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname ``` TCP using default port (3306) on localhost: @@ -457,13 +463,13 @@ Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-d ### Unicode support -Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. +Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. -See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. +See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. diff --git a/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/go-sql-driver/mysql/appengine.go deleted file mode 100644 index 914e662..0000000 --- a/vendor/github.com/go-sql-driver/mysql/appengine.go +++ /dev/null @@ -1,25 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build appengine - -package mysql - -import ( - "context" - "net" - - "google.golang.org/appengine/cloudsql" -) - -func init() { - RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { - // XXX: the cloudsql driver still does not export a Context-aware dialer. - return cloudsql.Dial(instance) - }) -} diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/go-sql-driver/mysql/conncheck.go index cc47aa5..024eb28 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build !windows,!appengine +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql @@ -19,35 +19,36 @@ import ( var errUnexpectedRead = errors.New("unexpected read from socket") -func connCheck(c net.Conn) error { - var ( - n int - err error - buff [1]byte - ) +func connCheck(conn net.Conn) error { + var sysErr error - sconn, ok := c.(syscall.Conn) + sysConn, ok := conn.(syscall.Conn) if !ok { return nil } - rc, err := sconn.SyscallConn() + rawConn, err := sysConn.SyscallConn() if err != nil { return err } - rerr := rc.Read(func(fd uintptr) bool { - n, err = syscall.Read(int(fd), buff[:]) + + err = rawConn.Read(func(fd uintptr) bool { + var buf [1]byte + n, err := syscall.Read(int(fd), buf[:]) + switch { + case n == 0 && err == nil: + sysErr = io.EOF + case n > 0: + sysErr = errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + sysErr = nil + default: + sysErr = err + } return true }) - switch { - case rerr != nil: - return rerr - case n == 0 && err == nil: - return io.EOF - case n > 0: - return errUnexpectedRead - case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: - return nil - default: + if err != nil { return err } + + return sysErr } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go index fd01f64..ea7fb60 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -6,12 +6,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build windows appengine +// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos package mysql import "net" -func connCheck(c net.Conn) error { +func connCheck(conn net.Conn) error { return nil } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go index b7234b0..5399551 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build go1.10,!windows +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index 565a548..b07cd76 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -12,6 +12,7 @@ import ( "context" "database/sql" "database/sql/driver" + "encoding/json" "io" "net" "strconv" @@ -154,7 +155,9 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, mc.markBadConn(err) + // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. + errLog.Print(err) + return nil, driver.ErrBadConn } stmt := &mysqlStmt{ @@ -269,6 +272,14 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin } buf = append(buf, '\'') } + case json.RawMessage: + buf = append(buf, '\'') + if mc.status&statusNoBackslashEscapes == 0 { + buf = escapeBytesBackslash(buf, v) + } else { + buf = escapeBytesQuotes(buf, v) + } + buf = append(buf, '\'') case []byte: if v == nil { buf = append(buf, "NULL"...) diff --git a/vendor/github.com/go-sql-driver/mysql/connection_test.go b/vendor/github.com/go-sql-driver/mysql/connection_test.go index 19c17ff..a6d6773 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection_test.go +++ b/vendor/github.com/go-sql-driver/mysql/connection_test.go @@ -11,6 +11,7 @@ package mysql import ( "context" "database/sql/driver" + "encoding/json" "errors" "net" "testing" @@ -36,6 +37,33 @@ func TestInterpolateParams(t *testing.T) { } } +func TestInterpolateParamsJSONRawMessage(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + buf, err := json.Marshal(struct { + Value int `json:"value"` + }{Value: 42}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + q, err := mc.interpolateParams("SELECT ?", []driver.Value{json.RawMessage(buf)}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT '{\"value\":42}'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { mc := &mysqlConn{ buf: newBuffer(nil), diff --git a/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/go-sql-driver/mysql/connector.go index 5aaaba4..d567b4e 100644 --- a/vendor/github.com/go-sql-driver/mysql/connector.go +++ b/vendor/github.com/go-sql-driver/mysql/connector.go @@ -37,17 +37,19 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { dial, ok := dials[mc.cfg.Net] dialsLock.RUnlock() if ok { - mc.netConn, err = dial(ctx, mc.cfg.Addr) + dctx := ctx + if mc.cfg.Timeout > 0 { + var cancel context.CancelFunc + dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) + defer cancel() + } + mc.netConn, err = dial(dctx, mc.cfg.Addr) } else { nd := net.Dialer{Timeout: mc.cfg.Timeout} mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) } if err != nil { - if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - errLog.Print("net.Error from Dial()': ", nerr.Error()) - return nil, driver.ErrBadConn - } return nil, err } @@ -64,6 +66,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { // Call startWatcher for context support (From Go 1.8) mc.startWatcher() if err := mc.watchCancel(ctx); err != nil { + mc.cleanup() return nil, err } defer mc.finish() diff --git a/vendor/github.com/go-sql-driver/mysql/connector_test.go b/vendor/github.com/go-sql-driver/mysql/connector_test.go new file mode 100644 index 0000000..976903c --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connector_test.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "context" + "net" + "testing" + "time" +) + +func TestConnectorReturnsTimeout(t *testing.T) { + connector := &connector{&Config{ + Net: "tcp", + Addr: "1.1.1.1:1234", + Timeout: 10 * time.Millisecond, + }} + + _, err := connector.Connect(context.Background()) + if err == nil { + t.Fatal("error expected") + } + + if nerr, ok := err.(*net.OpError); ok { + expected := "dial tcp 1.1.1.1:1234: i/o timeout" + if nerr.Error() != expected { + t.Fatalf("expected %q, got %q", expected, nerr.Error()) + } + } else { + t.Fatalf("expected %T, got %T", nerr, err) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index 1f9decf..c1bdf11 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -83,3 +83,25 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { func init() { sql.Register("mysql", &MySQLDriver{}) } + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110.go b/vendor/github.com/go-sql-driver/mysql/driver_go110.go deleted file mode 100644 index eb5a8fe..0000000 --- a/vendor/github.com/go-sql-driver/mysql/driver_go110.go +++ /dev/null @@ -1,37 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "database/sql/driver" -) - -// NewConnector returns new driver.Connector. -func NewConnector(cfg *Config) (driver.Connector, error) { - cfg = cfg.Clone() - // normalize the contents of cfg so calls to NewConnector have the same - // behavior as MySQLDriver.OpenConnector - if err := cfg.normalize(); err != nil { - return nil, err - } - return &connector{cfg: cfg}, nil -} - -// OpenConnector implements driver.DriverContext. -func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { - cfg, err := ParseDSN(dsn) - if err != nil { - return nil, err - } - return &connector{ - cfg: cfg, - }, nil -} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go b/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go deleted file mode 100644 index 19a0e59..0000000 --- a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "context" - "database/sql" - "database/sql/driver" - "fmt" - "net" - "testing" - "time" -) - -var _ driver.DriverContext = &MySQLDriver{} - -type dialCtxKey struct{} - -func TestConnectorObeysDialTimeouts(t *testing.T) { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - if !ctx.Value(dialCtxKey{}).(bool) { - return nil, fmt.Errorf("test error: query context is not propagated to our dialer") - } - return d.DialContext(ctx, prot, addr) - }) - - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) - if err != nil { - t.Fatalf("error connecting: %s", err.Error()) - } - defer db.Close() - - ctx := context.WithValue(context.Background(), dialCtxKey{}, true) - - _, err = db.ExecContext(ctx, "DO 1") - if err != nil { - t.Fatal(err) - } -} - -func configForTests(t *testing.T) *Config { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - mycnf := NewConfig() - mycnf.User = user - mycnf.Passwd = pass - mycnf.Addr = addr - mycnf.Net = prot - mycnf.DBName = dbname - return mycnf -} - -func TestNewConnector(t *testing.T) { - mycnf := configForTests(t) - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - db := sql.OpenDB(conn) - defer db.Close() - - if err := db.Ping(); err != nil { - t.Fatal(err) - } -} - -type slowConnection struct { - net.Conn - slowdown time.Duration -} - -func (sc *slowConnection) Read(b []byte) (int, error) { - time.Sleep(sc.slowdown) - return sc.Conn.Read(b) -} - -type connectorHijack struct { - driver.Connector - connErr error -} - -func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { - var conn driver.Conn - conn, cw.connErr = cw.Connector.Connect(ctx) - return conn, cw.connErr -} - -func TestConnectorTimeoutsDuringOpen(t *testing.T) { - RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, prot, addr) - if err != nil { - return nil, err - } - return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil - }) - - mycnf := configForTests(t) - mycnf.Net = "slowconn" - - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - hijack := &connectorHijack{Connector: conn} - - db := sql.OpenDB(hijack) - defer db.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - _, err = db.ExecContext(ctx, "DO 1") - if err != context.DeadlineExceeded { - t.Fatalf("ExecContext should have timed out") - } - if hijack.connErr != context.DeadlineExceeded { - t.Fatalf("(*Connector).Connect should have timed out") - } -} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_test.go b/vendor/github.com/go-sql-driver/mysql/driver_test.go index 3dee1ba..ace083d 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver_test.go +++ b/vendor/github.com/go-sql-driver/mysql/driver_test.go @@ -1874,7 +1874,7 @@ func TestDialNonRetryableNetErr(t *testing.T) { func TestDialTemporaryNetErr(t *testing.T) { testErr := netErrorMock{temporary: true} - testDialError(t, testErr, driver.ErrBadConn) + testDialError(t, testErr, testErr) } // Tests custom dial functions @@ -2994,3 +2994,172 @@ func TestRawBytesAreNotModified(t *testing.T) { } }) } + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} + +// A connection which can only be closed. +type dummyConnection struct { + net.Conn + closed bool +} + +func (d *dummyConnection) Close() error { + d.closed = true + return nil +} + +func TestConnectorTimeoutsWatchCancel(t *testing.T) { + var ( + cancel func() // Used to cancel the context just after connecting. + created *dummyConnection // The created connection. + ) + + RegisterDialContext("TestConnectorTimeoutsWatchCancel", func(ctx context.Context, addr string) (net.Conn, error) { + // Canceling at this time triggers the watchCancel error branch in Connect(). + cancel() + created = &dummyConnection{} + return created, nil + }) + + mycnf := NewConfig() + mycnf.User = "root" + mycnf.Addr = "foo" + mycnf.Net = "TestConnectorTimeoutsWatchCancel" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + if _, err := db.Conn(ctx); err != context.Canceled { + t.Errorf("got %v, want context.Canceled", err) + } + + if created == nil { + t.Fatal("no connection created") + } + if !created.closed { + t.Errorf("connection not closed") + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go index 1d9b4ab..75c8c24 100644 --- a/vendor/github.com/go-sql-driver/mysql/dsn.go +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -55,6 +55,7 @@ type Config struct { AllowCleartextPasswords bool // Allows the cleartext client side plugin AllowNativePasswords bool // Allows the native password authentication method AllowOldPasswords bool // Allows the old insecure password method + CheckConnLiveness bool // Check connections for liveness before using them ClientFoundRows bool // Return number of matching rows instead of rows changed ColumnsWithAlias bool // Prepend table alias to column names InterpolateParams bool // Interpolate placeholders into query string @@ -70,6 +71,7 @@ func NewConfig() *Config { Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, + CheckConnLiveness: true, } } @@ -148,6 +150,19 @@ func (cfg *Config) normalize() error { return nil } +func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { + buf.Grow(1 + len(name) + 1 + len(value)) + if !*hasParam { + *hasParam = true + buf.WriteByte('?') + } else { + buf.WriteByte('&') + } + buf.WriteString(name) + buf.WriteByte('=') + buf.WriteString(value) +} + // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. func (cfg *Config) FormatDSN() string { @@ -186,165 +201,75 @@ func (cfg *Config) FormatDSN() string { } if cfg.AllowCleartextPasswords { - if hasParam { - buf.WriteString("&allowCleartextPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowCleartextPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true") } if !cfg.AllowNativePasswords { - if hasParam { - buf.WriteString("&allowNativePasswords=false") - } else { - hasParam = true - buf.WriteString("?allowNativePasswords=false") - } + writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false") } if cfg.AllowOldPasswords { - if hasParam { - buf.WriteString("&allowOldPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowOldPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowOldPasswords", "true") + } + + if !cfg.CheckConnLiveness { + writeDSNParam(&buf, &hasParam, "checkConnLiveness", "false") } if cfg.ClientFoundRows { - if hasParam { - buf.WriteString("&clientFoundRows=true") - } else { - hasParam = true - buf.WriteString("?clientFoundRows=true") - } + writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } if col := cfg.Collation; col != defaultCollation && len(col) > 0 { - if hasParam { - buf.WriteString("&collation=") - } else { - hasParam = true - buf.WriteString("?collation=") - } - buf.WriteString(col) + writeDSNParam(&buf, &hasParam, "collation", col) } if cfg.ColumnsWithAlias { - if hasParam { - buf.WriteString("&columnsWithAlias=true") - } else { - hasParam = true - buf.WriteString("?columnsWithAlias=true") - } + writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true") } if cfg.InterpolateParams { - if hasParam { - buf.WriteString("&interpolateParams=true") - } else { - hasParam = true - buf.WriteString("?interpolateParams=true") - } + writeDSNParam(&buf, &hasParam, "interpolateParams", "true") } if cfg.Loc != time.UTC && cfg.Loc != nil { - if hasParam { - buf.WriteString("&loc=") - } else { - hasParam = true - buf.WriteString("?loc=") - } - buf.WriteString(url.QueryEscape(cfg.Loc.String())) + writeDSNParam(&buf, &hasParam, "loc", url.QueryEscape(cfg.Loc.String())) } if cfg.MultiStatements { - if hasParam { - buf.WriteString("&multiStatements=true") - } else { - hasParam = true - buf.WriteString("?multiStatements=true") - } + writeDSNParam(&buf, &hasParam, "multiStatements", "true") } if cfg.ParseTime { - if hasParam { - buf.WriteString("&parseTime=true") - } else { - hasParam = true - buf.WriteString("?parseTime=true") - } + writeDSNParam(&buf, &hasParam, "parseTime", "true") } if cfg.ReadTimeout > 0 { - if hasParam { - buf.WriteString("&readTimeout=") - } else { - hasParam = true - buf.WriteString("?readTimeout=") - } - buf.WriteString(cfg.ReadTimeout.String()) + writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } if cfg.RejectReadOnly { - if hasParam { - buf.WriteString("&rejectReadOnly=true") - } else { - hasParam = true - buf.WriteString("?rejectReadOnly=true") - } + writeDSNParam(&buf, &hasParam, "rejectReadOnly", "true") } if len(cfg.ServerPubKey) > 0 { - if hasParam { - buf.WriteString("&serverPubKey=") - } else { - hasParam = true - buf.WriteString("?serverPubKey=") - } - buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + writeDSNParam(&buf, &hasParam, "serverPubKey", url.QueryEscape(cfg.ServerPubKey)) } if cfg.Timeout > 0 { - if hasParam { - buf.WriteString("&timeout=") - } else { - hasParam = true - buf.WriteString("?timeout=") - } - buf.WriteString(cfg.Timeout.String()) + writeDSNParam(&buf, &hasParam, "timeout", cfg.Timeout.String()) } if len(cfg.TLSConfig) > 0 { - if hasParam { - buf.WriteString("&tls=") - } else { - hasParam = true - buf.WriteString("?tls=") - } - buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + writeDSNParam(&buf, &hasParam, "tls", url.QueryEscape(cfg.TLSConfig)) } if cfg.WriteTimeout > 0 { - if hasParam { - buf.WriteString("&writeTimeout=") - } else { - hasParam = true - buf.WriteString("?writeTimeout=") - } - buf.WriteString(cfg.WriteTimeout.String()) + writeDSNParam(&buf, &hasParam, "writeTimeout", cfg.WriteTimeout.String()) } if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { - if hasParam { - buf.WriteString("&maxAllowedPacket=") - } else { - hasParam = true - buf.WriteString("?maxAllowedPacket=") - } - buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) - + writeDSNParam(&buf, &hasParam, "maxAllowedPacket", strconv.Itoa(cfg.MaxAllowedPacket)) } // other params @@ -355,16 +280,7 @@ func (cfg *Config) FormatDSN() string { } sort.Strings(params) for _, param := range params { - if hasParam { - buf.WriteByte('&') - } else { - hasParam = true - buf.WriteByte('?') - } - - buf.WriteString(param) - buf.WriteByte('=') - buf.WriteString(url.QueryEscape(cfg.Params[param])) + writeDSNParam(&buf, &hasParam, param, url.QueryEscape(cfg.Params[param])) } } @@ -491,6 +407,14 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // Check connections for Liveness before using them + case "checkConnLiveness": + var isBool bool + cfg.CheckConnLiveness, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + // Switch "rowsAffected" mode case "clientFoundRows": var isBool bool diff --git a/vendor/github.com/go-sql-driver/mysql/dsn_test.go b/vendor/github.com/go-sql-driver/mysql/dsn_test.go index 50dc293..89815b3 100644 --- a/vendor/github.com/go-sql-driver/mysql/dsn_test.go +++ b/vendor/github.com/go-sql-driver/mysql/dsn_test.go @@ -22,55 +22,55 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { - "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, + "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, } diff --git a/vendor/github.com/go-sql-driver/mysql/go.mod b/vendor/github.com/go-sql-driver/mysql/go.mod new file mode 100644 index 0000000..fffbf6a --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/go.mod @@ -0,0 +1,3 @@ +module github.com/go-sql-driver/mysql + +go 1.10 diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime.go b/vendor/github.com/go-sql-driver/mysql/nulltime.go new file mode 100644 index 0000000..afa8a89 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime.go @@ -0,0 +1,50 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "fmt" + "time" +) + +// Scan implements the Scanner interface. +// The value type must be time.Time or string / []byte (formatted time-string), +// otherwise Scan fails. +func (nt *NullTime) Scan(value interface{}) (err error) { + if value == nil { + nt.Time, nt.Valid = time.Time{}, false + return + } + + switch v := value.(type) { + case time.Time: + nt.Time, nt.Valid = v, true + return + case []byte: + nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Valid = (err == nil) + return + case string: + nt.Time, err = parseDateTime(v, time.UTC) + nt.Valid = (err == nil) + return + } + + nt.Valid = false + return fmt.Errorf("Can't convert %T to time.Time", value) +} + +// Value implements the driver Valuer interface. +func (nt NullTime) Value() (driver.Value, error) { + if !nt.Valid { + return nil, nil + } + return nt.Time, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go new file mode 100644 index 0000000..c392594 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go @@ -0,0 +1,31 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.13 + +package mysql + +import ( + "database/sql" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime sql.NullTime diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go new file mode 100644 index 0000000..86d159d --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go @@ -0,0 +1,34 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !go1.13 + +package mysql + +import ( + "time" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_test.go b/vendor/github.com/go-sql-driver/mysql/nulltime_test.go new file mode 100644 index 0000000..a14ec06 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_test.go @@ -0,0 +1,62 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql" + "database/sql/driver" + "testing" + "time" +) + +var ( + // Check implementation of interfaces + _ driver.Valuer = NullTime{} + _ sql.Scanner = (*NullTime)(nil) +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index 30b3352..82ad7a2 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -115,7 +115,7 @@ func (mc *mysqlConn) writePacket(data []byte) error { if mc.cfg.ReadTimeout != 0 { err = conn.SetReadDeadline(time.Time{}) } - if err == nil { + if err == nil && mc.cfg.CheckConnLiveness { err = connCheck(conn) } if err != nil { diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index cfa10e9..9552e80 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -106,60 +106,6 @@ func readBool(input string) (value bool, valid bool) { * Time related utils * ******************************************************************************/ -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { base := "0000-00-00 00:00:00.0000000" switch len(str) { diff --git a/vendor/github.com/go-sql-driver/mysql/utils_test.go b/vendor/github.com/go-sql-driver/mysql/utils_test.go index 8951a7a..10a60c2 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils_test.go +++ b/vendor/github.com/go-sql-driver/mysql/utils_test.go @@ -14,49 +14,8 @@ import ( "database/sql/driver" "encoding/binary" "testing" - "time" ) -func TestScanNullTime(t *testing.T) { - var scanTests = []struct { - in interface{} - error bool - valid bool - time time.Time - }{ - {tDate, false, true, tDate}, - {sDate, false, true, tDate}, - {[]byte(sDate), false, true, tDate}, - {tDateTime, false, true, tDateTime}, - {sDateTime, false, true, tDateTime}, - {[]byte(sDateTime), false, true, tDateTime}, - {tDate0, false, true, tDate0}, - {sDate0, false, true, tDate0}, - {[]byte(sDate0), false, true, tDate0}, - {sDateTime0, false, true, tDate0}, - {[]byte(sDateTime0), false, true, tDate0}, - {"", true, false, tDate0}, - {"1234", true, false, tDate0}, - {0, true, false, tDate0}, - } - - var nt = NullTime{} - var err error - - for _, tst := range scanTests { - err = nt.Scan(tst.in) - if (err != nil) != tst.error { - t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) - } - if nt.Valid != tst.valid { - t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) - } - if nt.Time != tst.time { - t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) - } - } -} - func TestLengthEncodedInteger(t *testing.T) { var integerTests = []struct { num uint64 From 9f5edc923e5eaef2150b9680307dee02818e6e46 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Feb 2020 12:48:23 +0200 Subject: [PATCH 073/103] Support --mysql-timeout flag --- go/cmd/gh-ost/main.go | 1 + go/mysql/connection.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..74c62d7 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -48,6 +48,7 @@ func main() { flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") + flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user") flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password") flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host") diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 654998c..6855ee0 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -27,6 +27,7 @@ type ConnectionConfig struct { Password string ImpliedKey *InstanceKey tlsConfig *tls.Config + Timeout float64 } func NewConnectionConfig() *ConnectionConfig { @@ -44,6 +45,7 @@ func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionC User: this.User, Password: this.Password, tlsConfig: this.tlsConfig, + Timeout: this.Timeout, } config.ImpliedKey = &config.Key return config @@ -116,5 +118,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { if this.tlsConfig != nil { tlsOption = TLS_CONFIG_KEY } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?timeout=%fs&readTimeout=%fs&writeTimeout=%fs&interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, this.Timeout, this.Timeout, this.Timeout, interpolateParams, tlsOption) } From 97f2d7161637b56351d9ed2c3dad136558b869c0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Feb 2020 12:52:26 +0200 Subject: [PATCH 074/103] fix unit tests --- go/mysql/connection_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 6ce85a7..2befbc9 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") } func TestGetDBUriWithTLSSetup(t *testing.T) { @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") } From d5a7e43266c36d86b7ef478bc975983ca24317a9 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Tue, 24 Mar 2020 14:54:54 -0500 Subject: [PATCH 075/103] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 63f0df9..605546d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /bin/ /libexec/ /.vendor/ +.idea/ From 0b2702bf539aebea09948fe5e9486a30135e4241 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 14:12:00 -0500 Subject: [PATCH 076/103] Throttle on no metrics/error --- go/logic/throttler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 6f9f4bb..0923c7b 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -104,7 +104,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint } } // Got here? No metrics indicates we need throttling. - return false, "", base.NoThrottleReasonHint + return true, "", base.NoThrottleReasonHint } // parseChangelogHeartbeat parses a string timestamp and deduces replication lag From 64083e4705042a1d6a816a5671a61735139fe347 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:40:09 -0500 Subject: [PATCH 077/103] Throttle on no metrics/error --- go/logic/throttler.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 0923c7b..25ad6cf 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -84,6 +84,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint if statusCode != 0 && statusCode != http.StatusOK { return true, this.throttleHttpMessage(int(statusCode)), base.NoThrottleReasonHint } + // Replication lag throttle maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold) lag := atomic.LoadInt64(&this.migrationContext.CurrentLag) @@ -104,7 +105,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint } } // Got here? No metrics indicates we need throttling. - return true, "", base.NoThrottleReasonHint + return false, "", base.NoThrottleReasonHint } // parseChangelogHeartbeat parses a string timestamp and deduces replication lag @@ -288,7 +289,11 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- return false, nil } - collectFunc() + _, err := collectFunc() + if err != nil { + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } + firstThrottlingCollected <- true ticker := time.Tick(100 * time.Millisecond) From ca0b822a3de06345a36326fa10a15bead239adaf Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:41:23 -0500 Subject: [PATCH 078/103] Add comment --- go/logic/throttler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 25ad6cf..6113374 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -291,6 +291,7 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- _, err := collectFunc() if err != nil { + // If an error occurs during the HTTP throttle check, let's throttle to be safe atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } From d31f3aac0c09ecb258a29f1cf0f0bb3d2b4a639c Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:47:56 -0500 Subject: [PATCH 079/103] Add CLI flag for ignoring http errors --- go/base/context.go | 8 ++++++++ go/cmd/gh-ost/main.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/go/base/context.go b/go/base/context.go index 5ebf092..983ae8f 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,6 +119,7 @@ type MigrationContext struct { ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string + ignoreHTTPErrors bool ThrottleCommandedByUser int64 HibernateUntil int64 maxLoad LoadMap @@ -574,6 +575,13 @@ func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string) { this.throttleHTTP = throttleHTTP } +func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) { + this.throttleHTTPMutex.Lock() + defer this.throttleHTTPMutex.Unlock() + + this.ignoreHTTPErrors = ignoreHTTPErrors +} + func (this *MigrationContext) GetMaxLoad() LoadMap { this.throttleMutex.Lock() defer this.throttleMutex.Unlock() diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..bfd852a 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -106,6 +106,7 @@ func main() { throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") + ignoreHTTPErrors := flag.Bool("ignore-http-error", false, "ignore HTTP connection errors during throttle check") heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value") flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered") flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations") @@ -259,6 +260,7 @@ func main() { migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis) migrationContext.SetThrottleQuery(*throttleQuery) migrationContext.SetThrottleHTTP(*throttleHTTP) + migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors) migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() if err := migrationContext.SetupTLS(); err != nil { From 46dabd338b40ef73fb782046b792d3fc1f30d83d Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:48:19 -0500 Subject: [PATCH 080/103] go fmt --- go/base/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/base/context.go b/go/base/context.go index 983ae8f..9a3c577 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,7 +119,7 @@ type MigrationContext struct { ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string - ignoreHTTPErrors bool + ignoreHTTPErrors bool ThrottleCommandedByUser int64 HibernateUntil int64 maxLoad LoadMap From 57cf5f3c90c2e2db50863505362e484f1bb26695 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:58:32 -0500 Subject: [PATCH 081/103] add override to ignore http errors --- go/base/context.go | 4 ++-- go/cmd/gh-ost/main.go | 2 +- go/logic/throttler.go | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 9a3c577..52d02dd 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,7 +119,7 @@ type MigrationContext struct { ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string - ignoreHTTPErrors bool + IgnoreHTTPErrors bool ThrottleCommandedByUser int64 HibernateUntil int64 maxLoad LoadMap @@ -579,7 +579,7 @@ func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) { this.throttleHTTPMutex.Lock() defer this.throttleHTTPMutex.Unlock() - this.ignoreHTTPErrors = ignoreHTTPErrors + this.IgnoreHTTPErrors = ignoreHTTPErrors } func (this *MigrationContext) GetMaxLoad() LoadMap { diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index bfd852a..33fc6f4 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -106,7 +106,7 @@ func main() { throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") - ignoreHTTPErrors := flag.Bool("ignore-http-error", false, "ignore HTTP connection errors during throttle check") + ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check") heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value") flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered") flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations") diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 6113374..d3d184a 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -291,8 +291,10 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- _, err := collectFunc() if err != nil { - // If an error occurs during the HTTP throttle check, let's throttle to be safe - atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + // If not told to ignore errors, we'll throttle on HTTP connection issues + if !this.migrationContext.IgnoreHTTPErrors { + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } } firstThrottlingCollected <- true From df60fa4204bc7e84a02b5a9b82a19938353149e9 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 16:06:17 -0500 Subject: [PATCH 082/103] add error logging --- go/logic/throttler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index d3d184a..dd872e7 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -293,6 +293,7 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- if err != nil { // If not told to ignore errors, we'll throttle on HTTP connection issues if !this.migrationContext.IgnoreHTTPErrors { + log.Errorf("errors occurred during HTTP throttle check: %+v", err) atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } } From 7893f8e1c3b575523176705de127e790c2a3ca40 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Mon, 30 Mar 2020 09:53:10 -0500 Subject: [PATCH 083/103] catch error in collectFunc --- go/logic/throttler.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index dd872e7..ace1e49 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -306,7 +306,16 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- return } - if sleep, _ := collectFunc(); sleep { + sleep, err := collectFunc() + if err != nil { + // If not told to ignore errors, we'll throttle on HTTP connection issues + if !this.migrationContext.IgnoreHTTPErrors { + log.Errorf("errors occurred during HTTP throttle check: %+v", err) + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } + } + + if sleep { time.Sleep(1 * time.Second) } } From 5816ede7b3350d3810878d299dd8341f79b4eb2d Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Mon, 30 Mar 2020 09:55:49 -0500 Subject: [PATCH 084/103] add error message --- go/logic/throttler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index ace1e49..be752fe 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -19,20 +19,22 @@ import ( ) var ( - httpStatusMessages map[int]string = map[int]string{ + httpStatusMessages = map[int]string{ 200: "OK", 404: "Not found", 417: "Expectation failed", 429: "Too many requests", 500: "Internal server error", + -1: "Connection error", } // See https://github.com/github/freno/blob/master/doc/http.md - httpStatusFrenoMessages map[int]string = map[int]string{ + httpStatusFrenoMessages = map[int]string{ 200: "OK", 404: "freno: unknown metric", 417: "freno: access forbidden", 429: "freno: threshold exceeded", 500: "freno: internal error", + -1: "freno: connection error", } ) From 2178b5947be80da499fef2c31ac827eaf4920d48 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Mon, 30 Mar 2020 10:05:23 -0500 Subject: [PATCH 085/103] remove spammy error --- go/logic/throttler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index be752fe..2cf0d97 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -25,7 +25,7 @@ var ( 417: "Expectation failed", 429: "Too many requests", 500: "Internal server error", - -1: "Connection error", + -1: "Connection error", } // See https://github.com/github/freno/blob/master/doc/http.md httpStatusFrenoMessages = map[int]string{ @@ -295,7 +295,6 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- if err != nil { // If not told to ignore errors, we'll throttle on HTTP connection issues if !this.migrationContext.IgnoreHTTPErrors { - log.Errorf("errors occurred during HTTP throttle check: %+v", err) atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } } @@ -312,7 +311,6 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- if err != nil { // If not told to ignore errors, we'll throttle on HTTP connection issues if !this.migrationContext.IgnoreHTTPErrors { - log.Errorf("errors occurred during HTTP throttle check: %+v", err) atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } } From 61de0980721838f1bfe1e335ac3ebe272ca0deaf Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 31 Mar 2020 16:25:16 -0400 Subject: [PATCH 086/103] Add a check to rows.Err after processing all rows Closes #822. In https://github.com/go-sql-driver/mysql/issues/1075, @acharis notes that the way the go-sql driver is written, query timeout errors don't get set in `rows.Err()` until _after_ a call to `rows.Next()` is made. Because this kind of error means there will be no rows in the result set, the `for rows.Next()` will never enter the for loop, so we must check the value of `rows.Err()` after the loop, and surface the error up appropriately. --- go/logic/applier.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/logic/applier.go b/go/logic/applier.go index 5fb795b..1bffd7a 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -444,6 +444,9 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo } hasFurtherRange = true } + if err = rows.Err(); err != nil { + return hasFurtherRange, err + } if hasFurtherRange { this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues return hasFurtherRange, nil From 90ad7a061f9ace1fcd752ed42851a0017081268f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 21 Apr 2020 12:50:23 -0400 Subject: [PATCH 087/103] Handle the rest of `rows.Err` cases --- go/logic/applier.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/logic/applier.go b/go/logic/applier.go index 1bffd7a..783c190 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -371,6 +371,8 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { } } log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) + + err = rows.Err() return err } @@ -392,6 +394,8 @@ func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { } } log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) + + err = rows.Err() return err } From 1a8c372947bd441ab95bf6a4b2ea5b1922475eb4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jun 2020 08:39:16 +0300 Subject: [PATCH 088/103] Using golang 1.14 --- .github/workflows/ci.yml | 7 +++---- .github/workflows/replica-tests.yml | 7 +++---- Dockerfile.packaging | 2 +- Dockerfile.test | 2 +- build.sh | 4 ++-- script/ensure-go-installed | 4 ++-- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec8978d..ded3c8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - - name: Set up Go 1.12 + - name: Set up Go 1.14 uses: actions/setup-go@v1 with: - version: 1.12 - id: go + go-version: 1.14 - name: Build run: script/cibuild diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index d775131..31e2052 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -8,13 +8,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - - name: Set up Go 1.12 + - name: Set up Go 1.14 uses: actions/setup-go@v1 with: - version: 1.12 - id: go + go-version: 1.14 - name: migration tests run: script/cibuild-gh-ost-replica-tests diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 214c70c..74aa190 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,6 @@ # -FROM golang:1.12.6 +FROM golang:1.14.4 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index 6abc4a0..3b3bfe5 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.12.1 +FROM golang:1.14.4 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/build.sh b/build.sh index 46db9c2..98633cd 100755 --- a/build.sh +++ b/build.sh @@ -18,8 +18,8 @@ function build { GOOS=$3 GOARCH=$4 - if ! go version | egrep -q 'go(1\.1[234])' ; then - echo "go version must be 1.12 or above" + if ! go version | egrep -q 'go(1\.1[23456])' ; then + echo "go version must be 1.14 or above" exit 1 fi diff --git a/script/ensure-go-installed b/script/ensure-go-installed index c9c2ae8..1abdeb3 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,7 +1,7 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.12.6 -SUPPORTED_GO_VERSIONS='go1.1[234]' +PREFERRED_GO_VERSION=go1.14 +SUPPORTED_GO_VERSIONS='go1.1[23456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815 From fb4aca156751db491d8a83096367c379cdebcc64 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jun 2020 08:49:30 +0300 Subject: [PATCH 089/103] checksums --- script/ensure-go-installed | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/ensure-go-installed b/script/ensure-go-installed index 1abdeb3..32ade46 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,13 +1,13 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.14 +PREFERRED_GO_VERSION=go1.14.4 SUPPORTED_GO_VERSIONS='go1.1[23456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815 +GO_PKG_DARWIN_SHA=b518f21f823759ee30faddb1f623810a432499f050c9338777523d9c8551c62c GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz -GO_PKG_LINUX_SHA=dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c +GO_PKG_LINUX_SHA=aed845e4185a0b2a3c3d5e1d0a35491702c55889192bb9c30e67a3de6849c067 export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" cd $ROOTDIR From 2b71b73285338e96798d2530c8adec9aeb28910f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jun 2020 08:57:19 +0300 Subject: [PATCH 090/103] Actions/workflows: upload binary artifact --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ded3c8b..3556e1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,3 +17,9 @@ jobs: - name: Build run: script/cibuild + + - name: Upload gh-ost binary artifact + uses: actions/upload-artifact@v1 + with: + name: gh-ost + path: bin/gh-ost From 8eb300bdc8bb183e79e4f50f40288e114aa5c55a Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:52:47 +0300 Subject: [PATCH 091/103] expect 1.14 and above in build scripts; update to readme.md --- README.md | 2 +- build.sh | 2 +- script/ensure-go-installed | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cd6f592..7b2bc1d 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d [Download latest release here](https://github.com/github/gh-ost/releases/latest) -`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either: +`gh-ost` is a Go project; it is built with Go `1.14` and above. To build on your own, use either: - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary. - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost` diff --git a/build.sh b/build.sh index 98633cd..b5d4659 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ function build { GOOS=$3 GOARCH=$4 - if ! go version | egrep -q 'go(1\.1[23456])' ; then + if ! go version | egrep -q 'go(1\.1[456])' ; then echo "go version must be 1.14 or above" exit 1 fi diff --git a/script/ensure-go-installed b/script/ensure-go-installed index 32ade46..98ccdc1 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,7 +1,7 @@ #!/bin/bash PREFERRED_GO_VERSION=go1.14.4 -SUPPORTED_GO_VERSIONS='go1.1[23456]' +SUPPORTED_GO_VERSIONS='go1.1[456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg GO_PKG_DARWIN_SHA=b518f21f823759ee30faddb1f623810a432499f050c9338777523d9c8551c62c From 6c7b4736e15e0fc8c1768395d16cc514c95036a0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 22 Jul 2020 12:33:02 +0300 Subject: [PATCH 092/103] Support a complete ALTER TABLE statement in --alter --- go/cmd/gh-ost/main.go | 23 ++++++--- go/logic/migrator.go | 4 +- go/sql/parser.go | 83 +++++++++++++++++++++++++++------ go/sql/parser_test.go | 106 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 178 insertions(+), 38 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 33fc6f4..57c6ff8 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -14,6 +14,7 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" + "github.com/github/gh-ost/go/sql" _ "github.com/go-sql-driver/mysql" "github.com/outbrain/golib/log" @@ -172,15 +173,25 @@ func main() { log.SetLevel(log.ERROR) } - if migrationContext.DatabaseName == "" { - log.Fatalf("--database must be provided and database name must not be empty") - } - if migrationContext.OriginalTableName == "" { - log.Fatalf("--table must be provided and table name must not be empty") - } if migrationContext.AlterStatement == "" { log.Fatalf("--alter must be provided and statement must not be empty") } + parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement) + + if migrationContext.DatabaseName == "" { + if parser.HasExplicitSchema() { + migrationContext.DatabaseName = parser.GetExplicitSchema() + } else { + log.Fatalf("--database must be provided and database name must not be empty, or --alter must specify database name") + } + } + if migrationContext.OriginalTableName == "" { + if parser.HasExplicitTable() { + migrationContext.OriginalTableName = parser.GetExplicitTable() + } else { + log.Fatalf("--table must be provided and table name must not be empty, or --alter must specify table name") + } + } migrationContext.Noop = !(*executeFlag) if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica { log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") diff --git a/go/logic/migrator.go b/go/logic/migrator.go index b1a238f..cff64e1 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -62,7 +62,7 @@ const ( // Migrator is the main schema migration flow manager. type Migrator struct { - parser *sql.Parser + parser *sql.AlterTableParser inspector *Inspector applier *Applier eventsStreamer *EventsStreamer @@ -90,7 +90,7 @@ type Migrator struct { func NewMigrator(context *base.MigrationContext) *Migrator { migrator := &Migrator{ migrationContext: context, - parser: sql.NewParser(), + parser: sql.NewAlterTableParser(), ghostTableMigrated: make(chan bool), firstThrottlingCollected: make(chan bool, 3), rowCopyComplete: make(chan error), diff --git a/go/sql/parser.go b/go/sql/parser.go index 447cbcc..d050fcd 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -12,26 +12,47 @@ import ( ) var ( - sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')") - renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) - dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) - renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) + sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')") + renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) + dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) + renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) + alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{ + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`), + } + alterTableExplicitTableRegexps = []*regexp.Regexp{ + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`), + } ) -type Parser struct { +type AlterTableParser struct { columnRenameMap map[string]string droppedColumns map[string]bool isRenameTable bool + + alterTokens []string + + explicitSchema string + explicitTable string } -func NewParser() *Parser { - return &Parser{ +func NewAlterTableParser() *AlterTableParser { + return &AlterTableParser{ columnRenameMap: make(map[string]string), droppedColumns: make(map[string]bool), } } -func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) { +func NewParserFromAlterStatement(alterStatement string) *AlterTableParser { + parser := NewAlterTableParser() + parser.ParseAlterStatement(alterStatement) + return parser +} + +func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) { terminatingQuote := rune(0) f := func(c rune) bool { switch { @@ -58,13 +79,13 @@ func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []stri return tokens, nil } -func (this *Parser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) { +func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) { strippedStatement = alterStatement strippedStatement = sanitizeQuotesRegexp.ReplaceAllString(strippedStatement, "''") return strippedStatement } -func (this *Parser) parseAlterToken(alterToken string) (err error) { +func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) { { // rename allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1) @@ -97,16 +118,33 @@ func (this *Parser) parseAlterToken(alterToken string) (err error) { return nil } -func (this *Parser) ParseAlterStatement(alterStatement string) (err error) { +func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) { + + for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps { + if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + this.explicitSchema = submatch[1] + this.explicitTable = submatch[2] + alterStatement = submatch[3] + break + } + } + for _, alterTableRegexp := range alterTableExplicitTableRegexps { + if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + this.explicitTable = submatch[1] + alterStatement = submatch[2] + break + } + } alterTokens, _ := this.tokenizeAlterStatement(alterStatement) for _, alterToken := range alterTokens { alterToken = this.sanitizeQuotesFromAlterStatement(alterToken) this.parseAlterToken(alterToken) + this.alterTokens = append(this.alterTokens, alterToken) } return nil } -func (this *Parser) GetNonTrivialRenames() map[string]string { +func (this *AlterTableParser) GetNonTrivialRenames() map[string]string { result := make(map[string]string) for column, renamed := range this.columnRenameMap { if column != renamed { @@ -116,14 +154,29 @@ func (this *Parser) GetNonTrivialRenames() map[string]string { return result } -func (this *Parser) HasNonTrivialRenames() bool { +func (this *AlterTableParser) HasNonTrivialRenames() bool { return len(this.GetNonTrivialRenames()) > 0 } -func (this *Parser) DroppedColumnsMap() map[string]bool { +func (this *AlterTableParser) DroppedColumnsMap() map[string]bool { return this.droppedColumns } -func (this *Parser) IsRenameTable() bool { +func (this *AlterTableParser) IsRenameTable() bool { return this.isRenameTable } +func (this *AlterTableParser) GetExplicitSchema() string { + return this.explicitSchema +} + +func (this *AlterTableParser) HasExplicitSchema() bool { + return this.GetExplicitSchema() != "" +} + +func (this *AlterTableParser) GetExplicitTable() string { + return this.explicitTable +} + +func (this *AlterTableParser) HasExplicitTable() bool { + return this.GetExplicitTable() != "" +} diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 5d38130..626fe75 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -19,7 +19,7 @@ func init() { func TestParseAlterStatement(t *testing.T) { statement := "add column t int, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) @@ -27,7 +27,7 @@ func TestParseAlterStatement(t *testing.T) { func TestParseAlterStatementTrivialRename(t *testing.T) { statement := "add column t int, change ts ts timestamp, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) @@ -37,7 +37,7 @@ func TestParseAlterStatementTrivialRename(t *testing.T) { func TestParseAlterStatementTrivialRenames(t *testing.T) { statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) @@ -58,7 +58,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { } for _, statement := range statements { - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) renames := parser.GetNonTrivialRenames() @@ -69,7 +69,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { } func TestTokenizeAlterStatement(t *testing.T) { - parser := NewParser() + parser := NewAlterTableParser() { alterStatement := "add column t int" tokens, _ := parser.tokenizeAlterStatement(alterStatement) @@ -108,7 +108,7 @@ func TestTokenizeAlterStatement(t *testing.T) { } func TestSanitizeQuotesFromAlterStatement(t *testing.T) { - parser := NewParser() + parser := NewAlterTableParser() { alterStatement := "add column e enum('a','b','c')" strippedStatement := parser.sanitizeQuotesFromAlterStatement(alterStatement) @@ -124,7 +124,7 @@ func TestSanitizeQuotesFromAlterStatement(t *testing.T) { func TestParseAlterStatementDroppedColumns(t *testing.T) { { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -132,7 +132,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["b"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop key c_idx, drop column `d`" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -141,7 +141,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["d"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop key c_idx, drop column `d`, drop `e`, drop primary key, drop foreign key fk_1" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -151,7 +151,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["e"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop bad statement, add column i int" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -163,38 +163,114 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { func TestParseAlterStatementRenameTable(t *testing.T) { { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "engine=innodb rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "rename as something_else, engine=innodb" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } } + +func TestParseAlterStatementExplicitTable(t *testing.T) { + + { + parser := NewAlterTableParser() + statement := "drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm with spaces`.`tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm with spaces") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm`.`tbl with spaces` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl with spaces") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm`.tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table scm.`tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table scm.tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } +} From c9249f2b71ba30d13cbf0ead2717f287786e780f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 23 Jul 2020 11:38:05 +0300 Subject: [PATCH 093/103] Updating and using AlterTableOptions --- go/base/context.go | 7 ++++--- go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 2 +- go/sql/parser.go | 18 ++++++++++++------ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 1030463..cee66ef 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -77,9 +77,10 @@ func NewThrottleCheckResult(throttle bool, reason string, reasonHint ThrottleRea type MigrationContext struct { Uuid string - DatabaseName string - OriginalTableName string - AlterStatement string + DatabaseName string + OriginalTableName string + AlterStatement string + AlterStatementOptions string // anything following the 'ALTER TABLE [schema.]table' from AlterStatement CountTableRows bool ConcurrentCountTableRows bool diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 31729a5..d287d11 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -177,6 +177,7 @@ func main() { log.Fatalf("--alter must be provided and statement must not be empty") } parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement) + migrationContext.AlterStatementOptions = parser.GetAlterStatementOptions() if migrationContext.DatabaseName == "" { if parser.HasExplicitSchema() { diff --git a/go/logic/applier.go b/go/logic/applier.go index 926023f..3b1b9bf 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -190,7 +190,7 @@ func (this *Applier) AlterGhost() error { query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s %s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), - this.migrationContext.AlterStatement, + this.migrationContext.AlterStatementOptions, ) this.migrationContext.Log.Infof("Altering ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), diff --git a/go/sql/parser.go b/go/sql/parser.go index d050fcd..f7333a7 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -33,7 +33,8 @@ type AlterTableParser struct { droppedColumns map[string]bool isRenameTable bool - alterTokens []string + alterStatementOptions string + alterTokens []string explicitSchema string explicitTable string @@ -120,22 +121,23 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) { func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) { + this.alterStatementOptions = alterStatement for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps { - if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { this.explicitSchema = submatch[1] this.explicitTable = submatch[2] - alterStatement = submatch[3] + this.alterStatementOptions = submatch[3] break } } for _, alterTableRegexp := range alterTableExplicitTableRegexps { - if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { this.explicitTable = submatch[1] - alterStatement = submatch[2] + this.alterStatementOptions = submatch[2] break } } - alterTokens, _ := this.tokenizeAlterStatement(alterStatement) + alterTokens, _ := this.tokenizeAlterStatement(this.alterStatementOptions) for _, alterToken := range alterTokens { alterToken = this.sanitizeQuotesFromAlterStatement(alterToken) this.parseAlterToken(alterToken) @@ -180,3 +182,7 @@ func (this *AlterTableParser) GetExplicitTable() string { func (this *AlterTableParser) HasExplicitTable() bool { return this.GetExplicitTable() != "" } + +func (this *AlterTableParser) GetAlterStatementOptions() string { + return this.alterStatementOptions +} From 731df3cd152656ac21781bbdcf43e397542042e6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 23 Jul 2020 14:04:14 +0300 Subject: [PATCH 094/103] comments --- go/sql/parser.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/sql/parser.go b/go/sql/parser.go index f7333a7..ebb8b38 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -17,13 +17,19 @@ var ( dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{ + // ALTER TABLE `scm`.`tbl` something regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE `scm`.tbl something regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`), + // ALTER TABLE scm.`tbl` something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE scm.tbl something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`), } alterTableExplicitTableRegexps = []*regexp.Regexp{ + // ALTER TABLE `tbl` something regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE tbl something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`), } ) From ae4dd1867a230c2864538d3e45adbf2a2804c4ff Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 29 Jul 2020 15:06:13 +0300 Subject: [PATCH 095/103] extra unit test checks --- go/sql/parser_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 626fe75..79faa63 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -22,6 +22,7 @@ func TestParseAlterStatement(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) } @@ -30,6 +31,7 @@ func TestParseAlterStatementTrivialRename(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 1) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") @@ -40,6 +42,7 @@ func TestParseAlterStatementTrivialRenames(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 2) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") @@ -61,6 +64,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) renames := parser.GetNonTrivialRenames() test.S(t).ExpectEquals(len(renames), 2) test.S(t).ExpectEquals(renames["i"], "count") @@ -136,6 +140,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { statement := "drop column b, drop key c_idx, drop column `d`" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectEquals(len(parser.droppedColumns), 2) test.S(t).ExpectTrue(parser.droppedColumns["b"]) test.S(t).ExpectTrue(parser.droppedColumns["d"]) @@ -181,6 +186,7 @@ func TestParseAlterStatementRenameTable(t *testing.T) { statement := "drop column b, rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectTrue(parser.isRenameTable) } { @@ -208,6 +214,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "") test.S(t).ExpectEquals(parser.explicitTable, "") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -217,6 +224,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -226,6 +234,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -235,6 +244,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm with spaces") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -244,6 +254,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl with spaces") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -253,6 +264,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -262,6 +274,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -271,6 +284,17 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } + { + parser := NewAlterTableParser() + statement := "alter table scm.tbl drop column b, add index idx(i)" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b, add index idx(i)") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b", "add index idx(i)"})) + } } From 4bbc8deb774c16fa4d5375a2c61c03ed22554460 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 13 Aug 2020 15:50:38 +0200 Subject: [PATCH 096/103] Fix tabs from merge conflict --- go/logic/applier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 22cb6bc..df57160 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -371,7 +371,7 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { } this.migrationContext.Log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) - err = rows.Err() + err = rows.Err() return err } @@ -394,7 +394,7 @@ func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { } this.migrationContext.Log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) - err = rows.Err() + err = rows.Err() return err } From 25d28855b27bb4519d6951eb236118b1c0e44a15 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 19 Aug 2020 20:21:40 +0200 Subject: [PATCH 097/103] Copy of PR https://github.com/github/gh-ost/pull/861 --- Dockerfile.packaging | 2 +- Dockerfile.test | 2 +- README.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 74aa190..9c5cd29 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,6 @@ # -FROM golang:1.14.4 +FROM golang:1.14.7 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index 3b3bfe5..8f56be3 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.14.4 +FROM golang:1.14.7 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/README.md b/README.md index 7b2bc1d..d4a17e9 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,4 @@ Generally speaking, `master` branch is stable, but only [releases](https://githu - [@shlomi-noach](https://github.com/shlomi-noach) - [@jessbreckenridge](https://github.com/jessbreckenridge) - [@gtowey](https://github.com/gtowey) +- [@timvaillancourt](https://github.com/timvaillancourt) From 42bd9187c1cf93b37438ea6dae9ca0a93d14c863 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 19 Aug 2020 20:27:11 +0200 Subject: [PATCH 098/103] Use 1.14.7 --- script/ensure-go-installed | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/ensure-go-installed b/script/ensure-go-installed index 98ccdc1..baa5bd7 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,13 +1,13 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.14.4 +PREFERRED_GO_VERSION=go1.14.7 SUPPORTED_GO_VERSIONS='go1.1[456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=b518f21f823759ee30faddb1f623810a432499f050c9338777523d9c8551c62c +GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz -GO_PKG_LINUX_SHA=aed845e4185a0b2a3c3d5e1d0a35491702c55889192bb9c30e67a3de6849c067 +GO_PKG_LINUX_SHA=4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5 export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" cd $ROOTDIR From eef81c0f733889f93f11499ffede45dc3c2a7c30 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 20 Aug 2020 16:00:31 +0200 Subject: [PATCH 099/103] Update RELEASE_VERSION to 1.1.0 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index f133985..9084fa2 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.49 +1.1.0 From 69ce8e869a75058271d3bb3d53e877296ec81810 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 20 Aug 2020 20:45:46 +0200 Subject: [PATCH 100/103] Fix/hack build-deploy-tarball to allow buster builds --- script/build-deploy-tarball | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball index d36b6c9..31e0cf8 100755 --- a/script/build-deploy-tarball +++ b/script/build-deploy-tarball @@ -28,8 +28,10 @@ gzip $tarball mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ -### HACK HACK HACK ### -# blame @carlosmn and @mattr- -# Allow builds on stretch to also be used for jessie +### HACK HACK HACK HACK ### +# blame @carlosmn, @mattr and @timvaillancourt- +# Allow builds on buster to also be used for streth + jessie +stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/) jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-stretch-/-jessie-/) +cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz" cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" From bcaffc11a2f12cc1b8a28357fe5fb47f3301a497 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 20 Aug 2020 20:49:51 +0200 Subject: [PATCH 101/103] cp jessie package too --- script/build-deploy-tarball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball index 31e0cf8..c96c28b 100755 --- a/script/build-deploy-tarball +++ b/script/build-deploy-tarball @@ -32,6 +32,6 @@ cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ # blame @carlosmn, @mattr and @timvaillancourt- # Allow builds on buster to also be used for streth + jessie stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/) -jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-stretch-/-jessie-/) +jessie_tarball_name=$(echo $(basename "${stretch_tarball_name}") | sed s/-stretch-/-jessie-/) cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz" cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" From ee9174550a2351c2d28b7017a88574e2f3dec8bf Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 20 Aug 2020 20:57:33 +0200 Subject: [PATCH 102/103] Fix typo --- script/build-deploy-tarball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball index c96c28b..95da838 100755 --- a/script/build-deploy-tarball +++ b/script/build-deploy-tarball @@ -30,7 +30,7 @@ cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ ### HACK HACK HACK HACK ### # blame @carlosmn, @mattr and @timvaillancourt- -# Allow builds on buster to also be used for streth + jessie +# Allow builds on buster to also be used for stretch + jessie stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/) jessie_tarball_name=$(echo $(basename "${stretch_tarball_name}") | sed s/-stretch-/-jessie-/) cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz" From 0e2d33ad86c11a26abda216027e78831646d04c4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 20 Oct 2020 16:08:49 +0200 Subject: [PATCH 103/103] Merge in https://github.com/github/gh-ost/pull/755 --- go/logic/applier.go | 14 +++++++++----- go/logic/migrator.go | 8 ++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 089978c..8aa1c9c 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -17,6 +17,7 @@ import ( "github.com/github/gh-ost/go/sql" "github.com/outbrain/golib/sqlutils" + "sync" ) const ( @@ -787,7 +788,7 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error { } // AtomicCutOverMagicLock -func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error) error { +func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error, dropCutOverSentryTableOnce *sync.Once) error { tx, err := this.db.Begin() if err != nil { tableLocked <- err @@ -865,10 +866,13 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - if _, err := tx.Exec(query); err != nil { - this.migrationContext.Log.Errore(err) - // We DO NOT return here because we must `UNLOCK TABLES`! - } + + dropCutOverSentryTableOnce.Do(func() { + if _, err := tx.Exec(query); err != nil { + this.migrationContext.Log.Errore(err) + // We DO NOT return here because we must `UNLOCK TABLES`! + } + }) // Tables still locked this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 70af08d..85dae29 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -11,6 +11,7 @@ import ( "math" "os" "strings" + "sync" "sync/atomic" "time" @@ -606,9 +607,12 @@ func (this *Migrator) atomicCutOver() (err error) { defer atomic.StoreInt64(&this.migrationContext.InCutOverCriticalSectionFlag, 0) okToUnlockTable := make(chan bool, 4) + var dropCutOverSentryTableOnce sync.Once defer func() { okToUnlockTable <- true - this.applier.DropAtomicCutOverSentryTableIfExists() + dropCutOverSentryTableOnce.Do(func() { + this.applier.DropAtomicCutOverSentryTableIfExists() + }) }() atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 0) @@ -617,7 +621,7 @@ func (this *Migrator) atomicCutOver() (err error) { tableLocked := make(chan error, 2) tableUnlocked := make(chan error, 2) go func() { - if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil { + if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked, &dropCutOverSentryTableOnce); err != nil { this.migrationContext.Log.Errore(err) } }()