diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec8978d..3556e1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - - name: Set up Go 1.12 + - name: Set up Go 1.14 uses: actions/setup-go@v1 with: - version: 1.12 - id: go + go-version: 1.14 - name: Build run: script/cibuild + + - name: Upload gh-ost binary artifact + uses: actions/upload-artifact@v1 + with: + name: gh-ost + path: bin/gh-ost diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index d775131..31e2052 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -8,13 +8,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - - name: Set up Go 1.12 + - name: Set up Go 1.14 uses: actions/setup-go@v1 with: - version: 1.12 - id: go + go-version: 1.14 - name: migration tests run: script/cibuild-gh-ost-replica-tests diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 214c70c..9c5cd29 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,6 @@ # -FROM golang:1.12.6 +FROM golang:1.14.7 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index 6abc4a0..8f56be3 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.12.1 +FROM golang:1.14.7 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/README.md b/README.md index a27b143..d496e08 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d [Download latest release here](https://github.com/github/gh-ost/releases/latest) -`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either: +`gh-ost` is a Go project; it is built with Go `1.14` and above. To build on your own, use either: - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary. - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost` @@ -110,3 +110,4 @@ Generally speaking, `master` branch is stable, but only [releases](https://githu - [@shlomi-noach](https://github.com/shlomi-noach) - [@jessbreckenridge](https://github.com/jessbreckenridge) - [@gtowey](https://github.com/gtowey) +- [@timvaillancourt](https://github.com/timvaillancourt) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index f133985..9084fa2 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.49 +1.1.0 diff --git a/build.sh b/build.sh index 46db9c2..b5d4659 100755 --- a/build.sh +++ b/build.sh @@ -18,8 +18,8 @@ function build { GOOS=$3 GOARCH=$4 - if ! go version | egrep -q 'go(1\.1[234])' ; then - echo "go version must be 1.12 or above" + if ! go version | egrep -q 'go(1\.1[456])' ; then + echo "go version must be 1.14 or above" exit 1 fi diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 0cd25ed..e09ae4f 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -51,4 +51,5 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles. +- [Encrypted binary logs](https://www.percona.com/blog/2018/03/08/binlog-encryption-percona-server-mysql/) are not supported. - `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). diff --git a/go/base/context.go b/go/base/context.go index 15cae60..3211067 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -19,6 +19,7 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" + "github.com/outbrain/golib/log" "gopkg.in/gcfg.v1" gcfgscanner "gopkg.in/gcfg.v1/scanner" @@ -76,9 +77,10 @@ func NewThrottleCheckResult(throttle bool, reason string, reasonHint ThrottleRea type MigrationContext struct { Uuid string - DatabaseName string - OriginalTableName string - AlterStatement string + DatabaseName string + OriginalTableName string + AlterStatement string + AlterStatementOptions string // anything following the 'ALTER TABLE [schema.]table' from AlterStatement CountTableRows bool ConcurrentCountTableRows bool @@ -218,6 +220,25 @@ type MigrationContext struct { ForceTmpTableName string recentBinlogCoordinates mysql.BinlogCoordinates + + Log Logger +} + +type Logger interface { + Debug(args ...interface{}) + Debugf(format string, args ...interface{}) + Info(args ...interface{}) + Infof(format string, args ...interface{}) + Warning(args ...interface{}) error + Warningf(format string, args ...interface{}) error + Error(args ...interface{}) error + Errorf(format string, args ...interface{}) error + Errore(err error) error + Fatal(args ...interface{}) error + Fatalf(format string, args ...interface{}) error + Fatale(err error) error + SetLevel(level log.LogLevel) + SetPrintStackTrace(printStackTraceFlag bool) } type ContextConfig struct { @@ -252,6 +273,7 @@ func NewMigrationContext() *MigrationContext { pointOfInterestTimeMutex: &sync.Mutex{}, ColumnRenameMap: make(map[string]string), PanicAbort: make(chan error), + Log: NewDefaultLogger(), } } diff --git a/go/base/default_logger.go b/go/base/default_logger.go new file mode 100644 index 0000000..be6b1f2 --- /dev/null +++ b/go/base/default_logger.go @@ -0,0 +1,73 @@ +package base + +import ( + "github.com/outbrain/golib/log" +) + +type simpleLogger struct{} + +func NewDefaultLogger() *simpleLogger { + return &simpleLogger{} +} + +func (*simpleLogger) Debug(args ...interface{}) { + log.Debug(args[0].(string), args[1:]) + return +} + +func (*simpleLogger) Debugf(format string, args ...interface{}) { + log.Debugf(format, args...) + return +} + +func (*simpleLogger) Info(args ...interface{}) { + log.Info(args[0].(string), args[1:]) + return +} + +func (*simpleLogger) Infof(format string, args ...interface{}) { + log.Infof(format, args...) + return +} + +func (*simpleLogger) Warning(args ...interface{}) error { + return log.Warning(args[0].(string), args[1:]) +} + +func (*simpleLogger) Warningf(format string, args ...interface{}) error { + return log.Warningf(format, args...) +} + +func (*simpleLogger) Error(args ...interface{}) error { + return log.Error(args[0].(string), args[1:]) +} + +func (*simpleLogger) Errorf(format string, args ...interface{}) error { + return log.Errorf(format, args...) +} + +func (*simpleLogger) Errore(err error) error { + return log.Errore(err) +} + +func (*simpleLogger) Fatal(args ...interface{}) error { + return log.Fatal(args[0].(string), args[1:]) +} + +func (*simpleLogger) Fatalf(format string, args ...interface{}) error { + return log.Fatalf(format, args...) +} + +func (*simpleLogger) Fatale(err error) error { + return log.Fatale(err) +} + +func (*simpleLogger) SetLevel(level log.LogLevel) { + log.SetLevel(level) + return +} + +func (*simpleLogger) SetPrintStackTrace(printStackTraceFlag bool) { + log.SetPrintStackTrace(printStackTraceFlag) + return +} diff --git a/go/base/utils.go b/go/base/utils.go index 852f1ab..65e47f0 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -14,7 +14,6 @@ import ( gosql "database/sql" "github.com/github/gh-ost/go/mysql" - "github.com/outbrain/golib/log" ) var ( @@ -87,7 +86,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, } if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) { - log.Infof("connection validated on %+v", connectionConfig.Key) + migrationContext.Log.Infof("connection validated on %+v", connectionConfig.Key) return version, nil } else if extraPort == 0 { return "", fmt.Errorf("Unexpected database port reported: %+v", port) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 41d1ead..bc80cb5 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -13,13 +13,13 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" gomysql "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "golang.org/x/net/context" ) type GoMySQLReader struct { + migrationContext *base.MigrationContext connectionConfig *mysql.ConnectionConfig binlogSyncer *replication.BinlogSyncer binlogStreamer *replication.BinlogStreamer @@ -30,6 +30,7 @@ type GoMySQLReader struct { func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *GoMySQLReader, err error) { binlogReader = &GoMySQLReader{ + migrationContext: migrationContext, connectionConfig: migrationContext.InspectorConnectionConfig, currentCoordinates: mysql.BinlogCoordinates{}, currentCoordinatesMutex: &sync.Mutex{}, @@ -57,11 +58,11 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go // ConnectBinlogStreamer func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordinates) (err error) { if coordinates.IsEmpty() { - return log.Errorf("Empty coordinates at ConnectBinlogStreamer()") + return this.migrationContext.Log.Errorf("Empty coordinates at ConnectBinlogStreamer()") } this.currentCoordinates = coordinates - log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) + this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) // Start sync with specified binlog file and position this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{this.currentCoordinates.LogFile, uint32(this.currentCoordinates.LogPos)}) @@ -78,7 +79,7 @@ func (this *GoMySQLReader) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinate // StreamEvents func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error { if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) { - log.Debugf("Skipping handled query at %+v", this.currentCoordinates) + this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates) return nil } @@ -147,14 +148,14 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates.LogFile = string(rotateEvent.NextLogName) }() - log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName) + this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName) } else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok { if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil { return err } } } - log.Debugf("done streaming events") + this.migrationContext.Log.Debugf("done streaming events") return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index e7c05ba..b8557f9 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -14,6 +14,7 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" + "github.com/github/gh-ost/go/sql" _ "github.com/go-sql-driver/mysql" "github.com/outbrain/golib/log" @@ -31,7 +32,7 @@ func acceptSignals(migrationContext *base.MigrationContext) { for sig := range c { switch sig { case syscall.SIGHUP: - log.Infof("Received SIGHUP. Reloading configuration") + migrationContext.Log.Infof("Received SIGHUP. Reloading configuration") if err := migrationContext.ReadConfigFile(); err != nil { log.Errore(err) } else { @@ -48,6 +49,7 @@ func main() { flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") + flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user") flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password") flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host") @@ -158,69 +160,80 @@ func main() { return } - log.SetLevel(log.ERROR) + migrationContext.Log.SetLevel(log.ERROR) if *verbose { - log.SetLevel(log.INFO) + migrationContext.Log.SetLevel(log.INFO) } if *debug { - log.SetLevel(log.DEBUG) + migrationContext.Log.SetLevel(log.DEBUG) } if *stack { - log.SetPrintStackTrace(*stack) + migrationContext.Log.SetPrintStackTrace(*stack) } if *quiet { // Override!! - log.SetLevel(log.ERROR) + migrationContext.Log.SetLevel(log.ERROR) } - if migrationContext.DatabaseName == "" { - log.Fatalf("--database must be provided and database name must not be empty") - } - if migrationContext.OriginalTableName == "" { - log.Fatalf("--table must be provided and table name must not be empty") - } if migrationContext.AlterStatement == "" { log.Fatalf("--alter must be provided and statement must not be empty") } + parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement) + migrationContext.AlterStatementOptions = parser.GetAlterStatementOptions() + + if migrationContext.DatabaseName == "" { + if parser.HasExplicitSchema() { + migrationContext.DatabaseName = parser.GetExplicitSchema() + } else { + log.Fatalf("--database must be provided and database name must not be empty, or --alter must specify database name") + } + } + if migrationContext.OriginalTableName == "" { + if parser.HasExplicitTable() { + migrationContext.OriginalTableName = parser.GetExplicitTable() + } else { + log.Fatalf("--table must be provided and table name must not be empty, or --alter must specify table name") + } + } migrationContext.Noop = !(*executeFlag) if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica { - log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") } if migrationContext.AllowedRunningOnMaster && migrationContext.MigrateOnReplica { - log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive") } if migrationContext.MigrateOnReplica && migrationContext.TestOnReplica { - log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive") } if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR { - log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") + migrationContext.Log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") } if migrationContext.TestOnReplicaSkipReplicaStop { if !migrationContext.TestOnReplica { - log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") + migrationContext.Log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") } - log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") + migrationContext.Log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") } if migrationContext.CliMasterUser != "" && migrationContext.AssumeMasterHostname == "" { - log.Fatalf("--master-user requires --assume-master-host") + migrationContext.Log.Fatalf("--master-user requires --assume-master-host") } if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" { - log.Fatalf("--master-password requires --assume-master-host") + migrationContext.Log.Fatalf("--master-password requires --assume-master-host") } if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-ca requires --ssl") + migrationContext.Log.Fatalf("--ssl-ca requires --ssl") } if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-cert requires --ssl") + migrationContext.Log.Fatalf("--ssl-cert requires --ssl") } if migrationContext.TLSKey != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-key requires --ssl") + migrationContext.Log.Fatalf("--ssl-key requires --ssl") } if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS { - log.Fatalf("--ssl-allow-insecure requires --ssl") + migrationContext.Log.Fatalf("--ssl-allow-insecure requires --ssl") } if *replicationLagQuery != "" { - log.Warningf("--replication-lag-query is deprecated") + migrationContext.Log.Warningf("--replication-lag-query is deprecated") } switch *cutOver { @@ -229,19 +242,19 @@ func main() { case "two-step": migrationContext.CutOverType = base.CutOverTwoStep default: - log.Fatalf("Unknown cut-over: %s", *cutOver) + migrationContext.Log.Fatalf("Unknown cut-over: %s", *cutOver) } if err := migrationContext.ReadConfigFile(); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadThrottleControlReplicaKeys(*throttleControlReplicas); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadMaxLoad(*maxLoad); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadCriticalLoad(*criticalLoad); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if migrationContext.ServeSocketFile == "" { migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName) @@ -250,7 +263,7 @@ func main() { fmt.Println("Password:") bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) if err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } migrationContext.CliPassword = string(bytePassword) } @@ -265,13 +278,13 @@ func main() { migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() if err := migrationContext.SetupTLS(); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { - log.Errore(err) + migrationContext.Log.Errore(err) } if err := migrationContext.SetExponentialBackoffMaxInterval(*exponentialBackoffMaxInterval); err != nil { - log.Errore(err) + migrationContext.Log.Errore(err) } log.Infof("starting gh-ost %+v", AppVersion) @@ -281,7 +294,7 @@ func main() { err := migrator.Migrate() if err != nil { migrator.ExecOnFailureHook() - log.Fatale(err) + migrationContext.Log.Fatale(err) } fmt.Fprintf(os.Stdout, "# Done\n") } diff --git a/go/logic/applier.go b/go/logic/applier.go index 85265c8..821b4d9 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -16,8 +16,8 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" + "sync" ) const ( @@ -99,7 +99,7 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.readTableColumns(); err != nil { return err } - log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion) + this.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion) return nil } @@ -110,13 +110,13 @@ func (this *Applier) validateAndReadTimeZone() error { return err } - log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone) + this.migrationContext.Log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone) return nil } // readTableColumns reads table columns on applier func (this *Applier) readTableColumns() (err error) { - log.Infof("Examining table structure on applier") + this.migrationContext.Log.Infof("Examining table structure on applier") this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) if err != nil { return err @@ -157,7 +157,7 @@ func (this *Applier) ValidateOrDropExistingTables() error { } } if len(this.migrationContext.GetOldTableName()) > mysql.MaxTableNameLength { - log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength) + this.migrationContext.Log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength) } if this.tableExists(this.migrationContext.GetOldTableName()) { @@ -175,14 +175,14 @@ func (this *Applier) CreateGhostTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Creating ghost table %s.%s", + this.migrationContext.Log.Infof("Creating ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Ghost table created") + this.migrationContext.Log.Infof("Ghost table created") return nil } @@ -191,17 +191,17 @@ func (this *Applier) AlterGhost() error { query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s %s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), - this.migrationContext.AlterStatement, + this.migrationContext.AlterStatementOptions, ) - log.Infof("Altering ghost table %s.%s", + this.migrationContext.Log.Infof("Altering ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) - log.Debugf("ALTER statement: %s", query) + this.migrationContext.Log.Debugf("ALTER statement: %s", query) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Ghost table altered") + this.migrationContext.Log.Infof("Ghost table altered") return nil } @@ -222,14 +222,14 @@ func (this *Applier) CreateChangelogTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetChangelogTableName()), ) - log.Infof("Creating changelog table %s.%s", + this.migrationContext.Log.Infof("Creating changelog table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetChangelogTableName()), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Changelog table created") + this.migrationContext.Log.Infof("Changelog table created") return nil } @@ -239,14 +239,14 @@ func (this *Applier) dropTable(tableName string) error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) - log.Infof("Dropping table %s.%s", + this.migrationContext.Log.Infof("Dropping table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Table dropped") + this.migrationContext.Log.Infof("Table dropped") return nil } @@ -313,7 +313,7 @@ func (this *Applier) InitiateHeartbeat() { if _, err := this.WriteChangelog("heartbeat", time.Now().Format(time.RFC3339Nano)); err != nil { numSuccessiveFailures++ if numSuccessiveFailures > this.migrationContext.MaxRetries() { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } else { numSuccessiveFailures = 0 @@ -348,14 +348,14 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) { } var result int64 if err := this.db.QueryRow(throttleQuery).Scan(&result); err != nil { - return 0, log.Errore(err) + return 0, this.migrationContext.Log.Errore(err) } return result, nil } // ReadMigrationMinValues returns the minimum values to be iterated on rowcopy func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { - log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) + this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) if err != nil { return err @@ -370,13 +370,15 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { return err } } - log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) + this.migrationContext.Log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) + + err = rows.Err() return err } // ReadMigrationMaxValues returns the maximum values to be iterated on rowcopy func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { - log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) + this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) if err != nil { return err @@ -391,7 +393,9 @@ func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { return err } } - log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) + this.migrationContext.Log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) + + err = rows.Err() return err } @@ -444,12 +448,15 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo } hasFurtherRange = true } + if err = rows.Err(); err != nil { + return hasFurtherRange, err + } if hasFurtherRange { this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues return hasFurtherRange, nil } } - log.Debugf("Iteration complete: no further range to iterate") + this.migrationContext.Log.Debugf("Iteration complete: no further range to iterate") return hasFurtherRange, nil } @@ -507,7 +514,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } rowsAffected, _ = sqlResult.RowsAffected() duration = time.Since(startTime) - log.Debugf( + this.migrationContext.Log.Debugf( "Issued INSERT on range: [%s]..[%s]; iteration: %d; chunk-size: %d", this.migrationContext.MigrationIterationRangeMinValues, this.migrationContext.MigrationIterationRangeMaxValues, @@ -522,7 +529,7 @@ func (this *Applier) LockOriginalTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Locking %s.%s", + this.migrationContext.Log.Infof("Locking %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) @@ -530,18 +537,18 @@ func (this *Applier) LockOriginalTable() error { if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err } - log.Infof("Table locked") + this.migrationContext.Log.Infof("Table locked") return nil } // UnlockTables makes tea. No wait, it unlocks tables. func (this *Applier) UnlockTables() error { query := `unlock /* gh-ost */ tables` - log.Infof("Unlocking tables") + this.migrationContext.Log.Infof("Unlocking tables") if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err } - log.Infof("Tables unlocked") + this.migrationContext.Log.Infof("Tables unlocked") return nil } @@ -555,7 +562,7 @@ func (this *Applier) SwapTablesQuickAndBumpy() error { sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - log.Infof("Renaming original table") + this.migrationContext.Log.Infof("Renaming original table") this.migrationContext.RenameTablesStartTime = time.Now() if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err @@ -565,13 +572,13 @@ func (this *Applier) SwapTablesQuickAndBumpy() error { sql.EscapeName(this.migrationContext.GetGhostTableName()), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming ghost table") + this.migrationContext.Log.Infof("Renaming ghost table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } this.migrationContext.RenameTablesEndTime = time.Now() - log.Infof("Tables renamed") + this.migrationContext.Log.Infof("Tables renamed") return nil } @@ -590,7 +597,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming back both tables") + this.migrationContext.Log.Infof("Renaming back both tables") if _, err := sqlutils.ExecNoPrepare(this.db, query); err == nil { return nil } @@ -601,7 +608,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) - log.Infof("Renaming back to ghost table") + this.migrationContext.Log.Infof("Renaming back to ghost table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { renameError = err } @@ -611,11 +618,11 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming back to original table") + this.migrationContext.Log.Infof("Renaming back to original table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { renameError = err } - return log.Errore(renameError) + return this.migrationContext.Log.Errore(renameError) } // StopSlaveIOThread is applicable with --test-on-replica; it stops the IO thread, duh. @@ -623,44 +630,44 @@ func (this *Applier) RenameTablesRollback() (renameError error) { // and have them written to the binary log, so that we can then read them via streamer. func (this *Applier) StopSlaveIOThread() error { query := `stop /* gh-ost */ slave io_thread` - log.Infof("Stopping replication IO thread") + this.migrationContext.Log.Infof("Stopping replication IO thread") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Replication IO thread stopped") + this.migrationContext.Log.Infof("Replication IO thread stopped") return nil } // StartSlaveIOThread is applicable with --test-on-replica func (this *Applier) StartSlaveIOThread() error { query := `start /* gh-ost */ slave io_thread` - log.Infof("Starting replication IO thread") + this.migrationContext.Log.Infof("Starting replication IO thread") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Replication IO thread started") + this.migrationContext.Log.Infof("Replication IO thread started") return nil } // StartSlaveSQLThread is applicable with --test-on-replica func (this *Applier) StopSlaveSQLThread() error { query := `stop /* gh-ost */ slave sql_thread` - log.Infof("Verifying SQL thread is stopped") + this.migrationContext.Log.Infof("Verifying SQL thread is stopped") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("SQL thread stopped") + this.migrationContext.Log.Infof("SQL thread stopped") return nil } // StartSlaveSQLThread is applicable with --test-on-replica func (this *Applier) StartSlaveSQLThread() error { query := `start /* gh-ost */ slave sql_thread` - log.Infof("Verifying SQL thread is running") + this.migrationContext.Log.Infof("Verifying SQL thread is running") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("SQL thread started") + this.migrationContext.Log.Infof("SQL thread started") return nil } @@ -677,7 +684,7 @@ func (this *Applier) StopReplication() error { if err != nil { return err } - log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates) + this.migrationContext.Log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates) return nil } @@ -689,7 +696,7 @@ func (this *Applier) StartReplication() error { if err := this.StartSlaveSQLThread(); err != nil { return err } - log.Infof("Replication started") + this.migrationContext.Log.Infof("Replication started") return nil } @@ -703,7 +710,7 @@ func (this *Applier) ExpectUsedLock(sessionId int64) error { var result int64 query := `select is_used_lock(?)` lockName := this.GetSessionLockName(sessionId) - log.Infof("Checking session lock: %s", lockName) + this.migrationContext.Log.Infof("Checking session lock: %s", lockName) if err := this.db.QueryRow(query, lockName).Scan(&result); err != nil || result != sessionId { return fmt.Errorf("Session lock %s expected to be found but wasn't", lockName) } @@ -738,7 +745,7 @@ func (this *Applier) ExpectProcess(sessionId int64, stateHint, infoHint string) // DropAtomicCutOverSentryTableIfExists checks if the "old" table name // happens to be a cut-over magic table; if so, it drops it. func (this *Applier) DropAtomicCutOverSentryTableIfExists() error { - log.Infof("Looking for magic cut-over table") + this.migrationContext.Log.Infof("Looking for magic cut-over table") tableName := this.migrationContext.GetOldTableName() rowMap := this.showTableStatus(tableName) if rowMap == nil { @@ -748,7 +755,7 @@ func (this *Applier) DropAtomicCutOverSentryTableIfExists() error { if rowMap["Comment"].String != atomicCutOverMagicHint { return fmt.Errorf("Expected magic comment on %s, did not find it", tableName) } - log.Infof("Dropping magic cut-over table") + this.migrationContext.Log.Infof("Dropping magic cut-over table") return this.dropTable(tableName) } @@ -768,20 +775,20 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error { this.migrationContext.TableEngine, atomicCutOverMagicHint, ) - log.Infof("Creating magic cut-over table %s.%s", + this.migrationContext.Log.Infof("Creating magic cut-over table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Magic cut-over table created") + this.migrationContext.Log.Infof("Magic cut-over table created") return nil } // AtomicCutOverMagicLock -func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error) error { +func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error, dropCutOverSentryTableOnce *sync.Once) error { tx, err := this.db.Begin() if err != nil { tableLocked <- err @@ -804,7 +811,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke lockResult := 0 query := `select get_lock(?, 0)` lockName := this.GetSessionLockName(sessionId) - log.Infof("Grabbing voluntary lock: %s", lockName) + this.migrationContext.Log.Infof("Grabbing voluntary lock: %s", lockName) if err := tx.QueryRow(query, lockName).Scan(&lockResult); err != nil || lockResult != 1 { err := fmt.Errorf("Unable to acquire lock %s", lockName) tableLocked <- err @@ -812,7 +819,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke } tableLockTimeoutSeconds := this.migrationContext.CutOverLockTimeoutSeconds * 2 - log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds) + this.migrationContext.Log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds) query = fmt.Sprintf(`set session lock_wait_timeout:=%d`, tableLockTimeoutSeconds) if _, err := tx.Exec(query); err != nil { tableLocked <- err @@ -830,7 +837,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - log.Infof("Locking %s.%s, %s.%s", + this.migrationContext.Log.Infof("Locking %s.%s, %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.DatabaseName), @@ -841,7 +848,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke tableLocked <- err return err } - log.Infof("Tables locked") + this.migrationContext.Log.Infof("Tables locked") tableLocked <- nil // No error. // From this point on, we are committed to UNLOCK TABLES. No matter what happens, @@ -850,22 +857,25 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke // The cut-over phase will proceed to apply remaining backlog onto ghost table, // and issue RENAME. We wait here until told to proceed. <-okToUnlockTable - log.Infof("Will now proceed to drop magic table and unlock tables") + this.migrationContext.Log.Infof("Will now proceed to drop magic table and unlock tables") // The magic table is here because we locked it. And we are the only ones allowed to drop it. // And in fact, we will: - log.Infof("Dropping magic cut-over table") + this.migrationContext.Log.Infof("Dropping magic cut-over table") query = fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - if _, err := tx.Exec(query); err != nil { - log.Errore(err) - // We DO NOT return here because we must `UNLOCK TABLES`! - } + + dropCutOverSentryTableOnce.Do(func() { + if _, err := tx.Exec(query); err != nil { + this.migrationContext.Log.Errore(err) + // We DO NOT return here because we must `UNLOCK TABLES`! + } + }) // Tables still locked - log.Infof("Releasing lock from %s.%s, %s.%s", + this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.DatabaseName), @@ -874,9 +884,9 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke query = `unlock tables` if _, err := tx.Exec(query); err != nil { tableUnlocked <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } - log.Infof("Tables unlocked") + this.migrationContext.Log.Infof("Tables unlocked") tableUnlocked <- nil return nil } @@ -898,7 +908,7 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed } sessionIdChan <- sessionId - log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds) + this.migrationContext.Log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds) query := fmt.Sprintf(`set session lock_wait_timeout:=%d`, this.migrationContext.CutOverLockTimeoutSeconds) if _, err := tx.Exec(query); err != nil { return err @@ -914,13 +924,13 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Issuing and expecting this to block: %s", query) + this.migrationContext.Log.Infof("Issuing and expecting this to block: %s", query) if _, err := tx.Exec(query); err != nil { tablesRenamed <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } tablesRenamed <- nil - log.Infof("Tables renamed") + this.migrationContext.Log.Infof("Tables renamed") return nil } @@ -1026,19 +1036,19 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) }() if err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // no error atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, int64(len(dmlEvents))) if this.migrationContext.CountTableRows { atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, totalDelta) } - log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents)) + this.migrationContext.Log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents)) return nil } func (this *Applier) Teardown() { - log.Debugf("Tearing down...") + this.migrationContext.Log.Debugf("Tearing down...") this.db.Close() this.singletonDB.Close() atomic.StoreInt64(&this.finishedMigrating, 1) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index e4bbfde..95b8ad4 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -17,7 +17,6 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -69,7 +68,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.applyBinlogFormat(); err != nil { return err } - log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion) + this.migrationContext.Log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion) return nil } @@ -137,14 +136,14 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { switch column.Type { case sql.FloatColumnType: { - log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) + this.migrationContext.Log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) uniqueKeyIsValid = false } case sql.JSONColumnType: { // Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code // will remain in place to potentially handle the future case where JSON is supported in indexes. - log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) + this.migrationContext.Log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) uniqueKeyIsValid = false } } @@ -157,17 +156,17 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { if this.migrationContext.UniqueKey == nil { return fmt.Errorf("No shared unique key can be found after ALTER! Bailing out") } - log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name) + this.migrationContext.Log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name) if this.migrationContext.UniqueKey.HasNullable { if this.migrationContext.NullableUniqueKeyAllowed { - log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey) + this.migrationContext.Log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey) } else { return fmt.Errorf("Chosen key (%s) has nullable columns. Bailing out. To force this operation to continue, supply --allow-nullable-unique-key flag. Only do so if you are certain there are no actual NULL values in this key. As long as there aren't, migration should be fine. NULL values in columns of this key will corrupt migration's data", this.migrationContext.UniqueKey) } } this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) - log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) + this.migrationContext.Log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) // By fact that a non-empty unique key exists we also know the shared columns are non-empty // This additional step looks at which columns are unsigned. We could have merged this within @@ -250,19 +249,19 @@ func (this *Inspector) validateGrants() error { this.migrationContext.HasSuperPrivilege = foundSuper if foundAll { - log.Infof("User has ALL privileges") + this.migrationContext.Log.Infof("User has ALL privileges") return nil } if foundSuper && foundReplicationSlave && foundDBAll { - log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) return nil } if foundReplicationClient && foundReplicationSlave && foundDBAll { - log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) return nil } - log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll) - return log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll) + return this.migrationContext.Log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) } // restartReplication is required so that we are _certain_ the binlog format and @@ -270,7 +269,7 @@ func (this *Inspector) validateGrants() error { // It is entirely possible, for example, that the replication is using 'STATEMENT' // binlog format even as the variable says 'ROW' func (this *Inspector) restartReplication() error { - log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) masterKey, _ := mysql.GetMasterKeyFromSlaveStatus(this.connectionConfig) if masterKey == nil { @@ -289,7 +288,7 @@ func (this *Inspector) restartReplication() error { } time.Sleep(startSlavePostWaitMilliseconds) - log.Debugf("Replication restarted") + this.migrationContext.Log.Debugf("Replication restarted") return nil } @@ -309,7 +308,7 @@ func (this *Inspector) applyBinlogFormat() error { if err := this.restartReplication(); err != nil { return err } - log.Debugf("'ROW' binlog format applied") + this.migrationContext.Log.Debugf("'ROW' binlog format applied") return nil } // We already have RBR, no explicit switch @@ -347,7 +346,7 @@ func (this *Inspector) validateBinlogs() error { if countReplicas > 0 { return fmt.Errorf("%s:%d has %s binlog_format, but I'm too scared to change it to ROW because it has replicas. Bailing out", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) } - log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) + this.migrationContext.Log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) } query = `select @@global.binlog_row_image` if err := this.db.QueryRow(query).Scan(&this.migrationContext.OriginalBinlogRowImage); err != nil { @@ -359,7 +358,7 @@ func (this *Inspector) validateBinlogs() error { return fmt.Errorf("%s:%d has '%s' binlog_row_image, and only 'FULL' is supported. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogRowImage) } - log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -372,12 +371,12 @@ func (this *Inspector) validateLogSlaveUpdates() error { } if logSlaveUpdates { - log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } if this.migrationContext.IsTungsten { - log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -386,7 +385,7 @@ func (this *Inspector) validateLogSlaveUpdates() error { } if this.migrationContext.InspectorIsAlsoApplier() { - log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -413,17 +412,17 @@ func (this *Inspector) validateTable() error { return err } if !tableFound { - return log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine) - log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate) + this.migrationContext.Log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine) + this.migrationContext.Log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate) return nil } // validateTableForeignKeys makes sure no foreign keys exist on the migrated table func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error { if this.migrationContext.SkipForeignKeyChecks { - log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys") + this.migrationContext.Log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys") return nil } query := ` @@ -457,16 +456,16 @@ func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) erro return err } if numParentForeignKeys > 0 { - return log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } if numChildForeignKeys > 0 { if allowChildForeignKeys { - log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") + this.migrationContext.Log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") return nil } - return log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Debugf("Validated no foreign keys exist on table") + this.migrationContext.Log.Debugf("Validated no foreign keys exist on table") return nil } @@ -492,9 +491,9 @@ func (this *Inspector) validateTableTriggers() error { return err } if numTriggers > 0 { - return log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Debugf("Validated no triggers exist on table") + this.migrationContext.Log.Debugf("Validated no triggers exist on table") return nil } @@ -514,9 +513,9 @@ func (this *Inspector) estimateTableRowsViaExplain() error { return err } if !outputFound { - return log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate) + this.migrationContext.Log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate) return nil } @@ -525,7 +524,7 @@ func (this *Inspector) CountTableRows() error { atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 1) defer atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 0) - log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") + this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 @@ -535,7 +534,7 @@ func (this *Inspector) CountTableRows() error { atomic.StoreInt64(&this.migrationContext.RowsEstimate, rowsEstimate) this.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate - log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) + this.migrationContext.Log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) return nil } @@ -663,7 +662,7 @@ func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](* if err != nil { return uniqueKeys, err } - log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys) + this.migrationContext.Log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys) return uniqueKeys, nil } @@ -753,7 +752,7 @@ func (this *Inspector) readChangelogState(hint string) (string, error) { } func (this *Inspector) getMasterConnectionConfig() (applierConfig *mysql.ConnectionConfig, err error) { - log.Infof("Recursively searching for replication master") + this.migrationContext.Log.Infof("Recursively searching for replication master") visitedKeys := mysql.NewInstanceKeyMap() return mysql.GetMasterConnectionConfigSafe(this.connectionConfig, visitedKeys, this.migrationContext.AllowedMasterMaster) } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index b1a238f..85dae29 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -11,6 +11,7 @@ import ( "math" "os" "strings" + "sync" "sync/atomic" "time" @@ -18,8 +19,6 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - - "github.com/outbrain/golib/log" ) type ChangelogState string @@ -62,7 +61,7 @@ const ( // Migrator is the main schema migration flow manager. type Migrator struct { - parser *sql.Parser + parser *sql.AlterTableParser inspector *Inspector applier *Applier eventsStreamer *EventsStreamer @@ -90,7 +89,7 @@ type Migrator struct { func NewMigrator(context *base.MigrationContext) *Migrator { migrator := &Migrator{ migrationContext: context, - parser: sql.NewParser(), + parser: sql.NewAlterTableParser(), ghostTableMigrated: make(chan bool), firstThrottlingCollected: make(chan bool, 3), rowCopyComplete: make(chan error), @@ -216,7 +215,7 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er } changelogStateString := dmlEvent.NewColumnValues.StringColumn(3) changelogState := ReadChangelogState(changelogStateString) - log.Infof("Intercepted changelog state %s", changelogState) + this.migrationContext.Log.Infof("Intercepted changelog state %s", changelogState) switch changelogState { case GhostTableMigrated: { @@ -242,14 +241,14 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er return fmt.Errorf("Unknown changelog state: %+v", changelogState) } } - log.Infof("Handled changelog state %s", changelogState) + this.migrationContext.Log.Infof("Handled changelog state %s", changelogState) return nil } // listenOnPanicAbort aborts on abort request func (this *Migrator) listenOnPanicAbort() { err := <-this.migrationContext.PanicAbort - log.Fatale(err) + this.migrationContext.Log.Fatale(err) } // validateStatement validates the `alter` statement meets criteria. @@ -265,7 +264,7 @@ func (this *Migrator) validateStatement() (err error) { if !this.migrationContext.ApproveRenamedColumns { return fmt.Errorf("gh-ost believes the ALTER statement renames columns, as follows: %v; as precaution, you are asked to confirm gh-ost is correct, and provide with `--approve-renamed-columns`, and we're all happy. Or you can skip renamed columns via `--skip-renamed-columns`, in which case column data may be lost", this.parser.GetNonTrivialRenames()) } - log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames()) + this.migrationContext.Log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames()) } this.migrationContext.DroppedColumnsMap = this.parser.DroppedColumnsMap() return nil @@ -277,7 +276,7 @@ func (this *Migrator) countTableRows() (err error) { return nil } if this.migrationContext.Noop { - log.Debugf("Noop operation; not really counting table rows") + this.migrationContext.Log.Debugf("Noop operation; not really counting table rows") return nil } @@ -292,7 +291,7 @@ func (this *Migrator) countTableRows() (err error) { } if this.migrationContext.ConcurrentCountTableRows { - log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on") + this.migrationContext.Log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on") go countRowsFunc() // and we ignore errors, because this turns to be a background job return nil @@ -304,9 +303,9 @@ func (this *Migrator) createFlagFiles() (err error) { if this.migrationContext.PostponeCutOverFlagFile != "" { if !base.FileExists(this.migrationContext.PostponeCutOverFlagFile) { if err := base.TouchFile(this.migrationContext.PostponeCutOverFlagFile); err != nil { - return log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error()) + return this.migrationContext.Log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error()) } - log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile) + this.migrationContext.Log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile) } } return nil @@ -314,7 +313,7 @@ func (this *Migrator) createFlagFiles() (err error) { // Migrate executes the complete migration logic. This is *the* major gh-ost function. func (this *Migrator) Migrate() (err error) { - log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) this.migrationContext.StartTime = time.Now() if this.migrationContext.Hostname, err = os.Hostname(); err != nil { return err @@ -353,9 +352,9 @@ func (this *Migrator) Migrate() (err error) { } initialLag, _ := this.inspector.getReplicationLag() - log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) + this.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) <-this.ghostTableMigrated - log.Debugf("ghost table migrated") + this.migrationContext.Log.Debugf("ghost table migrated") // Yay! We now know the Ghost and Changelog tables are good to examine! // When running on replica, this means the replica has those tables. When running // on master this is always true, of course, and yet it also implies this knowledge @@ -393,9 +392,9 @@ func (this *Migrator) Migrate() (err error) { this.migrationContext.MarkRowCopyStartTime() go this.initiateStatus() - log.Debugf("Operating until row copy is complete") + this.migrationContext.Log.Debugf("Operating until row copy is complete") this.consumeRowCopyComplete() - log.Infof("Row copy complete") + this.migrationContext.Log.Infof("Row copy complete") if err := this.hooksExecutor.onRowCopyComplete(); err != nil { return err } @@ -421,7 +420,7 @@ func (this *Migrator) Migrate() (err error) { if err := this.hooksExecutor.onSuccess(); err != nil { return err } - log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -447,14 +446,14 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) { // and swap the tables. // The difference is that we will later swap the tables back. if err := this.hooksExecutor.onStartReplication(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if this.migrationContext.TestOnReplicaSkipReplicaStop { - log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.") + this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.") } else { - log.Debugf("testing on replica. Starting replication IO thread after cut-over failure") + this.migrationContext.Log.Debugf("testing on replica. Starting replication IO thread after cut-over failure") if err := this.retryOperation(this.applier.StartReplication); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } } @@ -465,16 +464,16 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) { // type (on replica? atomic? safe?) func (this *Migrator) cutOver() (err error) { if this.migrationContext.Noop { - log.Debugf("Noop operation; not really swapping tables") + this.migrationContext.Log.Debugf("Noop operation; not really swapping tables") return nil } this.migrationContext.MarkPointOfInterest() this.throttler.throttle(func() { - log.Debugf("throttling before swapping tables") + this.migrationContext.Log.Debugf("throttling before swapping tables") }) this.migrationContext.MarkPointOfInterest() - log.Debugf("checking for cut-over postpone") + this.migrationContext.Log.Debugf("checking for cut-over postpone") this.sleepWhileTrue( func() (bool, error) { if this.migrationContext.PostponeCutOverFlagFile == "" { @@ -499,7 +498,7 @@ func (this *Migrator) cutOver() (err error) { ) atomic.StoreInt64(&this.migrationContext.IsPostponingCutOver, 0) this.migrationContext.MarkPointOfInterest() - log.Debugf("checking for cut-over postpone: complete") + this.migrationContext.Log.Debugf("checking for cut-over postpone: complete") if this.migrationContext.TestOnReplica { // With `--test-on-replica` we stop replication thread, and then proceed to use @@ -510,9 +509,9 @@ func (this *Migrator) cutOver() (err error) { return err } if this.migrationContext.TestOnReplicaSkipReplicaStop { - log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") + this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") } else { - log.Debugf("testing on replica. Stopping replication IO thread") + this.migrationContext.Log.Debugf("testing on replica. Stopping replication IO thread") if err := this.retryOperation(this.applier.StopReplication); err != nil { return err } @@ -530,7 +529,7 @@ func (this *Migrator) cutOver() (err error) { this.handleCutOverResult(err) return err } - return log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType) + return this.migrationContext.Log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType) } // Inject the "AllEventsUpToLockProcessed" state hint, wait for it to appear in the binary logs, @@ -542,32 +541,32 @@ func (this *Migrator) waitForEventsUpToLock() (err error) { waitForEventsUpToLockStartTime := time.Now() allEventsUpToLockProcessedChallenge := fmt.Sprintf("%s:%d", string(AllEventsUpToLockProcessed), waitForEventsUpToLockStartTime.UnixNano()) - log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge) + this.migrationContext.Log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge) if _, err := this.applier.WriteChangelogState(allEventsUpToLockProcessedChallenge); err != nil { return err } - log.Infof("Waiting for events up to lock") + this.migrationContext.Log.Infof("Waiting for events up to lock") atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 1) for found := false; !found; { select { case <-timeout.C: { - return log.Errorf("Timeout while waiting for events up to lock") + return this.migrationContext.Log.Errorf("Timeout while waiting for events up to lock") } case state := <-this.allEventsUpToLockProcessed: { if state == allEventsUpToLockProcessedChallenge { - log.Infof("Waiting for events up to lock: got %s", state) + this.migrationContext.Log.Infof("Waiting for events up to lock: got %s", state) found = true } else { - log.Infof("Waiting for events up to lock: skipping %s", state) + this.migrationContext.Log.Infof("Waiting for events up to lock: skipping %s", state) } } } } waitForEventsUpToLockDuration := time.Since(waitForEventsUpToLockStartTime) - log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration) + this.migrationContext.Log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration) this.printStatus(ForcePrintStatusAndHintRule) return nil @@ -598,7 +597,7 @@ func (this *Migrator) cutOverTwoStep() (err error) { lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime) renameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.RenameTablesStartTime) - log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -608,9 +607,12 @@ func (this *Migrator) atomicCutOver() (err error) { defer atomic.StoreInt64(&this.migrationContext.InCutOverCriticalSectionFlag, 0) okToUnlockTable := make(chan bool, 4) + var dropCutOverSentryTableOnce sync.Once defer func() { okToUnlockTable <- true - this.applier.DropAtomicCutOverSentryTableIfExists() + dropCutOverSentryTableOnce.Do(func() { + this.applier.DropAtomicCutOverSentryTableIfExists() + }) }() atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 0) @@ -619,19 +621,19 @@ func (this *Migrator) atomicCutOver() (err error) { tableLocked := make(chan error, 2) tableUnlocked := make(chan error, 2) go func() { - if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil { - log.Errore(err) + if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked, &dropCutOverSentryTableOnce); err != nil { + this.migrationContext.Log.Errore(err) } }() if err := <-tableLocked; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } lockOriginalSessionId := <-lockOriginalSessionIdChan - log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId) + this.migrationContext.Log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId) // At this point we know the original table is locked. // We know any newly incoming DML on original table is blocked. if err := this.waitForEventsUpToLock(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // Step 2 @@ -649,7 +651,7 @@ func (this *Migrator) atomicCutOver() (err error) { } }() renameSessionId := <-renameSessionIdChan - log.Infof("Session renaming tables is %+v", renameSessionId) + this.migrationContext.Log.Infof("Session renaming tables is %+v", renameSessionId) waitForRename := func() error { if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 1 { @@ -666,13 +668,13 @@ func (this *Migrator) atomicCutOver() (err error) { return err } if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 0 { - log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)") + this.migrationContext.Log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)") } if err := this.applier.ExpectUsedLock(lockOriginalSessionId); err != nil { // Abort operation. Just make sure to drop the magic table. - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } - log.Infof("Connection holding lock on original table still exists") + this.migrationContext.Log.Infof("Connection holding lock on original table still exists") // Now that we've found the RENAME blocking, AND the locking connection still alive, // we know it is safe to proceed to release the lock @@ -681,16 +683,16 @@ func (this *Migrator) atomicCutOver() (err error) { // BAM! magic table dropped, original table lock is released // -> RENAME released -> queries on original are unblocked. if err := <-tableUnlocked; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if err := <-tablesRenamed; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } this.migrationContext.RenameTablesEndTime = time.Now() // ooh nice! We're actually truly and thankfully done lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime) - log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -736,7 +738,7 @@ func (this *Migrator) initiateInspector() (err error) { if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil { return err } - log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + this.migrationContext.Log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } else { // Forced master host. key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname) @@ -750,14 +752,14 @@ func (this *Migrator) initiateInspector() (err error) { if this.migrationContext.CliMasterPassword != "" { this.migrationContext.ApplierConnectionConfig.Password = this.migrationContext.CliMasterPassword } - log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + this.migrationContext.Log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } // validate configs if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica { if this.migrationContext.InspectorIsAlsoApplier() { return fmt.Errorf("Instructed to --test-on-replica or --migrate-on-replica, but the server we connect to doesn't seem to be a replica") } - log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", + this.migrationContext.Log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", *this.migrationContext.ApplierConnectionConfig.ImpliedKey, *this.migrationContext.InspectorConnectionConfig.ImpliedKey, ) this.migrationContext.ApplierConnectionConfig = this.migrationContext.InspectorConnectionConfig.Duplicate() @@ -998,12 +1000,12 @@ func (this *Migrator) initiateStreaming() error { ) go func() { - log.Debugf("Beginning streaming") + this.migrationContext.Log.Debugf("Beginning streaming") err := this.eventsStreamer.StreamEvents(this.canStopStreaming) if err != nil { this.migrationContext.PanicAbort <- err } - log.Debugf("Done streaming") + this.migrationContext.Log.Debugf("Done streaming") }() go func() { @@ -1038,11 +1040,11 @@ func (this *Migrator) initiateThrottler() error { this.throttler = NewThrottler(this.migrationContext, this.applier, this.inspector) go this.throttler.initiateThrottlerCollection(this.firstThrottlingCollected) - log.Infof("Waiting for first throttle metrics to be collected") + this.migrationContext.Log.Infof("Waiting for first throttle metrics to be collected") <-this.firstThrottlingCollected // replication lag <-this.firstThrottlingCollected // HTTP status <-this.firstThrottlingCollected // other, general metrics - log.Infof("First throttle metrics collected") + this.migrationContext.Log.Infof("First throttle metrics collected") go this.throttler.initiateThrottlerChecks() return nil @@ -1057,16 +1059,16 @@ func (this *Migrator) initiateApplier() error { return err } if err := this.applier.CreateChangelogTable(); err != nil { - log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") + this.migrationContext.Log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") return err } if err := this.applier.CreateGhostTable(); err != nil { - log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") + this.migrationContext.Log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") return err } if err := this.applier.AlterGhost(); err != nil { - log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out") + this.migrationContext.Log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out") return err } @@ -1080,14 +1082,14 @@ func (this *Migrator) initiateApplier() error { func (this *Migrator) iterateChunks() error { terminateRowIteration := func(err error) error { this.rowCopyComplete <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if this.migrationContext.Noop { - log.Debugf("Noop operation; not really copying data") + this.migrationContext.Log.Debugf("Noop operation; not really copying data") return terminateRowIteration(nil) } if this.migrationContext.MigrationRangeMinValues == nil { - log.Debugf("No rows found in table. Rowcopy will be implicitly empty") + this.migrationContext.Log.Debugf("No rows found in table. Rowcopy will be implicitly empty") return terminateRowIteration(nil) } @@ -1155,7 +1157,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error { if eventStruct.writeFunc != nil { if err := this.retryOperation(*eventStruct.writeFunc); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } return nil @@ -1189,13 +1191,13 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { return this.applier.ApplyDMLEventQueries(dmlEvents) } if err := this.retryOperation(applyEventFunc); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if nonDmlStructToApply != nil { // We pulled DML events from the queue, and then we hit a non-DML event. Wait! // We need to handle it! if err := handleNonDMLEventStruct(nonDmlStructToApply); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } } @@ -1207,7 +1209,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { // Both event backlog and rowcopy events are polled; the backlog events have precedence. func (this *Migrator) executeWriteFuncs() error { if this.migrationContext.Noop { - log.Debugf("Noop operation; not really executing write funcs") + this.migrationContext.Log.Debugf("Noop operation; not really executing write funcs") return nil } for { @@ -1234,7 +1236,7 @@ func (this *Migrator) executeWriteFuncs() error { copyRowsStartTime := time.Now() // Retries are handled within the copyRowsFunc if err := copyRowsFunc(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if niceRatio := this.migrationContext.GetNiceRatio(); niceRatio > 0 { copyRowsDuration := time.Since(copyRowsStartTime) @@ -1247,7 +1249,7 @@ func (this *Migrator) executeWriteFuncs() error { { // Hmmmmm... nothing in the queue; no events, but also no row copy. // This is possible upon load. Let's just sleep it over. - log.Debugf("Getting nothing in the write queue. Sleeping...") + this.migrationContext.Log.Debugf("Getting nothing in the write queue. Sleeping...") time.Sleep(time.Second) } } @@ -1263,14 +1265,14 @@ func (this *Migrator) finalCleanup() error { if this.migrationContext.Noop { if createTableStatement, err := this.inspector.showCreateTable(this.migrationContext.GetGhostTableName()); err == nil { - log.Infof("New table structure follows") + this.migrationContext.Log.Infof("New table structure follows") fmt.Println(createTableStatement) } else { - log.Errore(err) + this.migrationContext.Log.Errore(err) } } if err := this.eventsStreamer.Close(); err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } if err := this.retryOperation(this.applier.DropChangelogTable); err != nil { @@ -1282,8 +1284,8 @@ func (this *Migrator) finalCleanup() error { } } else { if !this.migrationContext.Noop { - log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:") - log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName())) + this.migrationContext.Log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:") + this.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName())) } } if this.migrationContext.Noop { @@ -1299,22 +1301,22 @@ func (this *Migrator) teardown() { atomic.StoreInt64(&this.finishedMigrating, 1) if this.inspector != nil { - log.Infof("Tearing down inspector") + this.migrationContext.Log.Infof("Tearing down inspector") this.inspector.Teardown() } if this.applier != nil { - log.Infof("Tearing down applier") + this.migrationContext.Log.Infof("Tearing down applier") this.applier.Teardown() } if this.eventsStreamer != nil { - log.Infof("Tearing down streamer") + this.migrationContext.Log.Infof("Tearing down streamer") this.eventsStreamer.Teardown() } if this.throttler != nil { - log.Infof("Tearing down throttler") + this.migrationContext.Log.Infof("Tearing down throttler") this.throttler.Teardown() } } diff --git a/go/logic/server.go b/go/logic/server.go index 7af80b5..1606884 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -16,7 +16,6 @@ import ( "sync/atomic" "github.com/github/gh-ost/go/base" - "github.com/outbrain/golib/log" ) type printStatusFunc func(PrintStatusRule, io.Writer) @@ -49,12 +48,12 @@ func (this *Server) BindSocketFile() (err error) { if err != nil { return err } - log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile) + this.migrationContext.Log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile) return nil } func (this *Server) RemoveSocketFile() (err error) { - log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile) + this.migrationContext.Log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile) return os.Remove(this.migrationContext.ServeSocketFile) } @@ -66,7 +65,7 @@ func (this *Server) BindTCPPort() (err error) { if err != nil { return err } - log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort) + this.migrationContext.Log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort) return nil } @@ -76,7 +75,7 @@ func (this *Server) Serve() (err error) { for { conn, err := this.unixListener.Accept() if err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } go this.handleConnection(conn) } @@ -88,7 +87,7 @@ func (this *Server) Serve() (err error) { for { conn, err := this.tcpListener.Accept() if err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } go this.handleConnection(conn) } @@ -118,7 +117,7 @@ func (this *Server) onServerCommand(command string, writer *bufio.Writer) (err e } else { fmt.Fprintf(writer, "%s\n", err.Error()) } - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // applyServerCommand parses and executes commands by user diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 25d9f53..5f11fd0 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -16,7 +16,6 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -160,7 +159,7 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { if !foundMasterStatus { return fmt.Errorf("Got no results from SHOW MASTER STATUS. Bailing out") } - log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates) + this.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates) return nil } @@ -186,7 +185,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { return nil } - log.Infof("StreamEvents encountered unexpected error: %+v", err) + this.migrationContext.Log.Infof("StreamEvents encountered unexpected error: %+v", err) this.migrationContext.MarkPointOfInterest() time.Sleep(ReconnectStreamerSleepSeconds * time.Second) @@ -202,7 +201,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { // Reposition at same binlog file. lastAppliedRowsEventHint = this.binlogReader.LastAppliedRowsEventHint - log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint) + this.migrationContext.Log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint) if err := this.initBinlogReader(this.GetReconnectBinlogCoordinates()); err != nil { return err } @@ -213,7 +212,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { func (this *EventsStreamer) Close() (err error) { err = this.binlogReader.Close() - log.Infof("Closed streamer connection. err=%+v", err) + this.migrationContext.Log.Infof("Closed streamer connection. err=%+v", err) return err } diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 2cf0d97..d234ea6 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -15,7 +15,6 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" ) var ( @@ -123,7 +122,7 @@ func parseChangelogHeartbeat(heartbeatValue string) (lag time.Duration, err erro // parseChangelogHeartbeat parses a string timestamp and deduces replication lag func (this *Throttler) parseChangelogHeartbeat(heartbeatValue string) (err error) { if lag, err := parseChangelogHeartbeat(heartbeatValue); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) return nil @@ -145,13 +144,13 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo // This means we will always get a good heartbeat value. // When running on replica, we should instead check the `SHOW SLAVE STATUS` output. if lag, err := mysql.GetReplicationLagFromSlaveStatus(this.inspector.informationSchemaDb); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) } } else { if heartbeatValue, err := this.inspector.readChangelogState("heartbeat"); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { this.parseChangelogHeartbeat(heartbeatValue) } @@ -348,7 +347,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error { hibernateDuration := time.Duration(this.migrationContext.CriticalLoadHibernateSeconds) * time.Second hibernateUntilTime := time.Now().Add(hibernateDuration) atomic.StoreInt64(&this.migrationContext.HibernateUntil, hibernateUntilTime.UnixNano()) - log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime) + this.migrationContext.Log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime) go func() { time.Sleep(hibernateDuration) this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(true, "leaving hibernation", base.LeavingHibernationThrottleReasonHint)) @@ -361,7 +360,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error { this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold) } if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 { - log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds) + this.migrationContext.Log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds) go func() { timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds)) <-timer.C @@ -479,6 +478,6 @@ func (this *Throttler) throttle(onThrottled func()) { } func (this *Throttler) Teardown() { - log.Debugf("Tearing down...") + this.migrationContext.Log.Debugf("Tearing down...") atomic.StoreInt64(&this.finishedMigrating, 1) } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 654998c..6855ee0 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -27,6 +27,7 @@ type ConnectionConfig struct { Password string ImpliedKey *InstanceKey tlsConfig *tls.Config + Timeout float64 } func NewConnectionConfig() *ConnectionConfig { @@ -44,6 +45,7 @@ func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionC User: this.User, Password: this.Password, tlsConfig: this.tlsConfig, + Timeout: this.Timeout, } config.ImpliedKey = &config.Key return config @@ -116,5 +118,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { if this.tlsConfig != nil { tlsOption = TLS_CONFIG_KEY } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?timeout=%fs&readTimeout=%fs&writeTimeout=%fs&interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, this.Timeout, this.Timeout, this.Timeout, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 6ce85a7..2befbc9 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") } func TestGetDBUriWithTLSSetup(t *testing.T) { @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") } diff --git a/go/sql/builder.go b/go/sql/builder.go index 2c5a7ae..4b019bc 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -492,6 +492,9 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol } setClause, err := BuildSetPreparedClause(mappedSharedColumns) + if err != nil { + return "", sharedArgs, uniqueKeyArgs, err + } equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names()) result = fmt.Sprintf(` diff --git a/go/sql/parser.go b/go/sql/parser.go index 447cbcc..ebb8b38 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -12,26 +12,54 @@ import ( ) var ( - sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')") - renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) - dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) - renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) + sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')") + renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) + dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) + renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) + alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{ + // ALTER TABLE `scm`.`tbl` something + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE `scm`.tbl something + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`), + // ALTER TABLE scm.`tbl` something + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE scm.tbl something + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`), + } + alterTableExplicitTableRegexps = []*regexp.Regexp{ + // ALTER TABLE `tbl` something + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE tbl something + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`), + } ) -type Parser struct { +type AlterTableParser struct { columnRenameMap map[string]string droppedColumns map[string]bool isRenameTable bool + + alterStatementOptions string + alterTokens []string + + explicitSchema string + explicitTable string } -func NewParser() *Parser { - return &Parser{ +func NewAlterTableParser() *AlterTableParser { + return &AlterTableParser{ columnRenameMap: make(map[string]string), droppedColumns: make(map[string]bool), } } -func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) { +func NewParserFromAlterStatement(alterStatement string) *AlterTableParser { + parser := NewAlterTableParser() + parser.ParseAlterStatement(alterStatement) + return parser +} + +func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) { terminatingQuote := rune(0) f := func(c rune) bool { switch { @@ -58,13 +86,13 @@ func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []stri return tokens, nil } -func (this *Parser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) { +func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) { strippedStatement = alterStatement strippedStatement = sanitizeQuotesRegexp.ReplaceAllString(strippedStatement, "''") return strippedStatement } -func (this *Parser) parseAlterToken(alterToken string) (err error) { +func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) { { // rename allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1) @@ -97,16 +125,34 @@ func (this *Parser) parseAlterToken(alterToken string) (err error) { return nil } -func (this *Parser) ParseAlterStatement(alterStatement string) (err error) { - alterTokens, _ := this.tokenizeAlterStatement(alterStatement) +func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) { + + this.alterStatementOptions = alterStatement + for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps { + if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { + this.explicitSchema = submatch[1] + this.explicitTable = submatch[2] + this.alterStatementOptions = submatch[3] + break + } + } + for _, alterTableRegexp := range alterTableExplicitTableRegexps { + if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { + this.explicitTable = submatch[1] + this.alterStatementOptions = submatch[2] + break + } + } + alterTokens, _ := this.tokenizeAlterStatement(this.alterStatementOptions) for _, alterToken := range alterTokens { alterToken = this.sanitizeQuotesFromAlterStatement(alterToken) this.parseAlterToken(alterToken) + this.alterTokens = append(this.alterTokens, alterToken) } return nil } -func (this *Parser) GetNonTrivialRenames() map[string]string { +func (this *AlterTableParser) GetNonTrivialRenames() map[string]string { result := make(map[string]string) for column, renamed := range this.columnRenameMap { if column != renamed { @@ -116,14 +162,33 @@ func (this *Parser) GetNonTrivialRenames() map[string]string { return result } -func (this *Parser) HasNonTrivialRenames() bool { +func (this *AlterTableParser) HasNonTrivialRenames() bool { return len(this.GetNonTrivialRenames()) > 0 } -func (this *Parser) DroppedColumnsMap() map[string]bool { +func (this *AlterTableParser) DroppedColumnsMap() map[string]bool { return this.droppedColumns } -func (this *Parser) IsRenameTable() bool { +func (this *AlterTableParser) IsRenameTable() bool { return this.isRenameTable } +func (this *AlterTableParser) GetExplicitSchema() string { + return this.explicitSchema +} + +func (this *AlterTableParser) HasExplicitSchema() bool { + return this.GetExplicitSchema() != "" +} + +func (this *AlterTableParser) GetExplicitTable() string { + return this.explicitTable +} + +func (this *AlterTableParser) HasExplicitTable() bool { + return this.GetExplicitTable() != "" +} + +func (this *AlterTableParser) GetAlterStatementOptions() string { + return this.alterStatementOptions +} diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 5d38130..79faa63 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -19,17 +19,19 @@ func init() { func TestParseAlterStatement(t *testing.T) { statement := "add column t int, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) } func TestParseAlterStatementTrivialRename(t *testing.T) { statement := "add column t int, change ts ts timestamp, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 1) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") @@ -37,9 +39,10 @@ func TestParseAlterStatementTrivialRename(t *testing.T) { func TestParseAlterStatementTrivialRenames(t *testing.T) { statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 2) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") @@ -58,9 +61,10 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { } for _, statement := range statements { - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) renames := parser.GetNonTrivialRenames() test.S(t).ExpectEquals(len(renames), 2) test.S(t).ExpectEquals(renames["i"], "count") @@ -69,7 +73,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { } func TestTokenizeAlterStatement(t *testing.T) { - parser := NewParser() + parser := NewAlterTableParser() { alterStatement := "add column t int" tokens, _ := parser.tokenizeAlterStatement(alterStatement) @@ -108,7 +112,7 @@ func TestTokenizeAlterStatement(t *testing.T) { } func TestSanitizeQuotesFromAlterStatement(t *testing.T) { - parser := NewParser() + parser := NewAlterTableParser() { alterStatement := "add column e enum('a','b','c')" strippedStatement := parser.sanitizeQuotesFromAlterStatement(alterStatement) @@ -124,7 +128,7 @@ func TestSanitizeQuotesFromAlterStatement(t *testing.T) { func TestParseAlterStatementDroppedColumns(t *testing.T) { { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -132,16 +136,17 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["b"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop key c_idx, drop column `d`" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectEquals(len(parser.droppedColumns), 2) test.S(t).ExpectTrue(parser.droppedColumns["b"]) test.S(t).ExpectTrue(parser.droppedColumns["d"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop key c_idx, drop column `d`, drop `e`, drop primary key, drop foreign key fk_1" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -151,7 +156,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["e"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop bad statement, add column i int" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -163,38 +168,133 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { func TestParseAlterStatementRenameTable(t *testing.T) { { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "engine=innodb rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "rename as something_else, engine=innodb" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } } + +func TestParseAlterStatementExplicitTable(t *testing.T) { + + { + parser := NewAlterTableParser() + statement := "drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm with spaces`.`tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm with spaces") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm`.`tbl with spaces` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl with spaces") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm`.tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table scm.`tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table scm.tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table scm.tbl drop column b, add index idx(i)" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b, add index idx(i)") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b", "add index idx(i)"})) + } +} diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball index d36b6c9..95da838 100755 --- a/script/build-deploy-tarball +++ b/script/build-deploy-tarball @@ -28,8 +28,10 @@ gzip $tarball mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ -### HACK HACK HACK ### -# blame @carlosmn and @mattr- -# Allow builds on stretch to also be used for jessie -jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-stretch-/-jessie-/) +### HACK HACK HACK HACK ### +# blame @carlosmn, @mattr and @timvaillancourt- +# Allow builds on buster to also be used for stretch + jessie +stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/) +jessie_tarball_name=$(echo $(basename "${stretch_tarball_name}") | sed s/-stretch-/-jessie-/) +cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz" cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" diff --git a/script/ensure-go-installed b/script/ensure-go-installed index c9c2ae8..baa5bd7 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,13 +1,13 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.12.6 -SUPPORTED_GO_VERSIONS='go1.1[234]' +PREFERRED_GO_VERSION=go1.14.7 +SUPPORTED_GO_VERSIONS='go1.1[456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815 +GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz -GO_PKG_LINUX_SHA=dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c +GO_PKG_LINUX_SHA=4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5 export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" cd $ROOTDIR diff --git a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/go-sql-driver/mysql/.github/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md rename to vendor/github.com/go-sql-driver/mysql/.github/CONTRIBUTING.md diff --git a/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/go-sql-driver/mysql/.travis.yml index eae311b..56fcf25 100644 --- a/vendor/github.com/go-sql-driver/mysql/.travis.yml +++ b/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -1,10 +1,10 @@ sudo: false language: go go: - - 1.9.x - 1.10.x - 1.11.x - 1.12.x + - 1.13.x - master before_install: @@ -105,6 +105,7 @@ matrix: homebrew: packages: - mysql + update: true go: 1.12.x before_install: - go get golang.org/x/tools/cmd/cover diff --git a/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/go-sql-driver/mysql/AUTHORS index bfe74c4..0896ba1 100644 --- a/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -13,6 +13,7 @@ Aaron Hopkins Achille Roussel +Alex Snast Alexey Palazhchenko Andrew Reid Arne Hormann @@ -44,6 +45,7 @@ James Harr Jeff Hodges Jeffrey Charles Jerome Meyer +Jiajia Zhong Jian Zhen Joshua Prunier Julien Lefevre @@ -62,6 +64,7 @@ Lucas Liu Luke Scott Maciej Zimnoch Michael Woolnough +Nathanial Murphy Nicola Peduzzi Olivier Mengué oscarzhao @@ -81,6 +84,7 @@ Steven Hartland Thomas Wodarek Tim Ruffles Tom Jenkinson +Vladimir Kovpak Xiangyu Hu Xiaobing Jiang Xiuming Chen @@ -90,6 +94,7 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +DigitalOcean Inc. Facebook Inc. GitHub Inc. Google Inc. diff --git a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md index 2d87d74..9cb97b3 100644 --- a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md +++ b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -1,3 +1,42 @@ +## Version 1.5 (2020-01-07) + +Changes: + + - Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017) + - Improve buffer handling (#890) + - Document potentially insecure TLS configs (#901) + - Use a double-buffering scheme to prevent data races (#943) + - Pass uint64 values without converting them to string (#838, #955) + - Update collations and make utf8mb4 default (#877, #1054) + - Make NullTime compatible with sql.NullTime in Go 1.13+ (#995) + - Removed CloudSQL support (#993, #1007) + - Add Go Module support (#1003) + +New Features: + + - Implement support of optional TLS (#900) + - Check connection liveness (#934, #964, #997, #1048, #1051, #1052) + - Implement Connector Interface (#941, #958, #1020, #1035) + +Bugfixes: + + - Mark connections as bad on error during ping (#875) + - Mark connections as bad on error during dial (#867) + - Fix connection leak caused by rapid context cancellation (#1024) + - Mark connections as bad on error during Conn.Prepare (#1030) + + +## Version 1.4.1 (2018-11-14) + +Bugfixes: + + - Fix TIME format for binary columns (#818) + - Fix handling of empty auth plugin names (#835) + - Fix caching_sha2_password with empty password (#826) + - Fix canceled context broke mysqlConn (#862) + - Fix OldAuthSwitchRequest support (#870) + - Fix Auth Response packet for cleartext password (#887) + ## Version 1.4 (2018-06-03) Changes: diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md index c6adf1d..d2627a4 100644 --- a/vendor/github.com/go-sql-driver/mysql/README.md +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.9 or higher. We aim to support the 3 latest versions of Go. + * Go 1.10 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- @@ -166,6 +166,17 @@ Sets the charset used for client-server interaction (`"SET NAMES "`). If Usage of the `charset` parameter is discouraged because it issues additional queries to the server. Unless you need the fallback behavior, please use `collation` instead. +##### `checkConnLiveness` + +``` +Type: bool +Valid Values: true, false +Default: true +``` + +On supported platforms connections retrieved from the connection pool are checked for liveness before using them. If the check fails, the respective connection is marked as bad and the query retried with another connection. +`checkConnLiveness=false` disables this liveness check of connections. + ##### `collation` ``` @@ -396,14 +407,9 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine (First Generation MySQL Server): +Google Cloud SQL on App Engine: ``` -user@cloudsql(project-id:instance-name)/dbname -``` - -Google Cloud SQL on App Engine (Second Generation MySQL Server): -``` -user@cloudsql(project-id:regionname:instance-name)/dbname +user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname ``` TCP using default port (3306) on localhost: @@ -457,13 +463,13 @@ Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-d ### Unicode support -Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. +Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. -See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. +See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. diff --git a/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/go-sql-driver/mysql/appengine.go deleted file mode 100644 index 914e662..0000000 --- a/vendor/github.com/go-sql-driver/mysql/appengine.go +++ /dev/null @@ -1,25 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build appengine - -package mysql - -import ( - "context" - "net" - - "google.golang.org/appengine/cloudsql" -) - -func init() { - RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { - // XXX: the cloudsql driver still does not export a Context-aware dialer. - return cloudsql.Dial(instance) - }) -} diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/go-sql-driver/mysql/conncheck.go index cc47aa5..024eb28 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build !windows,!appengine +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql @@ -19,35 +19,36 @@ import ( var errUnexpectedRead = errors.New("unexpected read from socket") -func connCheck(c net.Conn) error { - var ( - n int - err error - buff [1]byte - ) +func connCheck(conn net.Conn) error { + var sysErr error - sconn, ok := c.(syscall.Conn) + sysConn, ok := conn.(syscall.Conn) if !ok { return nil } - rc, err := sconn.SyscallConn() + rawConn, err := sysConn.SyscallConn() if err != nil { return err } - rerr := rc.Read(func(fd uintptr) bool { - n, err = syscall.Read(int(fd), buff[:]) + + err = rawConn.Read(func(fd uintptr) bool { + var buf [1]byte + n, err := syscall.Read(int(fd), buf[:]) + switch { + case n == 0 && err == nil: + sysErr = io.EOF + case n > 0: + sysErr = errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + sysErr = nil + default: + sysErr = err + } return true }) - switch { - case rerr != nil: - return rerr - case n == 0 && err == nil: - return io.EOF - case n > 0: - return errUnexpectedRead - case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: - return nil - default: + if err != nil { return err } + + return sysErr } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go index fd01f64..ea7fb60 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -6,12 +6,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build windows appengine +// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos package mysql import "net" -func connCheck(c net.Conn) error { +func connCheck(conn net.Conn) error { return nil } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go index b7234b0..5399551 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build go1.10,!windows +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index 565a548..b07cd76 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -12,6 +12,7 @@ import ( "context" "database/sql" "database/sql/driver" + "encoding/json" "io" "net" "strconv" @@ -154,7 +155,9 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, mc.markBadConn(err) + // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. + errLog.Print(err) + return nil, driver.ErrBadConn } stmt := &mysqlStmt{ @@ -269,6 +272,14 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin } buf = append(buf, '\'') } + case json.RawMessage: + buf = append(buf, '\'') + if mc.status&statusNoBackslashEscapes == 0 { + buf = escapeBytesBackslash(buf, v) + } else { + buf = escapeBytesQuotes(buf, v) + } + buf = append(buf, '\'') case []byte: if v == nil { buf = append(buf, "NULL"...) diff --git a/vendor/github.com/go-sql-driver/mysql/connection_test.go b/vendor/github.com/go-sql-driver/mysql/connection_test.go index 19c17ff..a6d6773 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection_test.go +++ b/vendor/github.com/go-sql-driver/mysql/connection_test.go @@ -11,6 +11,7 @@ package mysql import ( "context" "database/sql/driver" + "encoding/json" "errors" "net" "testing" @@ -36,6 +37,33 @@ func TestInterpolateParams(t *testing.T) { } } +func TestInterpolateParamsJSONRawMessage(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + buf, err := json.Marshal(struct { + Value int `json:"value"` + }{Value: 42}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + q, err := mc.interpolateParams("SELECT ?", []driver.Value{json.RawMessage(buf)}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT '{\"value\":42}'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { mc := &mysqlConn{ buf: newBuffer(nil), diff --git a/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/go-sql-driver/mysql/connector.go index 5aaaba4..d567b4e 100644 --- a/vendor/github.com/go-sql-driver/mysql/connector.go +++ b/vendor/github.com/go-sql-driver/mysql/connector.go @@ -37,17 +37,19 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { dial, ok := dials[mc.cfg.Net] dialsLock.RUnlock() if ok { - mc.netConn, err = dial(ctx, mc.cfg.Addr) + dctx := ctx + if mc.cfg.Timeout > 0 { + var cancel context.CancelFunc + dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) + defer cancel() + } + mc.netConn, err = dial(dctx, mc.cfg.Addr) } else { nd := net.Dialer{Timeout: mc.cfg.Timeout} mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) } if err != nil { - if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - errLog.Print("net.Error from Dial()': ", nerr.Error()) - return nil, driver.ErrBadConn - } return nil, err } @@ -64,6 +66,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { // Call startWatcher for context support (From Go 1.8) mc.startWatcher() if err := mc.watchCancel(ctx); err != nil { + mc.cleanup() return nil, err } defer mc.finish() diff --git a/vendor/github.com/go-sql-driver/mysql/connector_test.go b/vendor/github.com/go-sql-driver/mysql/connector_test.go new file mode 100644 index 0000000..976903c --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connector_test.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "context" + "net" + "testing" + "time" +) + +func TestConnectorReturnsTimeout(t *testing.T) { + connector := &connector{&Config{ + Net: "tcp", + Addr: "1.1.1.1:1234", + Timeout: 10 * time.Millisecond, + }} + + _, err := connector.Connect(context.Background()) + if err == nil { + t.Fatal("error expected") + } + + if nerr, ok := err.(*net.OpError); ok { + expected := "dial tcp 1.1.1.1:1234: i/o timeout" + if nerr.Error() != expected { + t.Fatalf("expected %q, got %q", expected, nerr.Error()) + } + } else { + t.Fatalf("expected %T, got %T", nerr, err) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index 1f9decf..c1bdf11 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -83,3 +83,25 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { func init() { sql.Register("mysql", &MySQLDriver{}) } + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110.go b/vendor/github.com/go-sql-driver/mysql/driver_go110.go deleted file mode 100644 index eb5a8fe..0000000 --- a/vendor/github.com/go-sql-driver/mysql/driver_go110.go +++ /dev/null @@ -1,37 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "database/sql/driver" -) - -// NewConnector returns new driver.Connector. -func NewConnector(cfg *Config) (driver.Connector, error) { - cfg = cfg.Clone() - // normalize the contents of cfg so calls to NewConnector have the same - // behavior as MySQLDriver.OpenConnector - if err := cfg.normalize(); err != nil { - return nil, err - } - return &connector{cfg: cfg}, nil -} - -// OpenConnector implements driver.DriverContext. -func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { - cfg, err := ParseDSN(dsn) - if err != nil { - return nil, err - } - return &connector{ - cfg: cfg, - }, nil -} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go b/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go deleted file mode 100644 index 19a0e59..0000000 --- a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "context" - "database/sql" - "database/sql/driver" - "fmt" - "net" - "testing" - "time" -) - -var _ driver.DriverContext = &MySQLDriver{} - -type dialCtxKey struct{} - -func TestConnectorObeysDialTimeouts(t *testing.T) { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - if !ctx.Value(dialCtxKey{}).(bool) { - return nil, fmt.Errorf("test error: query context is not propagated to our dialer") - } - return d.DialContext(ctx, prot, addr) - }) - - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) - if err != nil { - t.Fatalf("error connecting: %s", err.Error()) - } - defer db.Close() - - ctx := context.WithValue(context.Background(), dialCtxKey{}, true) - - _, err = db.ExecContext(ctx, "DO 1") - if err != nil { - t.Fatal(err) - } -} - -func configForTests(t *testing.T) *Config { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - mycnf := NewConfig() - mycnf.User = user - mycnf.Passwd = pass - mycnf.Addr = addr - mycnf.Net = prot - mycnf.DBName = dbname - return mycnf -} - -func TestNewConnector(t *testing.T) { - mycnf := configForTests(t) - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - db := sql.OpenDB(conn) - defer db.Close() - - if err := db.Ping(); err != nil { - t.Fatal(err) - } -} - -type slowConnection struct { - net.Conn - slowdown time.Duration -} - -func (sc *slowConnection) Read(b []byte) (int, error) { - time.Sleep(sc.slowdown) - return sc.Conn.Read(b) -} - -type connectorHijack struct { - driver.Connector - connErr error -} - -func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { - var conn driver.Conn - conn, cw.connErr = cw.Connector.Connect(ctx) - return conn, cw.connErr -} - -func TestConnectorTimeoutsDuringOpen(t *testing.T) { - RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, prot, addr) - if err != nil { - return nil, err - } - return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil - }) - - mycnf := configForTests(t) - mycnf.Net = "slowconn" - - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - hijack := &connectorHijack{Connector: conn} - - db := sql.OpenDB(hijack) - defer db.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - _, err = db.ExecContext(ctx, "DO 1") - if err != context.DeadlineExceeded { - t.Fatalf("ExecContext should have timed out") - } - if hijack.connErr != context.DeadlineExceeded { - t.Fatalf("(*Connector).Connect should have timed out") - } -} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_test.go b/vendor/github.com/go-sql-driver/mysql/driver_test.go index 3dee1ba..ace083d 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver_test.go +++ b/vendor/github.com/go-sql-driver/mysql/driver_test.go @@ -1874,7 +1874,7 @@ func TestDialNonRetryableNetErr(t *testing.T) { func TestDialTemporaryNetErr(t *testing.T) { testErr := netErrorMock{temporary: true} - testDialError(t, testErr, driver.ErrBadConn) + testDialError(t, testErr, testErr) } // Tests custom dial functions @@ -2994,3 +2994,172 @@ func TestRawBytesAreNotModified(t *testing.T) { } }) } + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} + +// A connection which can only be closed. +type dummyConnection struct { + net.Conn + closed bool +} + +func (d *dummyConnection) Close() error { + d.closed = true + return nil +} + +func TestConnectorTimeoutsWatchCancel(t *testing.T) { + var ( + cancel func() // Used to cancel the context just after connecting. + created *dummyConnection // The created connection. + ) + + RegisterDialContext("TestConnectorTimeoutsWatchCancel", func(ctx context.Context, addr string) (net.Conn, error) { + // Canceling at this time triggers the watchCancel error branch in Connect(). + cancel() + created = &dummyConnection{} + return created, nil + }) + + mycnf := NewConfig() + mycnf.User = "root" + mycnf.Addr = "foo" + mycnf.Net = "TestConnectorTimeoutsWatchCancel" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + if _, err := db.Conn(ctx); err != context.Canceled { + t.Errorf("got %v, want context.Canceled", err) + } + + if created == nil { + t.Fatal("no connection created") + } + if !created.closed { + t.Errorf("connection not closed") + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go index 1d9b4ab..75c8c24 100644 --- a/vendor/github.com/go-sql-driver/mysql/dsn.go +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -55,6 +55,7 @@ type Config struct { AllowCleartextPasswords bool // Allows the cleartext client side plugin AllowNativePasswords bool // Allows the native password authentication method AllowOldPasswords bool // Allows the old insecure password method + CheckConnLiveness bool // Check connections for liveness before using them ClientFoundRows bool // Return number of matching rows instead of rows changed ColumnsWithAlias bool // Prepend table alias to column names InterpolateParams bool // Interpolate placeholders into query string @@ -70,6 +71,7 @@ func NewConfig() *Config { Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, + CheckConnLiveness: true, } } @@ -148,6 +150,19 @@ func (cfg *Config) normalize() error { return nil } +func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { + buf.Grow(1 + len(name) + 1 + len(value)) + if !*hasParam { + *hasParam = true + buf.WriteByte('?') + } else { + buf.WriteByte('&') + } + buf.WriteString(name) + buf.WriteByte('=') + buf.WriteString(value) +} + // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. func (cfg *Config) FormatDSN() string { @@ -186,165 +201,75 @@ func (cfg *Config) FormatDSN() string { } if cfg.AllowCleartextPasswords { - if hasParam { - buf.WriteString("&allowCleartextPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowCleartextPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true") } if !cfg.AllowNativePasswords { - if hasParam { - buf.WriteString("&allowNativePasswords=false") - } else { - hasParam = true - buf.WriteString("?allowNativePasswords=false") - } + writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false") } if cfg.AllowOldPasswords { - if hasParam { - buf.WriteString("&allowOldPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowOldPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowOldPasswords", "true") + } + + if !cfg.CheckConnLiveness { + writeDSNParam(&buf, &hasParam, "checkConnLiveness", "false") } if cfg.ClientFoundRows { - if hasParam { - buf.WriteString("&clientFoundRows=true") - } else { - hasParam = true - buf.WriteString("?clientFoundRows=true") - } + writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } if col := cfg.Collation; col != defaultCollation && len(col) > 0 { - if hasParam { - buf.WriteString("&collation=") - } else { - hasParam = true - buf.WriteString("?collation=") - } - buf.WriteString(col) + writeDSNParam(&buf, &hasParam, "collation", col) } if cfg.ColumnsWithAlias { - if hasParam { - buf.WriteString("&columnsWithAlias=true") - } else { - hasParam = true - buf.WriteString("?columnsWithAlias=true") - } + writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true") } if cfg.InterpolateParams { - if hasParam { - buf.WriteString("&interpolateParams=true") - } else { - hasParam = true - buf.WriteString("?interpolateParams=true") - } + writeDSNParam(&buf, &hasParam, "interpolateParams", "true") } if cfg.Loc != time.UTC && cfg.Loc != nil { - if hasParam { - buf.WriteString("&loc=") - } else { - hasParam = true - buf.WriteString("?loc=") - } - buf.WriteString(url.QueryEscape(cfg.Loc.String())) + writeDSNParam(&buf, &hasParam, "loc", url.QueryEscape(cfg.Loc.String())) } if cfg.MultiStatements { - if hasParam { - buf.WriteString("&multiStatements=true") - } else { - hasParam = true - buf.WriteString("?multiStatements=true") - } + writeDSNParam(&buf, &hasParam, "multiStatements", "true") } if cfg.ParseTime { - if hasParam { - buf.WriteString("&parseTime=true") - } else { - hasParam = true - buf.WriteString("?parseTime=true") - } + writeDSNParam(&buf, &hasParam, "parseTime", "true") } if cfg.ReadTimeout > 0 { - if hasParam { - buf.WriteString("&readTimeout=") - } else { - hasParam = true - buf.WriteString("?readTimeout=") - } - buf.WriteString(cfg.ReadTimeout.String()) + writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } if cfg.RejectReadOnly { - if hasParam { - buf.WriteString("&rejectReadOnly=true") - } else { - hasParam = true - buf.WriteString("?rejectReadOnly=true") - } + writeDSNParam(&buf, &hasParam, "rejectReadOnly", "true") } if len(cfg.ServerPubKey) > 0 { - if hasParam { - buf.WriteString("&serverPubKey=") - } else { - hasParam = true - buf.WriteString("?serverPubKey=") - } - buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + writeDSNParam(&buf, &hasParam, "serverPubKey", url.QueryEscape(cfg.ServerPubKey)) } if cfg.Timeout > 0 { - if hasParam { - buf.WriteString("&timeout=") - } else { - hasParam = true - buf.WriteString("?timeout=") - } - buf.WriteString(cfg.Timeout.String()) + writeDSNParam(&buf, &hasParam, "timeout", cfg.Timeout.String()) } if len(cfg.TLSConfig) > 0 { - if hasParam { - buf.WriteString("&tls=") - } else { - hasParam = true - buf.WriteString("?tls=") - } - buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + writeDSNParam(&buf, &hasParam, "tls", url.QueryEscape(cfg.TLSConfig)) } if cfg.WriteTimeout > 0 { - if hasParam { - buf.WriteString("&writeTimeout=") - } else { - hasParam = true - buf.WriteString("?writeTimeout=") - } - buf.WriteString(cfg.WriteTimeout.String()) + writeDSNParam(&buf, &hasParam, "writeTimeout", cfg.WriteTimeout.String()) } if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { - if hasParam { - buf.WriteString("&maxAllowedPacket=") - } else { - hasParam = true - buf.WriteString("?maxAllowedPacket=") - } - buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) - + writeDSNParam(&buf, &hasParam, "maxAllowedPacket", strconv.Itoa(cfg.MaxAllowedPacket)) } // other params @@ -355,16 +280,7 @@ func (cfg *Config) FormatDSN() string { } sort.Strings(params) for _, param := range params { - if hasParam { - buf.WriteByte('&') - } else { - hasParam = true - buf.WriteByte('?') - } - - buf.WriteString(param) - buf.WriteByte('=') - buf.WriteString(url.QueryEscape(cfg.Params[param])) + writeDSNParam(&buf, &hasParam, param, url.QueryEscape(cfg.Params[param])) } } @@ -491,6 +407,14 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // Check connections for Liveness before using them + case "checkConnLiveness": + var isBool bool + cfg.CheckConnLiveness, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + // Switch "rowsAffected" mode case "clientFoundRows": var isBool bool diff --git a/vendor/github.com/go-sql-driver/mysql/dsn_test.go b/vendor/github.com/go-sql-driver/mysql/dsn_test.go index 50dc293..89815b3 100644 --- a/vendor/github.com/go-sql-driver/mysql/dsn_test.go +++ b/vendor/github.com/go-sql-driver/mysql/dsn_test.go @@ -22,55 +22,55 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { - "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, + "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, } diff --git a/vendor/github.com/go-sql-driver/mysql/go.mod b/vendor/github.com/go-sql-driver/mysql/go.mod new file mode 100644 index 0000000..fffbf6a --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/go.mod @@ -0,0 +1,3 @@ +module github.com/go-sql-driver/mysql + +go 1.10 diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime.go b/vendor/github.com/go-sql-driver/mysql/nulltime.go new file mode 100644 index 0000000..afa8a89 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime.go @@ -0,0 +1,50 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "fmt" + "time" +) + +// Scan implements the Scanner interface. +// The value type must be time.Time or string / []byte (formatted time-string), +// otherwise Scan fails. +func (nt *NullTime) Scan(value interface{}) (err error) { + if value == nil { + nt.Time, nt.Valid = time.Time{}, false + return + } + + switch v := value.(type) { + case time.Time: + nt.Time, nt.Valid = v, true + return + case []byte: + nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Valid = (err == nil) + return + case string: + nt.Time, err = parseDateTime(v, time.UTC) + nt.Valid = (err == nil) + return + } + + nt.Valid = false + return fmt.Errorf("Can't convert %T to time.Time", value) +} + +// Value implements the driver Valuer interface. +func (nt NullTime) Value() (driver.Value, error) { + if !nt.Valid { + return nil, nil + } + return nt.Time, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go new file mode 100644 index 0000000..c392594 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go @@ -0,0 +1,31 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.13 + +package mysql + +import ( + "database/sql" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime sql.NullTime diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go new file mode 100644 index 0000000..86d159d --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go @@ -0,0 +1,34 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !go1.13 + +package mysql + +import ( + "time" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_test.go b/vendor/github.com/go-sql-driver/mysql/nulltime_test.go new file mode 100644 index 0000000..a14ec06 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_test.go @@ -0,0 +1,62 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql" + "database/sql/driver" + "testing" + "time" +) + +var ( + // Check implementation of interfaces + _ driver.Valuer = NullTime{} + _ sql.Scanner = (*NullTime)(nil) +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index 30b3352..82ad7a2 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -115,7 +115,7 @@ func (mc *mysqlConn) writePacket(data []byte) error { if mc.cfg.ReadTimeout != 0 { err = conn.SetReadDeadline(time.Time{}) } - if err == nil { + if err == nil && mc.cfg.CheckConnLiveness { err = connCheck(conn) } if err != nil { diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index cfa10e9..9552e80 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -106,60 +106,6 @@ func readBool(input string) (value bool, valid bool) { * Time related utils * ******************************************************************************/ -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { base := "0000-00-00 00:00:00.0000000" switch len(str) { diff --git a/vendor/github.com/go-sql-driver/mysql/utils_test.go b/vendor/github.com/go-sql-driver/mysql/utils_test.go index 8951a7a..10a60c2 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils_test.go +++ b/vendor/github.com/go-sql-driver/mysql/utils_test.go @@ -14,49 +14,8 @@ import ( "database/sql/driver" "encoding/binary" "testing" - "time" ) -func TestScanNullTime(t *testing.T) { - var scanTests = []struct { - in interface{} - error bool - valid bool - time time.Time - }{ - {tDate, false, true, tDate}, - {sDate, false, true, tDate}, - {[]byte(sDate), false, true, tDate}, - {tDateTime, false, true, tDateTime}, - {sDateTime, false, true, tDateTime}, - {[]byte(sDateTime), false, true, tDateTime}, - {tDate0, false, true, tDate0}, - {sDate0, false, true, tDate0}, - {[]byte(sDate0), false, true, tDate0}, - {sDateTime0, false, true, tDate0}, - {[]byte(sDateTime0), false, true, tDate0}, - {"", true, false, tDate0}, - {"1234", true, false, tDate0}, - {0, true, false, tDate0}, - } - - var nt = NullTime{} - var err error - - for _, tst := range scanTests { - err = nt.Scan(tst.in) - if (err != nil) != tst.error { - t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) - } - if nt.Valid != tst.valid { - t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) - } - if nt.Time != tst.time { - t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) - } - } -} - func TestLengthEncodedInteger(t *testing.T) { var integerTests = []struct { num uint64