From a954887fd5de2d0feb26dfac82b5e85dcd7ca259 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 20 Jul 2017 17:05:45 +0300 Subject: [PATCH 001/176] latin1 tests with TEXT columns --- localtests/latin1text/create.sql | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 localtests/latin1text/create.sql diff --git a/localtests/latin1text/create.sql b/localtests/latin1text/create.sql new file mode 100644 index 0000000..58837cb --- /dev/null +++ b/localtests/latin1text/create.sql @@ -0,0 +1,25 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + t text charset latin1 collate latin1_swedish_ci, + primary key(id) +) auto_increment=1 charset latin1 collate latin1_swedish_ci; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, md5(rand())); + insert into gh_ost_test values (null, 'átesting'); + insert into gh_ost_test values (null, 'ádelete'); + insert into gh_ost_test values (null, 'testátest'); + update gh_ost_test set t='áupdated' order by id desc limit 1; + update gh_ost_test set t='áupdated1' where t='áupdated' order by id desc limit 1; + delete from gh_ost_test where t='ádelete'; +end ;; From 0fd4603ae6ef33f3ffe39cc21bda31bcf6f3dfd6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 1 Nov 2017 09:51:45 +0200 Subject: [PATCH 002/176] minor updates to test --- localtests/datetime-submillis/create.sql | 2 +- localtests/test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/localtests/datetime-submillis/create.sql b/localtests/datetime-submillis/create.sql index b4e0b0b..6c04adb 100644 --- a/localtests/datetime-submillis/create.sql +++ b/localtests/datetime-submillis/create.sql @@ -17,7 +17,7 @@ create event gh_ost_test starts current_timestamp ends current_timestamp + interval 60 second on completion not preserve - enable + disable on slave do begin insert into gh_ost_test values (null, 11, now(), now(), now(), 0); diff --git a/localtests/test.sh b/localtests/test.sh index b901ce3..10cf796 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -164,6 +164,7 @@ test_single() { diff $orig_content_output_file $ghost_content_output_file echo "diff $orig_content_output_file $ghost_content_output_file" + return 1 fi } From 35becabf3cb94479f47f46a9cb9a2ff852d24803 Mon Sep 17 00:00:00 2001 From: ceshihao Date: Mon, 8 Jan 2018 18:25:33 +0800 Subject: [PATCH 003/176] support gbk encoding --- go/sql/encoding.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/sql/encoding.go b/go/sql/encoding.go index ac38f85..767bd9d 100644 --- a/go/sql/encoding.go +++ b/go/sql/encoding.go @@ -8,6 +8,7 @@ package sql import ( "golang.org/x/text/encoding" "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/simplifiedchinese" ) type charsetEncoding map[string]encoding.Encoding @@ -18,4 +19,5 @@ func init() { charsetEncodingMap = make(map[string]encoding.Encoding) // Begin mappings charsetEncodingMap["latin1"] = charmap.Windows1252 + charsetEncodingMap["gbk"] = simplifiedchinese.GBK } From 346138772c3fbeb618f4779994507d472741b011 Mon Sep 17 00:00:00 2001 From: ceshihao Date: Mon, 8 Jan 2018 23:43:10 +0800 Subject: [PATCH 004/176] add an option --include-multibyte-charset --- go/base/context.go | 1 + go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 2 +- go/logic/inspect.go | 2 +- go/logic/streamer.go | 2 +- go/logic/throttler.go | 2 +- go/mysql/connection.go | 8 ++++++-- go/mysql/connection_test.go | 11 +++++++++++ go/mysql/utils.go | 2 +- 9 files changed, 24 insertions(+), 7 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 23fe6f6..e1db5c6 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -91,6 +91,7 @@ type MigrationContext struct { SkipRenamedColumns bool IsTungsten bool DiscardForeignKeys bool + IncludeMultibyteCharset bool config ContextConfig configMutex *sync.Mutex diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 74d0f05..49b8849 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -121,6 +121,7 @@ func main() { version := flag.Bool("version", false, "Print version & exit") checkFlag := flag.Bool("check-flag", false, "Check if another flag exists/supported. This allows for cross-version scripting. Exits with 0 when all additional provided flags exist, nonzero otherwise. You must provide (dummy) values for flags that require a value. Example: gh-ost --check-flag --cut-over-lock-timeout-seconds --nice-ratio 0") flag.StringVar(&migrationContext.ForceTmpTableName, "force-table-names", "", "table name prefix to be used on the temporary tables") + flag.BoolVar(&migrationContext.IncludeMultibyteCharset, "include-multibyte-charset", false, "charset includes multibyte encoding, e.g. gbk, gb2312, big5, cp932, sjis") flag.CommandLine.SetOutput(os.Stdout) flag.Parse() diff --git a/go/logic/applier.go b/go/logic/applier.go index 227b59e..58cd1df 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -69,7 +69,7 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { func (this *Applier) InitDBConnections() (err error) { - applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName) + applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeMultibyteCharset) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, applierUri); err != nil { return err } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31c81dc..53b9579 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -40,7 +40,7 @@ func NewInspector(migrationContext *base.MigrationContext) *Inspector { } func (this *Inspector) InitDBConnections() (err error) { - inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName) + inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeMultibyteCharset) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, inspectorUri); err != nil { return err } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 37e7195..713dffd 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -103,7 +103,7 @@ func (this *EventsStreamer) notifyListeners(binlogEvent *binlog.BinlogDMLEvent) } func (this *EventsStreamer) InitDBConnections() (err error) { - EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName) + EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeMultibyteCharset) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil { return err } diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 624956a..5bb7b07 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -183,7 +183,7 @@ func (this *Throttler) collectControlReplicasLag() { ) readReplicaLag := func(connectionConfig *mysql.ConnectionConfig) (lag time.Duration, err error) { - dbUri := connectionConfig.GetDBUri("information_schema") + dbUri := connectionConfig.GetDBUri("information_schema", false) var heartbeatValue string if db, _, err := mysql.GetDB(this.migrationContext.Uuid, dbUri); err != nil { diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 96ae08b..c41ccb9 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -49,12 +49,16 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) GetDBUri(databaseName string) string { +func (this *ConnectionConfig) GetDBUri(databaseName string, includeMultibyte bool) string { hostname := this.Key.Hostname var ip = net.ParseIP(hostname) if (ip != nil) && (ip.To4() == nil) { // Wrap IPv6 literals in square brackets hostname = fmt.Sprintf("[%s]", hostname) } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1", this.User, this.Password, hostname, this.Key.Port, databaseName) + var multibyteCharset string + if includeMultibyte { + multibyteCharset = ",gbk,gb2312,big5,cp932,sjis" + } + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1%s", this.User, this.Password, hostname, this.Key.Port, databaseName, !includeMultibyte, multibyteCharset) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 657feb0..da4b825 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -55,3 +55,14 @@ func TestDuplicate(t *testing.T) { test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") } +func TestGetDBUri(t *testing.T) { + c := NewConnectionConfig() + c.Key = InstanceKey{Hostname: "myhost", Port: 3306} + c.User = "gromit" + c.Password = "penguin" + + uri := c.GetDBUri("test", true) + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=false&autocommit=true&charset=utf8mb4,utf8,latin1,gbk,gb2312,big5,cp932,sjis") + uri = c.GetDBUri("test", false) + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1") +} diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4..23c4fa6 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -75,7 +75,7 @@ func GetReplicationLag(informationSchemaDb *gosql.DB, connectionConfig *Connecti } func GetMasterKeyFromSlaveStatus(connectionConfig *ConnectionConfig) (masterKey *InstanceKey, err error) { - currentUri := connectionConfig.GetDBUri("information_schema") + currentUri := connectionConfig.GetDBUri("information_schema", false) // This function is only called once, okay to not have a cached connection pool db, err := gosql.Open("mysql", currentUri) if err != nil { From 14315ee2ccf32a69aa9869e80244d28157adc080 Mon Sep 17 00:00:00 2001 From: ceshihao Date: Mon, 15 Jan 2018 23:39:02 +0800 Subject: [PATCH 005/176] resolve merge conflict --- go/logic/inspect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 53b9579..bf7faf9 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -45,7 +45,7 @@ func (this *Inspector) InitDBConnections() (err error) { return err } - informationSchemaUri := this.connectionConfig.GetDBUri("information_schema") + informationSchemaUri := this.connectionConfig.GetDBUri("information_schema", false) if this.informationSchemaDb, _, err = mysql.GetDB(this.migrationContext.Uuid, informationSchemaUri); err != nil { return err } From 330650baa91e726740e1188f55dea6eb4a9d8e23 Mon Sep 17 00:00:00 2001 From: ceshihao Date: Wed, 17 Jan 2018 00:15:50 +0800 Subject: [PATCH 006/176] rename includeMultibyteCharset -> includeRiskCharset, multibyteCharset -> riskCharset --- go/base/context.go | 2 +- go/cmd/gh-ost/main.go | 2 +- go/logic/applier.go | 2 +- go/logic/inspect.go | 2 +- go/logic/streamer.go | 2 +- go/mysql/connection.go | 11 ++++++----- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index e1db5c6..c3f9df6 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -91,7 +91,7 @@ type MigrationContext struct { SkipRenamedColumns bool IsTungsten bool DiscardForeignKeys bool - IncludeMultibyteCharset bool + IncludeRiskCharset bool config ContextConfig configMutex *sync.Mutex diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 49b8849..130cff0 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -121,7 +121,7 @@ func main() { version := flag.Bool("version", false, "Print version & exit") checkFlag := flag.Bool("check-flag", false, "Check if another flag exists/supported. This allows for cross-version scripting. Exits with 0 when all additional provided flags exist, nonzero otherwise. You must provide (dummy) values for flags that require a value. Example: gh-ost --check-flag --cut-over-lock-timeout-seconds --nice-ratio 0") flag.StringVar(&migrationContext.ForceTmpTableName, "force-table-names", "", "table name prefix to be used on the temporary tables") - flag.BoolVar(&migrationContext.IncludeMultibyteCharset, "include-multibyte-charset", false, "charset includes multibyte encoding, e.g. gbk, gb2312, big5, cp932, sjis") + flag.BoolVar(&migrationContext.IncludeRiskCharset, "include-risk-charset", false, "charset includes risk encoding, e.g. gbk, gb2312, big5, cp932, sjis") flag.CommandLine.SetOutput(os.Stdout) flag.Parse() diff --git a/go/logic/applier.go b/go/logic/applier.go index 58cd1df..4812445 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -69,7 +69,7 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { func (this *Applier) InitDBConnections() (err error) { - applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeMultibyteCharset) + applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskCharset) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, applierUri); err != nil { return err } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index bf7faf9..cc98107 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -40,7 +40,7 @@ func NewInspector(migrationContext *base.MigrationContext) *Inspector { } func (this *Inspector) InitDBConnections() (err error) { - inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeMultibyteCharset) + inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskCharset) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, inspectorUri); err != nil { return err } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 713dffd..5700e84 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -103,7 +103,7 @@ func (this *EventsStreamer) notifyListeners(binlogEvent *binlog.BinlogDMLEvent) } func (this *EventsStreamer) InitDBConnections() (err error) { - EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeMultibyteCharset) + EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskCharset) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil { return err } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index c41ccb9..db1474c 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -49,16 +49,17 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) GetDBUri(databaseName string, includeMultibyte bool) string { +func (this *ConnectionConfig) GetDBUri(databaseName string, includeRiskCharset bool) string { hostname := this.Key.Hostname var ip = net.ParseIP(hostname) if (ip != nil) && (ip.To4() == nil) { // Wrap IPv6 literals in square brackets hostname = fmt.Sprintf("[%s]", hostname) } - var multibyteCharset string - if includeMultibyte { - multibyteCharset = ",gbk,gb2312,big5,cp932,sjis" + var riskCharset string + interpolateParams := !includeRiskCharset + if includeRiskCharset { + riskCharset = ",gbk,gb2312,big5,cp932,sjis" } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1%s", this.User, this.Password, hostname, this.Key.Port, databaseName, !includeMultibyte, multibyteCharset) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, riskCharset) } From 6f3d54a13698b5079d285f8c2abadfd133021844 Mon Sep 17 00:00:00 2001 From: zj118228 Date: Tue, 23 Jan 2018 13:51:41 +0800 Subject: [PATCH 007/176] Aliyun RDS hide some mysql variables for secrity so gh-ost will get ilegal values. Add --aliyun-rds flag to avoid getting them. --- go/base/context.go | 3 +++ go/base/utils.go | 13 +++++++++++-- go/cmd/gh-ost/main.go | 2 ++ go/logic/applier.go | 6 ++++-- go/logic/inspect.go | 6 ++++-- go/mysql/utils.go | 6 ++++-- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 23fe6f6..5f63c88 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -91,6 +91,7 @@ type MigrationContext struct { SkipRenamedColumns bool IsTungsten bool DiscardForeignKeys bool + AliyunRDS bool config ContextConfig configMutex *sync.Mutex @@ -214,6 +215,8 @@ type ContextConfig struct { } } +var Context *MigrationContext + func NewMigrationContext() *MigrationContext { return &MigrationContext{ Uuid: uuid.NewV4().String(), diff --git a/go/base/utils.go b/go/base/utils.go index 727bc57..8d9e4f0 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -65,16 +65,25 @@ func StringContainsAll(s string, substrings ...string) bool { } func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) (string, error) { - query := `select @@global.port, @@global.version` + versionQuery := `select @@global.version` var port, extraPort int var version string - if err := db.QueryRow(query).Scan(&port, &version); err != nil { + if err := db.QueryRow(versionQuery).Scan(&version); err != nil { return "", err } extraPortQuery := `select @@global.extra_port` if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { // swallow this error. not all servers support extra_port } + // AliyunRDS set users port to "NULL", replace it by gh-ost param + if Context.AliyunRDS { + port = connectionConfig.Key.Port + } else { + portQuery := `select @@global.port` + if err := db.QueryRow(portQuery).Scan(&port); err != nil { + return "", err + } + } if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) { log.Infof("connection validated on %+v", connectionConfig.Key) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 74d0f05..335d6cc 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -44,6 +44,7 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() + base.Context = migrationContext flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") @@ -68,6 +69,7 @@ func main() { flag.BoolVar(&migrationContext.IsTungsten, "tungsten", false, "explicitly let gh-ost know that you are running on a tungsten-replication based topology (you are likely to also provide --assume-master-host)") flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys") flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that") + flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index 227b59e..1a3e618 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -89,10 +89,12 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.validateAndReadTimeZone(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { return err } else { - this.connectionConfig.ImpliedKey = impliedKey + if this.migrationContext.AliyunRDS != true { + this.connectionConfig.ImpliedKey = impliedKey + } } if err := this.readTableColumns(); err != nil { return err diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31c81dc..1417ffa 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,10 +53,12 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { return err } else { - this.connectionConfig.ImpliedKey = impliedKey + if this.migrationContext.AliyunRDS != true { + this.connectionConfig.ImpliedKey = impliedKey + } } if err := this.validateGrants(); err != nil { return err diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4..c513ae5 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -171,9 +171,11 @@ func GetSelfBinlogCoordinates(db *gosql.DB) (selfBinlogCoordinates *BinlogCoordi } // GetInstanceKey reads hostname and port on given DB -func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { +func GetInstanceKey(db *gosql.DB, isAliyunRDS bool) (instanceKey *InstanceKey, err error) { instanceKey = &InstanceKey{} - err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) + if isAliyunRDS != true { + err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) + } return instanceKey, err } From 858cd187d2a10922efbbb4c478ef23d3dd61efd7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 28 Jan 2018 08:57:14 +0200 Subject: [PATCH 008/176] Testing '1970-01-01 00:00:00' error --- localtests/datetime-1970/create.sql | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 localtests/datetime-1970/create.sql diff --git a/localtests/datetime-1970/create.sql b/localtests/datetime-1970/create.sql new file mode 100644 index 0000000..7000c34 --- /dev/null +++ b/localtests/datetime-1970/create.sql @@ -0,0 +1,23 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + create_time timestamp NULL, + update_time timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + counter int(10) unsigned DEFAULT NULL, + primary key(id) +) auto_increment=1; + +insert into gh_ost_test values (1,'0000-00-00 00:00:00',now(),0); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + update gh_ost_test set counter = counter + 1 where id = 1; +end ;; From ac1b85b12943615cae440c196c4d8aa410f69359 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 28 Jan 2018 08:58:10 +0200 Subject: [PATCH 009/176] Testing '1970-01-01 00:00:00' error --- localtests/datetime-1970/create.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localtests/datetime-1970/create.sql b/localtests/datetime-1970/create.sql index 7000c34..9fcf9ef 100644 --- a/localtests/datetime-1970/create.sql +++ b/localtests/datetime-1970/create.sql @@ -1,13 +1,13 @@ drop table if exists gh_ost_test; create table gh_ost_test ( id int auto_increment, - create_time timestamp NULL, + create_time timestamp NULL DEFAULT '0000-00-00 00:00:00', update_time timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, counter int(10) unsigned DEFAULT NULL, primary key(id) ) auto_increment=1; -insert into gh_ost_test values (1,'0000-00-00 00:00:00',now(),0); +insert into gh_ost_test values (1, '0000-00-00 00:00:00', now(), 0); drop event if exists gh_ost_test; delimiter ;; From 3ad51b30cde9ad3e7ff2b75f7b6fda60c1c1b669 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 28 Jan 2018 09:14:10 +0200 Subject: [PATCH 010/176] to be perfectly aligned with bug report, explicitly setting the alter statement --- localtests/datetime-1970/extra_args | 1 + 1 file changed, 1 insertion(+) create mode 100644 localtests/datetime-1970/extra_args diff --git a/localtests/datetime-1970/extra_args b/localtests/datetime-1970/extra_args new file mode 100644 index 0000000..453f761 --- /dev/null +++ b/localtests/datetime-1970/extra_args @@ -0,0 +1 @@ +--alter='add column name varchar(1)' From e65e0eeeccb9542925b6ee18306ddd7bd0b329c1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 28 Jan 2018 09:16:12 +0200 Subject: [PATCH 011/176] explicitly naming columns --- localtests/datetime-1970/ghost_columns | 1 + localtests/datetime-1970/orig_columns | 1 + 2 files changed, 2 insertions(+) create mode 100644 localtests/datetime-1970/ghost_columns create mode 100644 localtests/datetime-1970/orig_columns diff --git a/localtests/datetime-1970/ghost_columns b/localtests/datetime-1970/ghost_columns new file mode 100644 index 0000000..581038f --- /dev/null +++ b/localtests/datetime-1970/ghost_columns @@ -0,0 +1 @@ +id, create_time, update_time, counter diff --git a/localtests/datetime-1970/orig_columns b/localtests/datetime-1970/orig_columns new file mode 100644 index 0000000..581038f --- /dev/null +++ b/localtests/datetime-1970/orig_columns @@ -0,0 +1 @@ +id, create_time, update_time, counter From 22091362582183ff56ceba781f20eac71439df63 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 7 Mar 2018 16:30:02 +0200 Subject: [PATCH 012/176] rename parameters --- go/base/context.go | 2 +- go/cmd/gh-ost/main.go | 2 +- go/logic/applier.go | 2 +- go/logic/inspect.go | 2 +- go/logic/streamer.go | 2 +- go/mysql/connection.go | 13 +++++++------ 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index c3f9df6..78402c6 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -91,7 +91,7 @@ type MigrationContext struct { SkipRenamedColumns bool IsTungsten bool DiscardForeignKeys bool - IncludeRiskCharset bool + IncludeRiskyCharsets bool config ContextConfig configMutex *sync.Mutex diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index b509a25..a75bc0a 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -120,7 +120,7 @@ func main() { version := flag.Bool("version", false, "Print version & exit") checkFlag := flag.Bool("check-flag", false, "Check if another flag exists/supported. This allows for cross-version scripting. Exits with 0 when all additional provided flags exist, nonzero otherwise. You must provide (dummy) values for flags that require a value. Example: gh-ost --check-flag --cut-over-lock-timeout-seconds --nice-ratio 0") flag.StringVar(&migrationContext.ForceTmpTableName, "force-table-names", "", "table name prefix to be used on the temporary tables") - flag.BoolVar(&migrationContext.IncludeRiskCharset, "include-risk-charset", false, "charset includes risk encoding, e.g. gbk, gb2312, big5, cp932, sjis") + flag.BoolVar(&migrationContext.IncludeRiskyCharsets, "include-risk-charset", false, "charset includes risk encoding, e.g. gbk, gb2312, big5, cp932, sjis") flag.CommandLine.SetOutput(os.Stdout) flag.Parse() diff --git a/go/logic/applier.go b/go/logic/applier.go index 4812445..2f47a77 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -69,7 +69,7 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { func (this *Applier) InitDBConnections() (err error) { - applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskCharset) + applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskyCharsets) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, applierUri); err != nil { return err } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index cc98107..8924d2f 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -40,7 +40,7 @@ func NewInspector(migrationContext *base.MigrationContext) *Inspector { } func (this *Inspector) InitDBConnections() (err error) { - inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskCharset) + inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskyCharsets) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, inspectorUri); err != nil { return err } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 5700e84..a4ac9c3 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -103,7 +103,7 @@ func (this *EventsStreamer) notifyListeners(binlogEvent *binlog.BinlogDMLEvent) } func (this *EventsStreamer) InitDBConnections() (err error) { - EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskCharset) + EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskyCharsets) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil { return err } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index db1474c..41978ac 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -49,17 +49,18 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) GetDBUri(databaseName string, includeRiskCharset bool) string { +func (this *ConnectionConfig) GetDBUri(databaseName string, IncludeRiskyCharsets bool) string { hostname := this.Key.Hostname var ip = net.ParseIP(hostname) if (ip != nil) && (ip.To4() == nil) { // Wrap IPv6 literals in square brackets hostname = fmt.Sprintf("[%s]", hostname) } - var riskCharset string - interpolateParams := !includeRiskCharset - if includeRiskCharset { - riskCharset = ",gbk,gb2312,big5,cp932,sjis" + riskyCharsets := "" + interpolateParams := true + if IncludeRiskyCharsets { + interpolateParams = false + riskyCharsets = ",gbk,gb2312,big5,cp932,sjis" } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, riskCharset) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, riskyCharsets) } From b7eacbb75fdb043f9917fb4368e50b947dd1fb34 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 8 Mar 2018 07:21:50 +0200 Subject: [PATCH 013/176] testing gbk charset --- localtests/gbk-charset/create.sql | 24 ++++++++++++++++++++++++ localtests/gbk-charset/extra_args | 0 2 files changed, 24 insertions(+) create mode 100644 localtests/gbk-charset/create.sql create mode 100644 localtests/gbk-charset/extra_args diff --git a/localtests/gbk-charset/create.sql b/localtests/gbk-charset/create.sql new file mode 100644 index 0000000..e58e4fe --- /dev/null +++ b/localtests/gbk-charset/create.sql @@ -0,0 +1,24 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int(11) NOT NULL AUTO_INCREMENT, + name varchar(512) DEFAULT NULL, + v varchar(255) DEFAULT NULL COMMENT '添加普通列测试', + PRIMARY KEY (id) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=gbk; + +insert into gh_ost_test values (null, 'gbk-test-initial', '添加普通列测试-添加普通列测试'); +insert into gh_ost_test values (null, 'gbk-test-initial', '添加普通列测试-添加普通列测试'); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (name) values ('gbk-test-default'); + insert into gh_ost_test values (null, 'gbk-test', '添加普通列测试-添加普通列测试'); +end ;; diff --git a/localtests/gbk-charset/extra_args b/localtests/gbk-charset/extra_args new file mode 100644 index 0000000..e69de29 From a17f2df3592ba96c1ce4bca0a5aa6b8a094560c2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 8 Mar 2018 07:25:52 +0200 Subject: [PATCH 014/176] more elaborate test --- localtests/gbk-charset/create.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/localtests/gbk-charset/create.sql b/localtests/gbk-charset/create.sql index e58e4fe..08e1fae 100644 --- a/localtests/gbk-charset/create.sql +++ b/localtests/gbk-charset/create.sql @@ -21,4 +21,5 @@ create event gh_ost_test begin insert into gh_ost_test (name) values ('gbk-test-default'); insert into gh_ost_test values (null, 'gbk-test', '添加普通列测试-添加普通列测试'); + update gh_ost_test set v='添加普通列测试' where v='添加普通列测试-添加普通列测试' order by id desc limit 1; end ;; From 2311df412dea20efdb1075ca2fe804c88f107dd7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 12 Mar 2018 16:17:53 +0200 Subject: [PATCH 015/176] gbk support, simplified --- go/base/context.go | 1 - go/cmd/gh-ost/main.go | 1 - go/logic/applier.go | 2 +- go/logic/inspect.go | 4 ++-- go/logic/streamer.go | 2 +- go/logic/throttler.go | 2 +- go/mysql/connection.go | 9 ++------- go/mysql/connection_test.go | 4 ++-- go/mysql/utils.go | 2 +- 9 files changed, 10 insertions(+), 17 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 78402c6..23fe6f6 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -91,7 +91,6 @@ type MigrationContext struct { SkipRenamedColumns bool IsTungsten bool DiscardForeignKeys bool - IncludeRiskyCharsets bool config ContextConfig configMutex *sync.Mutex diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a75bc0a..6d29fc4 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -120,7 +120,6 @@ func main() { version := flag.Bool("version", false, "Print version & exit") checkFlag := flag.Bool("check-flag", false, "Check if another flag exists/supported. This allows for cross-version scripting. Exits with 0 when all additional provided flags exist, nonzero otherwise. You must provide (dummy) values for flags that require a value. Example: gh-ost --check-flag --cut-over-lock-timeout-seconds --nice-ratio 0") flag.StringVar(&migrationContext.ForceTmpTableName, "force-table-names", "", "table name prefix to be used on the temporary tables") - flag.BoolVar(&migrationContext.IncludeRiskyCharsets, "include-risk-charset", false, "charset includes risk encoding, e.g. gbk, gb2312, big5, cp932, sjis") flag.CommandLine.SetOutput(os.Stdout) flag.Parse() diff --git a/go/logic/applier.go b/go/logic/applier.go index 2f47a77..227b59e 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -69,7 +69,7 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { func (this *Applier) InitDBConnections() (err error) { - applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskyCharsets) + applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, applierUri); err != nil { return err } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 8924d2f..31c81dc 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -40,12 +40,12 @@ func NewInspector(migrationContext *base.MigrationContext) *Inspector { } func (this *Inspector) InitDBConnections() (err error) { - inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskyCharsets) + inspectorUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, inspectorUri); err != nil { return err } - informationSchemaUri := this.connectionConfig.GetDBUri("information_schema", false) + informationSchemaUri := this.connectionConfig.GetDBUri("information_schema") if this.informationSchemaDb, _, err = mysql.GetDB(this.migrationContext.Uuid, informationSchemaUri); err != nil { return err } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index a4ac9c3..37e7195 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -103,7 +103,7 @@ func (this *EventsStreamer) notifyListeners(binlogEvent *binlog.BinlogDMLEvent) } func (this *EventsStreamer) InitDBConnections() (err error) { - EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName, this.migrationContext.IncludeRiskyCharsets) + EventsStreamerUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName) if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil { return err } diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 5bb7b07..624956a 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -183,7 +183,7 @@ func (this *Throttler) collectControlReplicasLag() { ) readReplicaLag := func(connectionConfig *mysql.ConnectionConfig) (lag time.Duration, err error) { - dbUri := connectionConfig.GetDBUri("information_schema", false) + dbUri := connectionConfig.GetDBUri("information_schema") var heartbeatValue string if db, _, err := mysql.GetDB(this.migrationContext.Uuid, dbUri); err != nil { diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 41978ac..c9c75f2 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -49,18 +49,13 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) GetDBUri(databaseName string, IncludeRiskyCharsets bool) string { +func (this *ConnectionConfig) GetDBUri(databaseName string) string { hostname := this.Key.Hostname var ip = net.ParseIP(hostname) if (ip != nil) && (ip.To4() == nil) { // Wrap IPv6 literals in square brackets hostname = fmt.Sprintf("[%s]", hostname) } - riskyCharsets := "" interpolateParams := true - if IncludeRiskyCharsets { - interpolateParams = false - riskyCharsets = ",gbk,gb2312,big5,cp932,sjis" - } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, riskyCharsets) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index da4b825..ae7d17b 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -61,8 +61,8 @@ func TestGetDBUri(t *testing.T) { c.User = "gromit" c.Password = "penguin" - uri := c.GetDBUri("test", true) + uri := c.GetDBUri("test") test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=false&autocommit=true&charset=utf8mb4,utf8,latin1,gbk,gb2312,big5,cp932,sjis") - uri = c.GetDBUri("test", false) + uri = c.GetDBUri("test") test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1") } diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 23c4fa6..532cbb4 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -75,7 +75,7 @@ func GetReplicationLag(informationSchemaDb *gosql.DB, connectionConfig *Connecti } func GetMasterKeyFromSlaveStatus(connectionConfig *ConnectionConfig) (masterKey *InstanceKey, err error) { - currentUri := connectionConfig.GetDBUri("information_schema", false) + currentUri := connectionConfig.GetDBUri("information_schema") // This function is only called once, okay to not have a cached connection pool db, err := gosql.Open("mysql", currentUri) if err != nil { From 710abd5a5bedbd585a46872bc48e615237d4a0d6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 12 Mar 2018 16:20:46 +0200 Subject: [PATCH 016/176] Fixed test --- go/mysql/connection_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index ae7d17b..19badf3 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -55,6 +55,7 @@ func TestDuplicate(t *testing.T) { test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") } + func TestGetDBUri(t *testing.T) { c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} @@ -62,7 +63,5 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=false&autocommit=true&charset=utf8mb4,utf8,latin1,gbk,gb2312,big5,cp932,sjis") - uri = c.GetDBUri("test") test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1") } From 3f0443833d4773fbf4f39baa3dc8378c9bc9b57c Mon Sep 17 00:00:00 2001 From: Kurt Kotzur Date: Thu, 15 Mar 2018 17:40:15 -0700 Subject: [PATCH 017/176] add flags for enabling exponential backoff and maximum backoff interval --- go/base/context.go | 54 +++++++++++++++++++++++++------------------ go/cmd/gh-ost/main.go | 5 ++++ 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 23fe6f6..a05864b 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -100,28 +100,30 @@ type MigrationContext struct { CliMasterUser string CliMasterPassword string - HeartbeatIntervalMilliseconds int64 - defaultNumRetries int64 - ChunkSize int64 - niceRatio float64 - MaxLagMillisecondsThrottleThreshold int64 - throttleControlReplicaKeys *mysql.InstanceKeyMap - ThrottleFlagFile string - ThrottleAdditionalFlagFile string - throttleQuery string - throttleHTTP string - ThrottleCommandedByUser int64 - HibernateUntil int64 - maxLoad LoadMap - criticalLoad LoadMap - CriticalLoadIntervalMilliseconds int64 - CriticalLoadHibernateSeconds int64 - PostponeCutOverFlagFile string - CutOverLockTimeoutSeconds int64 - ForceNamedCutOverCommand bool - PanicFlagFile string - HooksPath string - HooksHintMessage string + HeartbeatIntervalMilliseconds int64 + defaultNumRetries int64 + ChunkSize int64 + niceRatio float64 + MaxLagMillisecondsThrottleThreshold int64 + throttleControlReplicaKeys *mysql.InstanceKeyMap + ThrottleFlagFile string + ThrottleAdditionalFlagFile string + throttleQuery string + throttleHTTP string + ThrottleCommandedByUser int64 + HibernateUntil int64 + maxLoad LoadMap + criticalLoad LoadMap + CriticalLoadIntervalMilliseconds int64 + CriticalLoadHibernateSeconds int64 + PostponeCutOverFlagFile string + CutOverLockTimeoutSeconds int64 + CutOverExponentialBackoff bool + CutOverExponentialBackoffMaxInterval int64 + ForceNamedCutOverCommand bool + PanicFlagFile string + HooksPath string + HooksHintMessage string DropServeSocket bool ServeSocketFile string @@ -341,6 +343,14 @@ func (this *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64) return nil } +func (this *MigrationContext) SetCutOverExponentialBackoffMaxInterval(intervalSeconds int64) error { + if intervalSeconds < 2 { + return fmt.Errorf("Minimal maximum interval is 2sec. Timeout remains at %d", this.CutOverExponentialBackoffMaxInterval) + } + this.CutOverExponentialBackoffMaxInterval = intervalSeconds + return nil +} + func (this *MigrationContext) SetDefaultNumRetries(retries int64) { this.throttleMutex.Lock() defer this.throttleMutex.Unlock() diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 6d29fc4..a8f629b 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -82,6 +82,8 @@ func main() { flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running") flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") + flag.BoolVar(&migrationContext.CutOverExponentialBackoff, "cut-over-exponential-backoff", false, "Wait exponentially longer times between failed cut-over attempts (obeys a maximum interval configurable with 'cut-over-exponential-backoff-max-interval'). Ignores 'default-retries.'") + cutOverExponentialBackoffMaxInterval := flag.Int64("cut-over-exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between failed cut-over attempts. Ignored unless 'cut-over-exponential-backoff' is 'true.' When the maximum is reached, attempts will stop, regardless of whether the last was successful.") chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 100-100,000)") dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)") defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking") @@ -237,6 +239,9 @@ func main() { if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { log.Errore(err) } + if err := migrationContext.SetCutOverExponentialBackoffMaxInterval(*cutOverExponentialBackoffMaxInterval); err != nil { + log.Errore(err) + } log.Infof("starting gh-ost %+v", AppVersion) acceptSignals(migrationContext) From bd62b19b0bc239503e8ba13c9bfebcb75028fdbb Mon Sep 17 00:00:00 2001 From: Kurt Kotzur Date: Thu, 15 Mar 2018 17:41:15 -0700 Subject: [PATCH 018/176] remove nonFatalHint argument from retryOperation; add retryOperationWithExponentialBackoff and conditionally use it --- go/logic/migrator.go | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 3937a45..4dc9976 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -130,7 +130,7 @@ func (this *Migrator) sleepWhileTrue(operation func() (bool, error)) error { // retryOperation attempts up to `count` attempts at running given function, // exiting as soon as it returns with non-error. -func (this *Migrator) retryOperation(operation func() error, notFatalHint ...bool) (err error) { +func (this *Migrator) retryOperation(operation func() error) (err error) { maxRetries := int(this.migrationContext.MaxRetries()) for i := 0; i < maxRetries; i++ { if i != 0 { @@ -143,8 +143,23 @@ func (this *Migrator) retryOperation(operation func() error, notFatalHint ...boo } // there's an error. Let's try again. } - if len(notFatalHint) == 0 { - this.migrationContext.PanicAbort <- err + return err +} + +func (this *Migrator) retryOperationWithExponentialBackoff(operation func() error) (err error) { + var numAttempts float64 + for { + err = operation() + if err == nil { + return nil + } + + interval := math.Exp2(numAttempts) + if interval > this.migrationContext.CutOverExponentialBackoffMaxInterval { + break + } else { + time.Sleep(interval * time.Second) + } } return err } @@ -372,7 +387,13 @@ func (this *Migrator) Migrate() (err error) { if err := this.hooksExecutor.onBeforeCutOver(); err != nil { return err } - if err := this.retryOperation(this.cutOver); err != nil { + var retrier func(func() error) error + if this.migrationContext.CutOverExponentialBackoff { + retrier = this.retryOperationWithExponentialBackoff + } else { + retrier = this.retryOperation + } + if err := retrier(this.cutOver); err != nil { return err } atomic.StoreInt64(&this.migrationContext.CutOverCompleteFlag, 1) From 64f66c4abb903643515acac56e7759fd7618d089 Mon Sep 17 00:00:00 2001 From: Kurt Kotzur Date: Thu, 15 Mar 2018 17:44:50 -0700 Subject: [PATCH 019/176] add back nonFatalHint, fix type mismatches --- go/base/context.go | 4 ++-- go/cmd/gh-ost/main.go | 2 +- go/logic/migrator.go | 18 ++++++++++++------ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index a05864b..e5f3471 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,7 +119,7 @@ type MigrationContext struct { PostponeCutOverFlagFile string CutOverLockTimeoutSeconds int64 CutOverExponentialBackoff bool - CutOverExponentialBackoffMaxInterval int64 + CutOverExponentialBackoffMaxInterval int ForceNamedCutOverCommand bool PanicFlagFile string HooksPath string @@ -343,7 +343,7 @@ func (this *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64) return nil } -func (this *MigrationContext) SetCutOverExponentialBackoffMaxInterval(intervalSeconds int64) error { +func (this *MigrationContext) SetCutOverExponentialBackoffMaxInterval(intervalSeconds int) error { if intervalSeconds < 2 { return fmt.Errorf("Minimal maximum interval is 2sec. Timeout remains at %d", this.CutOverExponentialBackoffMaxInterval) } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a8f629b..62398b6 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -83,7 +83,7 @@ func main() { flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running") flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") flag.BoolVar(&migrationContext.CutOverExponentialBackoff, "cut-over-exponential-backoff", false, "Wait exponentially longer times between failed cut-over attempts (obeys a maximum interval configurable with 'cut-over-exponential-backoff-max-interval'). Ignores 'default-retries.'") - cutOverExponentialBackoffMaxInterval := flag.Int64("cut-over-exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between failed cut-over attempts. Ignored unless 'cut-over-exponential-backoff' is 'true.' When the maximum is reached, attempts will stop, regardless of whether the last was successful.") + cutOverExponentialBackoffMaxInterval := flag.Int("cut-over-exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between failed cut-over attempts. Ignored unless 'cut-over-exponential-backoff' is 'true.' When the maximum is reached, attempts will stop, regardless of whether the last was successful.") chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 100-100,000)") dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)") defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking") diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 4dc9976..ae18c31 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -130,7 +130,7 @@ func (this *Migrator) sleepWhileTrue(operation func() (bool, error)) error { // retryOperation attempts up to `count` attempts at running given function, // exiting as soon as it returns with non-error. -func (this *Migrator) retryOperation(operation func() error) (err error) { +func (this *Migrator) retryOperation(operation func() error, notFatalHint ...bool) (err error) { maxRetries := int(this.migrationContext.MaxRetries()) for i := 0; i < maxRetries; i++ { if i != 0 { @@ -143,24 +143,30 @@ func (this *Migrator) retryOperation(operation func() error) (err error) { } // there's an error. Let's try again. } + if len(notFatalHint) == 0 { + this.migrationContext.PanicAbort <- err + } return err } -func (this *Migrator) retryOperationWithExponentialBackoff(operation func() error) (err error) { - var numAttempts float64 +func (this *Migrator) retryOperationWithExponentialBackoff(operation func() error, notFatalHint ...bool) (err error) { + var numAttempts int for { err = operation() if err == nil { return nil } - interval := math.Exp2(numAttempts) + interval := int(math.Exp2(float64(numAttempts))) if interval > this.migrationContext.CutOverExponentialBackoffMaxInterval { break } else { - time.Sleep(interval * time.Second) + time.Sleep(time.Duration(interval) * time.Second) } } + if len(notFatalHint) == 0 { + this.migrationContext.PanicAbort <- err + } return err } @@ -387,7 +393,7 @@ func (this *Migrator) Migrate() (err error) { if err := this.hooksExecutor.onBeforeCutOver(); err != nil { return err } - var retrier func(func() error) error + var retrier func(func() error, ...bool) error if this.migrationContext.CutOverExponentialBackoff { retrier = this.retryOperationWithExponentialBackoff } else { From 628983f78fce1d8d6514a489f0c3a07ee8210c11 Mon Sep 17 00:00:00 2001 From: Kurt Kotzur Date: Fri, 16 Mar 2018 12:39:55 -0700 Subject: [PATCH 020/176] :art: --- go/base/context.go | 4 ++-- go/cmd/gh-ost/main.go | 2 +- go/logic/migrator.go | 20 +++++++++++--------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index e5f3471..a05864b 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,7 +119,7 @@ type MigrationContext struct { PostponeCutOverFlagFile string CutOverLockTimeoutSeconds int64 CutOverExponentialBackoff bool - CutOverExponentialBackoffMaxInterval int + CutOverExponentialBackoffMaxInterval int64 ForceNamedCutOverCommand bool PanicFlagFile string HooksPath string @@ -343,7 +343,7 @@ func (this *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64) return nil } -func (this *MigrationContext) SetCutOverExponentialBackoffMaxInterval(intervalSeconds int) error { +func (this *MigrationContext) SetCutOverExponentialBackoffMaxInterval(intervalSeconds int64) error { if intervalSeconds < 2 { return fmt.Errorf("Minimal maximum interval is 2sec. Timeout remains at %d", this.CutOverExponentialBackoffMaxInterval) } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 62398b6..a8f629b 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -83,7 +83,7 @@ func main() { flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running") flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") flag.BoolVar(&migrationContext.CutOverExponentialBackoff, "cut-over-exponential-backoff", false, "Wait exponentially longer times between failed cut-over attempts (obeys a maximum interval configurable with 'cut-over-exponential-backoff-max-interval'). Ignores 'default-retries.'") - cutOverExponentialBackoffMaxInterval := flag.Int("cut-over-exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between failed cut-over attempts. Ignored unless 'cut-over-exponential-backoff' is 'true.' When the maximum is reached, attempts will stop, regardless of whether the last was successful.") + cutOverExponentialBackoffMaxInterval := flag.Int64("cut-over-exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between failed cut-over attempts. Ignored unless 'cut-over-exponential-backoff' is 'true.' When the maximum is reached, attempts will stop, regardless of whether the last was successful.") chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 100-100,000)") dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)") defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking") diff --git a/go/logic/migrator.go b/go/logic/migrator.go index ae18c31..c888970 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -149,20 +149,22 @@ func (this *Migrator) retryOperation(operation func() error, notFatalHint ...boo return err } +// retryOperation attempts running given function, waiting 2^(n-1) seconds +// between each attempt, where n is the running number of attempts. exits +// as soon as the function returns with non-error, or as soon as the next +// wait interval exceeds `CutOverExponentialBackoffMaxInterval`. func (this *Migrator) retryOperationWithExponentialBackoff(operation func() error, notFatalHint ...bool) (err error) { - var numAttempts int - for { + numAttempts := 0 + var interval int64 + maxInterval := this.migrationContext.CutOverExponentialBackoffMaxInterval + for interval < maxInterval { + time.Sleep(time.Duration(interval) * time.Second) err = operation() if err == nil { return nil } - - interval := int(math.Exp2(float64(numAttempts))) - if interval > this.migrationContext.CutOverExponentialBackoffMaxInterval { - break - } else { - time.Sleep(time.Duration(interval) * time.Second) - } + interval = int64(math.Exp2(float64(numAttempts))) + numAttempts++ } if len(notFatalHint) == 0 { this.migrationContext.PanicAbort <- err From 117b197b2aaa9c3bbd8e14d051a1e8588bd91b66 Mon Sep 17 00:00:00 2001 From: Kurt Kotzur Date: Mon, 19 Mar 2018 12:26:46 -0700 Subject: [PATCH 021/176] Obey defaultNumRetries in retryOperationWithExponentialBackoff; name max interval flag more generically (#2) * rename flags, obey defaultNumRetries * capitalization fixes * fix flag description typo * fix sleep algorithm --- go/base/context.go | 54 +++++++++++++++++++++---------------------- go/cmd/gh-ost/main.go | 6 ++--- go/logic/migrator.go | 16 ++++++++----- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index a05864b..bc81f32 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -100,30 +100,30 @@ type MigrationContext struct { CliMasterUser string CliMasterPassword string - HeartbeatIntervalMilliseconds int64 - defaultNumRetries int64 - ChunkSize int64 - niceRatio float64 - MaxLagMillisecondsThrottleThreshold int64 - throttleControlReplicaKeys *mysql.InstanceKeyMap - ThrottleFlagFile string - ThrottleAdditionalFlagFile string - throttleQuery string - throttleHTTP string - ThrottleCommandedByUser int64 - HibernateUntil int64 - maxLoad LoadMap - criticalLoad LoadMap - CriticalLoadIntervalMilliseconds int64 - CriticalLoadHibernateSeconds int64 - PostponeCutOverFlagFile string - CutOverLockTimeoutSeconds int64 - CutOverExponentialBackoff bool - CutOverExponentialBackoffMaxInterval int64 - ForceNamedCutOverCommand bool - PanicFlagFile string - HooksPath string - HooksHintMessage string + HeartbeatIntervalMilliseconds int64 + defaultNumRetries int64 + ChunkSize int64 + niceRatio float64 + MaxLagMillisecondsThrottleThreshold int64 + throttleControlReplicaKeys *mysql.InstanceKeyMap + ThrottleFlagFile string + ThrottleAdditionalFlagFile string + throttleQuery string + throttleHTTP string + ThrottleCommandedByUser int64 + HibernateUntil int64 + maxLoad LoadMap + criticalLoad LoadMap + CriticalLoadIntervalMilliseconds int64 + CriticalLoadHibernateSeconds int64 + PostponeCutOverFlagFile string + CutOverLockTimeoutSeconds int64 + CutOverExponentialBackoff bool + ExponentialBackoffMaxInterval int64 + ForceNamedCutOverCommand bool + PanicFlagFile string + HooksPath string + HooksHintMessage string DropServeSocket bool ServeSocketFile string @@ -343,11 +343,11 @@ func (this *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64) return nil } -func (this *MigrationContext) SetCutOverExponentialBackoffMaxInterval(intervalSeconds int64) error { +func (this *MigrationContext) SetExponentialBackoffMaxInterval(intervalSeconds int64) error { if intervalSeconds < 2 { - return fmt.Errorf("Minimal maximum interval is 2sec. Timeout remains at %d", this.CutOverExponentialBackoffMaxInterval) + return fmt.Errorf("Minimal maximum interval is 2sec. Timeout remains at %d", this.ExponentialBackoffMaxInterval) } - this.CutOverExponentialBackoffMaxInterval = intervalSeconds + this.ExponentialBackoffMaxInterval = intervalSeconds return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a8f629b..3433fab 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -82,8 +82,8 @@ func main() { flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running") flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") - flag.BoolVar(&migrationContext.CutOverExponentialBackoff, "cut-over-exponential-backoff", false, "Wait exponentially longer times between failed cut-over attempts (obeys a maximum interval configurable with 'cut-over-exponential-backoff-max-interval'). Ignores 'default-retries.'") - cutOverExponentialBackoffMaxInterval := flag.Int64("cut-over-exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between failed cut-over attempts. Ignored unless 'cut-over-exponential-backoff' is 'true.' When the maximum is reached, attempts will stop, regardless of whether the last was successful.") + flag.BoolVar(&migrationContext.CutOverExponentialBackoff, "cut-over-exponential-backoff", false, "Wait exponentially longer intervals between failed cut-over attempts. Wait intervals obey a maximum configurable with 'exponential-backoff-max-interval').") + exponentialBackoffMaxInterval := flag.Int64("exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between attempts when performing various operations with exponential backoff.") chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 100-100,000)") dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)") defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking") @@ -239,7 +239,7 @@ func main() { if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { log.Errore(err) } - if err := migrationContext.SetCutOverExponentialBackoffMaxInterval(*cutOverExponentialBackoffMaxInterval); err != nil { + if err := migrationContext.SetExponentialBackoffMaxInterval(*exponentialBackoffMaxInterval); err != nil { log.Errore(err) } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index c888970..0dcb3bb 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -154,17 +154,21 @@ func (this *Migrator) retryOperation(operation func() error, notFatalHint ...boo // as soon as the function returns with non-error, or as soon as the next // wait interval exceeds `CutOverExponentialBackoffMaxInterval`. func (this *Migrator) retryOperationWithExponentialBackoff(operation func() error, notFatalHint ...bool) (err error) { - numAttempts := 0 var interval int64 - maxInterval := this.migrationContext.CutOverExponentialBackoffMaxInterval - for interval < maxInterval { - time.Sleep(time.Duration(interval) * time.Second) + maxRetries := int(this.migrationContext.MaxRetries()) + maxInterval := this.migrationContext.ExponentialBackoffMaxInterval + for i := 0; i < maxRetries; i++ { + newInterval := int64(math.Exp2(float64(i - 1))) + if newInterval <= maxInterval { + interval = newInterval + } + if i != 0 { + time.Sleep(time.Duration(interval) * time.Second) + } err = operation() if err == nil { return nil } - interval = int64(math.Exp2(float64(numAttempts))) - numAttempts++ } if len(notFatalHint) == 0 { this.migrationContext.PanicAbort <- err From 15e7417fc5f863d01492f123388ed2b638dc09b1 Mon Sep 17 00:00:00 2001 From: Kurt Kotzur Date: Mon, 19 Mar 2018 12:29:49 -0700 Subject: [PATCH 022/176] update comment --- go/logic/migrator.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 0dcb3bb..673a98e 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -149,10 +149,11 @@ func (this *Migrator) retryOperation(operation func() error, notFatalHint ...boo return err } -// retryOperation attempts running given function, waiting 2^(n-1) seconds -// between each attempt, where n is the running number of attempts. exits -// as soon as the function returns with non-error, or as soon as the next -// wait interval exceeds `CutOverExponentialBackoffMaxInterval`. +// `retryOperationWithExponentialBackoff` attempts running given function, waiting 2^(n-1) +// seconds between each attempt, where `n` is the running number of attempts. Exits +// as soon as the function returns with non-error, or as soon as `MaxRetries` +// attempts are reached. Wait intervals between attempts obey a maximum of +// `ExponentialBackoffMaxInterval`. func (this *Migrator) retryOperationWithExponentialBackoff(operation func() error, notFatalHint ...bool) (err error) { var interval int64 maxRetries := int(this.migrationContext.MaxRetries()) From d3b98d9e746115834e5b9dd64e29a157eddcd854 Mon Sep 17 00:00:00 2001 From: zj118228 Date: Thu, 29 Mar 2018 11:19:20 +0800 Subject: [PATCH 023/176] Refactor the code and remove the global variable of support-aliyun-rds branch. --- go/base/context.go | 2 -- go/base/utils.go | 4 ++-- go/cmd/gh-ost/main.go | 2 -- go/logic/applier.go | 12 ++++++------ go/logic/inspect.go | 10 +++++----- go/logic/streamer.go | 2 +- go/mysql/utils.go | 6 ++---- 7 files changed, 16 insertions(+), 22 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 5f63c88..981c386 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -215,8 +215,6 @@ type ContextConfig struct { } } -var Context *MigrationContext - func NewMigrationContext() *MigrationContext { return &MigrationContext{ Uuid: uuid.NewV4().String(), diff --git a/go/base/utils.go b/go/base/utils.go index 8d9e4f0..d6bbf47 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -64,7 +64,7 @@ func StringContainsAll(s string, substrings ...string) bool { return nonEmptyStringsFound } -func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) (string, error) { +func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext) (string, error) { versionQuery := `select @@global.version` var port, extraPort int var version string @@ -76,7 +76,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) // swallow this error. not all servers support extra_port } // AliyunRDS set users port to "NULL", replace it by gh-ost param - if Context.AliyunRDS { + if migrationContext.AliyunRDS { port = connectionConfig.Key.Port } else { portQuery := `select @@global.port` diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 335d6cc..70a4e90 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -44,8 +44,6 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() - base.Context = migrationContext - flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") diff --git a/go/logic/applier.go b/go/logic/applier.go index 1a3e618..0a2951e 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -78,21 +78,21 @@ func (this *Applier) InitDBConnections() (err error) { return err } this.singletonDB.SetMaxOpenConns(1) - version, err := base.ValidateConnection(this.db, this.connectionConfig) + version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext) if err != nil { return err } - if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig); err != nil { + if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig, this.migrationContext); err != nil { return err } this.migrationContext.ApplierMySQLVersion = version if err := this.validateAndReadTimeZone(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { - return err - } else { - if this.migrationContext.AliyunRDS != true { + if !this.migrationContext.AliyunRDS { + if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + return err + } else { this.connectionConfig.ImpliedKey = impliedKey } } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 1417ffa..bb57447 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,10 +53,10 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { - return err - } else { - if this.migrationContext.AliyunRDS != true { + if !this.migrationContext.AliyunRDS { + if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + return err + } else { this.connectionConfig.ImpliedKey = impliedKey } } @@ -205,7 +205,7 @@ func (this *Inspector) validateConnection() error { return fmt.Errorf("MySQL replication length limited to 32 characters. See https://dev.mysql.com/doc/refman/5.7/en/assigning-passwords.html") } - version, err := base.ValidateConnection(this.db, this.connectionConfig) + version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext) this.migrationContext.InspectorMySQLVersion = version return err } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 37e7195..25d9f53 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -107,7 +107,7 @@ func (this *EventsStreamer) InitDBConnections() (err error) { if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil { return err } - if _, err := base.ValidateConnection(this.db, this.connectionConfig); err != nil { + if _, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext); err != nil { return err } if err := this.readCurrentBinlogCoordinates(); err != nil { diff --git a/go/mysql/utils.go b/go/mysql/utils.go index c513ae5..532cbb4 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -171,11 +171,9 @@ func GetSelfBinlogCoordinates(db *gosql.DB) (selfBinlogCoordinates *BinlogCoordi } // GetInstanceKey reads hostname and port on given DB -func GetInstanceKey(db *gosql.DB, isAliyunRDS bool) (instanceKey *InstanceKey, err error) { +func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { instanceKey = &InstanceKey{} - if isAliyunRDS != true { - err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) - } + err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) return instanceKey, err } From 2376c917045f88848da423423dc241ca7ea010e9 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 11 Apr 2018 09:11:27 +0300 Subject: [PATCH 024/176] Fix to long JSON values --- localtests/json57dml/create.sql | 3 ++- .../siddontang/go-mysql/cmd/go-mysqlbinlog/main.go | 2 +- .../siddontang/go-mysql/replication/binlogsyncer.go | 6 +++--- .../siddontang/go-mysql/replication/json_binary.go | 8 ++++---- .../go-mysql/replication/row_event_test.go | 13 +++++++++++++ 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/localtests/json57dml/create.sql b/localtests/json57dml/create.sql index da8cd57..ac20941 100644 --- a/localtests/json57dml/create.sql +++ b/localtests/json57dml/create.sql @@ -19,7 +19,8 @@ create event gh_ost_test begin insert into gh_ost_test (id, i, j) values (null, 11, '"sometext"'); insert into gh_ost_test (id, i, j) values (null, 13, '{"key":"val"}'); - insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); + insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); + insert into gh_ost_test (id, i, j) values (null, 19, '{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}'); update gh_ost_test set j = '{"updated": 11}', updated = 1 where i = 11 and updated = 0; update gh_ost_test set j = json_set(j, '$.count', 13, '$.id', id), updated = 1 where i = 13 and updated = 0; diff --git a/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go b/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go index 309cbac..80fa73d 100644 --- a/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go +++ b/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go @@ -41,7 +41,7 @@ func main() { Port: uint16(*port), User: *user, Password: *password, - RawModeEanbled: *rawMode, + RawModeEnabled: *rawMode, SemiSyncEnabled: *semiSync, } diff --git a/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go b/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go index d217724..2a72c6d 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go +++ b/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go @@ -43,8 +43,8 @@ type BinlogSyncerConfig struct { // SemiSyncEnabled enables semi-sync or not. SemiSyncEnabled bool - // RawModeEanbled is for not parsing binlog event. - RawModeEanbled bool + // RawModeEnabled is for not parsing binlog event. + RawModeEnabled bool // If not nil, use the provided tls.Config to connect to the database using TLS/SSL. TLSConfig *tls.Config @@ -83,7 +83,7 @@ func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { b.cfg = cfg b.parser = NewBinlogParser() - b.parser.SetRawMode(b.cfg.RawModeEanbled) + b.parser.SetRawMode(b.cfg.RawModeEnabled) b.running = false b.ctx, b.cancel = context.WithCancel(context.Background()) diff --git a/vendor/github.com/siddontang/go-mysql/replication/json_binary.go b/vendor/github.com/siddontang/go-mysql/replication/json_binary.go index 83b69d0..24a9b27 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/json_binary.go +++ b/vendor/github.com/siddontang/go-mysql/replication/json_binary.go @@ -338,7 +338,7 @@ func (d *jsonBinaryDecoder) decodeString(data []byte) string { l, n := d.decodeVariableLength(data) - if d.isDataShort(data, int(l)+n) { + if d.isDataShort(data, l+n) { return "" } @@ -358,11 +358,11 @@ func (d *jsonBinaryDecoder) decodeOpaque(data []byte) interface{} { l, n := d.decodeVariableLength(data) - if d.isDataShort(data, int(l)+n) { + if d.isDataShort(data, l+n) { return nil } - data = data[n : int(l)+n] + data = data[n : l+n] switch tp { case MYSQL_TYPE_NEWDECIMAL: @@ -459,7 +459,7 @@ func (d *jsonBinaryDecoder) decodeVariableLength(data []byte) (int, int) { length := uint64(0) for ; pos < maxCount; pos++ { v := data[pos] - length = (length << 7) + uint64(v&0x7F) + length |= uint64(v&0x7F) << uint(7*pos) if v&0x80 == 0 { if length > math.MaxUint32 { diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go b/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go index 2d63bb5..b1144c3 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go @@ -403,6 +403,8 @@ func (_ *testDecodeSuite) TestParseJson(c *C) { // INSERT INTO `t10` (`c2`) VALUES (1); // INSERT INTO `t10` (`c1`, `c2`) VALUES ('{"key1": "value1", "key2": "value2"}', 1); + // test json deserialization + // INSERT INTO `t10`(`c1`,`c2`) VALUES ('{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}',101); tableMapEventData := []byte("m\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x03t10\x00\x02\xf5\xf6\x03\x04\n\x00\x03") @@ -428,4 +430,15 @@ func (_ *testDecodeSuite) TestParseJson(c *C) { c.Assert(err, IsNil) c.Assert(rows.Rows[0][1], Equals, float64(1)) } + + longTbls := [][]byte{ + []byte("m\x00\x00\x00\x00\x00\x01\x00\x02\x00\x02\xff\xfc\xd0\n\x00\x00\x00\x01\x00\xcf\n\v\x00\x04\x00\f\x0f\x00text\xbe\x15Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac\x80\x00\x00\x00e"), + } + + for _, ltbl := range longTbls { + rows.Rows = nil + err = rows.Decode(ltbl) + c.Assert(err, IsNil) + c.Assert(rows.Rows[0][1], Equals, float64(101)) + } } From 68bf933659f161f37f05cd68eab1cec993ce310c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 17 Apr 2018 09:00:50 +0300 Subject: [PATCH 025/176] Release 10.0.45 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 6ae122f..2fec751 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.44 +1.0.45 From 160e7a11f5221aa239edd549bd2090c0303b5189 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 24 Apr 2018 05:56:03 -0700 Subject: [PATCH 026/176] Enforcing binlog_row_image=FULL --- go/logic/inspect.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index bb57447..2568afe 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -165,11 +165,6 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { return fmt.Errorf("Chosen key (%s) has nullable columns. Bailing out. To force this operation to continue, supply --allow-nullable-unique-key flag. Only do so if you are certain there are no actual NULL values in this key. As long as there aren't, migration should be fine. NULL values in columns of this key will corrupt migration's data", this.migrationContext.UniqueKey) } } - if !this.migrationContext.UniqueKey.IsPrimary() { - if this.migrationContext.OriginalBinlogRowImage != "FULL" { - return fmt.Errorf("binlog_row_image is '%s' and chosen key is %s, which is not the primary key. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.migrationContext.OriginalBinlogRowImage, this.migrationContext.UniqueKey) - } - } this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.ColumnRenameMap) log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) @@ -358,6 +353,9 @@ func (this *Inspector) validateBinlogs() error { this.migrationContext.OriginalBinlogRowImage = "FULL" } this.migrationContext.OriginalBinlogRowImage = strings.ToUpper(this.migrationContext.OriginalBinlogRowImage) + if this.migrationContext.OriginalBinlogRowImage != "FULL" { + return fmt.Errorf("%s:%d has '%s' binlog_row_image, and only 'FULL' is supported. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogRowImage) + } log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil From f56ddcd3fa6bbba5e28319145c7e8e20aad3e884 Mon Sep 17 00:00:00 2001 From: Ryan Hall Date: Wed, 2 May 2018 11:41:45 -0700 Subject: [PATCH 027/176] Fix the grammar for the opening sentence --- doc/rds.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rds.md b/doc/rds.md index 889d480..37d1d80 100644 --- a/doc/rds.md +++ b/doc/rds.md @@ -1,4 +1,4 @@ -`gh-ost` has been updated to work with Amazon RDS however due to GitHub not relying using AWS for databases, this documentation is community driven so if you find a bug please [open an issue][new_issue]! +`gh-ost` has been updated to work with Amazon RDS however due to GitHub not using AWS for databases, this documentation is community driven so if you find a bug please [open an issue][new_issue]! # Amazon RDS From 581a764ad707cbabc84d37eeb1a45bd280a8d0de Mon Sep 17 00:00:00 2001 From: Patrick Killian Date: Wed, 2 May 2018 21:36:01 -0700 Subject: [PATCH 028/176] [Typo] Fix typo in 'PULL_REQUEST_TEMPLATE.md' Line 14: "... explain what is does" -> "... explain what it does" --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8301e70..c0c2464 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,7 +11,7 @@ Related issue: https://github.com/github/gh-ost/issues/0123456789 ### Description -This PR [briefly explain what is does] +This PR [briefly explain what it does] > In case this PR introduced Go code changes: From 99fcbc35f39f5d3e6d1477944f14d8c5ef4389da Mon Sep 17 00:00:00 2001 From: Atsushi Ishibashi Date: Fri, 4 May 2018 01:45:14 +0900 Subject: [PATCH 029/176] Fix custom type assignment --- go/base/context.go | 12 ++++++------ go/binlog/binlog_dml_event.go | 9 +++++---- go/sql/builder.go | 10 +++++----- go/sql/types.go | 14 +++++++------- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 69cb7dd..94039e3 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -28,23 +28,23 @@ type RowsEstimateMethod string const ( TableStatusRowsEstimate RowsEstimateMethod = "TableStatusRowsEstimate" - ExplainRowsEstimate = "ExplainRowsEstimate" - CountRowsEstimate = "CountRowsEstimate" + ExplainRowsEstimate RowsEstimateMethod = "ExplainRowsEstimate" + CountRowsEstimate RowsEstimateMethod = "CountRowsEstimate" ) type CutOver int const ( - CutOverAtomic CutOver = iota - CutOverTwoStep = iota + CutOverAtomic CutOver = iota + CutOverTwoStep ) type ThrottleReasonHint string const ( NoThrottleReasonHint ThrottleReasonHint = "NoThrottleReasonHint" - UserCommandThrottleReasonHint = "UserCommandThrottleReasonHint" - LeavingHibernationThrottleReasonHint = "LeavingHibernationThrottleReasonHint" + UserCommandThrottleReasonHint ThrottleReasonHint = "UserCommandThrottleReasonHint" + LeavingHibernationThrottleReasonHint ThrottleReasonHint = "LeavingHibernationThrottleReasonHint" ) const ( diff --git a/go/binlog/binlog_dml_event.go b/go/binlog/binlog_dml_event.go index 4fab87a..2c7aa36 100644 --- a/go/binlog/binlog_dml_event.go +++ b/go/binlog/binlog_dml_event.go @@ -7,17 +7,18 @@ package binlog import ( "fmt" - "github.com/github/gh-ost/go/sql" "strings" + + "github.com/github/gh-ost/go/sql" ) type EventDML string const ( NotDML EventDML = "NoDML" - InsertDML = "Insert" - UpdateDML = "Update" - DeleteDML = "Delete" + InsertDML EventDML = "Insert" + UpdateDML EventDML = "Update" + DeleteDML EventDML = "Delete" ) func ToEventDML(description string) EventDML { diff --git a/go/sql/builder.go b/go/sql/builder.go index 251a874..c3a6229 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -15,11 +15,11 @@ type ValueComparisonSign string const ( LessThanComparisonSign ValueComparisonSign = "<" - LessThanOrEqualsComparisonSign = "<=" - EqualsComparisonSign = "=" - GreaterThanOrEqualsComparisonSign = ">=" - GreaterThanComparisonSign = ">" - NotEqualsComparisonSign = "!=" + LessThanOrEqualsComparisonSign ValueComparisonSign = "<=" + EqualsComparisonSign ValueComparisonSign = "=" + GreaterThanOrEqualsComparisonSign ValueComparisonSign = ">=" + GreaterThanComparisonSign ValueComparisonSign = ">" + NotEqualsComparisonSign ValueComparisonSign = "!=" ) // EscapeName will escape a db/table/column/... name by wrapping with backticks. diff --git a/go/sql/types.go b/go/sql/types.go index 9c7a660..ef83819 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -15,13 +15,13 @@ import ( type ColumnType int const ( - UnknownColumnType ColumnType = iota - TimestampColumnType = iota - DateTimeColumnType = iota - EnumColumnType = iota - MediumIntColumnType = iota - JSONColumnType = iota - FloatColumnType = iota + UnknownColumnType ColumnType = iota + TimestampColumnType + DateTimeColumnType + EnumColumnType + MediumIntColumnType + JSONColumnType + FloatColumnType ) const maxMediumintUnsigned int32 = 16777215 From c6a6a533a1e5d82bfaa50f11286bd713c0914ce4 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Fri, 4 May 2018 08:22:54 +0200 Subject: [PATCH 030/176] go/base: Do not ignore f.Close error --- go/base/utils.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/go/base/utils.go b/go/base/utils.go index d6bbf47..fc1858d 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -40,10 +40,9 @@ func FileExists(fileName string) bool { func TouchFile(fileName string) error { f, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE, 0755) if err != nil { - return (err) + return err } - defer f.Close() - return nil + return f.Close() } // StringContainsAll returns true if `s` contains all non empty given `substrings` From fb00a1387161d2678e280dc55eb380c9a2a8da3a Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 6 May 2018 11:19:03 +0300 Subject: [PATCH 031/176] Rejecting RENAME TO|AS --- go/logic/migrator.go | 4 +++ go/sql/parser.go | 12 +++++++ go/sql/parser_test.go | 39 +++++++++++++++++++++ localtests/fail-rename-table/create.sql | 22 ++++++++++++ localtests/fail-rename-table/expect_failure | 1 + localtests/fail-rename-table/extra_args | 1 + 6 files changed, 79 insertions(+) create mode 100644 localtests/fail-rename-table/create.sql create mode 100644 localtests/fail-rename-table/expect_failure create mode 100644 localtests/fail-rename-table/extra_args diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 673a98e..f52e2b8 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -255,7 +255,11 @@ func (this *Migrator) listenOnPanicAbort() { // validateStatement validates the `alter` statement meets criteria. // At this time this means: // - column renames are approved +// - no table rename allowed func (this *Migrator) validateStatement() (err error) { + if this.parser.IsRenameTable() { + return fmt.Errorf("ALTER statement seems to RENAME the table. This is not supported, and you should run your RENAME outside gh-ost.") + } if this.parser.HasNonTrivialRenames() && !this.migrationContext.SkipRenamedColumns { this.migrationContext.ColumnRenameMap = this.parser.GetNonTrivialRenames() if !this.migrationContext.ApproveRenamedColumns { diff --git a/go/sql/parser.go b/go/sql/parser.go index 7114e10..447cbcc 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -15,11 +15,13 @@ var ( sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')") renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) + renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) ) type Parser struct { columnRenameMap map[string]string droppedColumns map[string]bool + isRenameTable bool } func NewParser() *Parser { @@ -86,6 +88,12 @@ func (this *Parser) parseAlterToken(alterToken string) (err error) { this.droppedColumns[submatch[2]] = true } } + { + // rename table + if renameTableRegexp.MatchString(alterToken) { + this.isRenameTable = true + } + } return nil } @@ -115,3 +123,7 @@ func (this *Parser) HasNonTrivialRenames() bool { func (this *Parser) DroppedColumnsMap() map[string]bool { return this.droppedColumns } + +func (this *Parser) IsRenameTable() bool { + return this.isRenameTable +} diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 3e1d845..5d38130 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -159,3 +159,42 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["b"]) } } + +func TestParseAlterStatementRenameTable(t *testing.T) { + + { + parser := NewParser() + statement := "drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectFalse(parser.isRenameTable) + } + { + parser := NewParser() + statement := "rename as something_else" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectTrue(parser.isRenameTable) + } + { + parser := NewParser() + statement := "drop column b, rename as something_else" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectTrue(parser.isRenameTable) + } + { + parser := NewParser() + statement := "engine=innodb rename as something_else" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectTrue(parser.isRenameTable) + } + { + parser := NewParser() + statement := "rename as something_else, engine=innodb" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectTrue(parser.isRenameTable) + } +} diff --git a/localtests/fail-rename-table/create.sql b/localtests/fail-rename-table/create.sql new file mode 100644 index 0000000..5bb45f2 --- /dev/null +++ b/localtests/fail-rename-table/create.sql @@ -0,0 +1,22 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + ts timestamp, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, now()); + insert into gh_ost_test values (null, 13, now()); + insert into gh_ost_test values (null, 17, now()); +end ;; diff --git a/localtests/fail-rename-table/expect_failure b/localtests/fail-rename-table/expect_failure new file mode 100644 index 0000000..e444c17 --- /dev/null +++ b/localtests/fail-rename-table/expect_failure @@ -0,0 +1 @@ +ALTER statement seems to RENAME the table diff --git a/localtests/fail-rename-table/extra_args b/localtests/fail-rename-table/extra_args new file mode 100644 index 0000000..28a7587 --- /dev/null +++ b/localtests/fail-rename-table/extra_args @@ -0,0 +1 @@ +--alter="rename as something_else" From b23f0a6874b146d4ca46e241d37ea38478135b07 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 7 May 2018 12:34:42 +0300 Subject: [PATCH 032/176] build.sh: supporting go1.9 and above --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index ad6eeee..4f38512 100755 --- a/build.sh +++ b/build.sh @@ -10,7 +10,7 @@ function build { GOOS=$3 GOARCH=$4 - if [[ $(go version | egrep "go1[.][012345678]") ]]; then + if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then echo "go version is too low. Must use 1.9 or above" exit 1 fi From 37ed92d4118815c4c1602d2bc4b3668a7b6cc733 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 7 May 2018 19:03:32 +0300 Subject: [PATCH 033/176] go version, updated README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2138109..008c65b 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d [Download latest release here](https://github.com/github/gh-ost/releases/latest) -`gh-ost` is a Go project; it is built with Go `1.8` (though `1.7` should work as well). To build on your own, use either: +`gh-ost` is a Go project; it is built with Go `1.9` and above. To build on your own, use either: - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary. - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost` From 8cbfa26adf22c393b06125876c53ee47e9772020 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 7 May 2018 19:07:38 +0300 Subject: [PATCH 034/176] updated limitations --- doc/requirements-and-limitations.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index e12f3f0..3d174a7 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -52,3 +52,5 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - If you have en `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520) - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles. + +- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). \ No newline at end of file From 80387d91603b6ca1a0c38d9a0bb987843604f3c6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 15 May 2018 12:45:45 +0300 Subject: [PATCH 035/176] experiment: handle zero MYSQL_TYPE_DATETIME --- .../github.com/siddontang/go-mysql/replication/row_event.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index c30d9ae..0465947 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -400,6 +400,10 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ case MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) + + if i64 == 0 { + return "0000-00-00 00:00:00", n, nil + } d := i64 / 1000000 t := i64 % 1000000 v = time.Date(int(d/10000), From 08637009a6c6d81f1a78a1a97f429eddfc73963c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 15 May 2018 13:47:33 +0300 Subject: [PATCH 036/176] supporting modified sql_mode --- localtests/datetime-1970/sql_mode | 0 localtests/test.sh | 12 ++++++++++++ 2 files changed, 12 insertions(+) create mode 100644 localtests/datetime-1970/sql_mode diff --git a/localtests/datetime-1970/sql_mode b/localtests/datetime-1970/sql_mode new file mode 100644 index 0000000..e69de29 diff --git a/localtests/test.sh b/localtests/test.sh index 80bacaa..41c9822 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -87,6 +87,7 @@ start_replication() { test_single() { local test_name test_name="$1" + original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" if [ -f $tests_path/$test_name/ignore_versions ] ; then ignore_versions=$(cat $tests_path/$test_name/ignore_versions) @@ -102,6 +103,12 @@ test_single() { echo_dot start_replication echo_dot + + if [ -f $tests_path/$test_name/sql_mode ] ; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + fi + gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql extra_args="" @@ -154,6 +161,11 @@ test_single() { execution_result=$? + if [ -f $tests_path/$test_name/sql_mode ] ; then + gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" + fi + if [ -f $tests_path/$test_name/destroy.sql ] ; then gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/destroy.sql fi From 0f9a26ef6c4b3b4c5bb1ce8f83ebcfa45cdcca8c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 15 May 2018 14:03:10 +0300 Subject: [PATCH 037/176] experiment: removing patch --- .../github.com/siddontang/go-mysql/replication/row_event.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index 0465947..3cf2186 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -401,9 +401,9 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ n = 8 i64 := binary.LittleEndian.Uint64(data) - if i64 == 0 { - return "0000-00-00 00:00:00", n, nil - } + // if i64 == 0 { + // return "0000-00-00 00:00:00", n, nil + // } d := i64 / 1000000 t := i64 % 1000000 v = time.Date(int(d/10000), From 1f772dfcd9adc0e83c2b052b95c516f471135bfe Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 15 May 2018 15:07:12 +0300 Subject: [PATCH 038/176] experiment done: restoring patch --- .../github.com/siddontang/go-mysql/replication/row_event.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index 3cf2186..85a3e43 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -401,9 +401,9 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ n = 8 i64 := binary.LittleEndian.Uint64(data) - // if i64 == 0 { - // return "0000-00-00 00:00:00", n, nil - // } + if i64 == 0 { // commented by Shlomi Noach. Yes I know about `git blame` + return "0000-00-00 00:00:00", n, nil + } d := i64 / 1000000 t := i64 % 1000000 v = time.Date(int(d/10000), From dfeefed5dc0d8704799448fa90cb3d3c1bf0609b Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 20 May 2018 13:46:33 +0300 Subject: [PATCH 039/176] avoid prepared statements for changelog table --- go/logic/applier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 0a2951e..afdc45f 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -290,7 +290,7 @@ func (this *Applier) WriteChangelog(hint, value string) (string, error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetChangelogTableName()), ) - _, err := sqlutils.Exec(this.db, query, explicitId, hint, value) + _, err := sqlutils.ExecNoPrepare(this.db, query, explicitId, hint, value) return hint, err } From db871b42c4947a8657b7860452bff0b47ada20ae Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 12:36:52 +0300 Subject: [PATCH 040/176] Support for GENERATED (aka virtual) columns --- go/base/context.go | 2 ++ go/logic/applier.go | 2 +- go/logic/inspect.go | 30 ++++++++++++------- go/mysql/utils.go | 17 +++++++---- localtests/generated-columns57/create.sql | 30 +++++++++++++++++++ .../generated-columns57/ignore_versions | 1 + localtests/json57dml/create.sql | 2 +- 7 files changed, 67 insertions(+), 17 deletions(-) create mode 100644 localtests/generated-columns57/create.sql create mode 100644 localtests/generated-columns57/ignore_versions diff --git a/go/base/context.go b/go/base/context.go index 94039e3..2357c54 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -186,8 +186,10 @@ type MigrationContext struct { OriginalTableColumnsOnApplier *sql.ColumnList OriginalTableColumns *sql.ColumnList + OriginalTableVirtualColumns *sql.ColumnList OriginalTableUniqueKeys [](*sql.UniqueKey) GhostTableColumns *sql.ColumnList + GhostTableVirtualColumns *sql.ColumnList GhostTableUniqueKeys [](*sql.UniqueKey) UniqueKey *sql.UniqueKey SharedColumns *sql.ColumnList diff --git a/go/logic/applier.go b/go/logic/applier.go index afdc45f..ae59f62 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -117,7 +117,7 @@ func (this *Applier) validateAndReadTimeZone() error { // readTableColumns reads table columns on applier func (this *Applier) readTableColumns() (err error) { log.Infof("Examining table structure on applier") - this.migrationContext.OriginalTableColumnsOnApplier, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) + this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) if err != nil { return err } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 2568afe..dc69b81 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -89,24 +89,24 @@ func (this *Inspector) ValidateOriginalTable() (err error) { return nil } -func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) { +func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, virtualColumns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) { uniqueKeys, err = this.getCandidateUniqueKeys(tableName) if err != nil { - return columns, uniqueKeys, err + return columns, virtualColumns, uniqueKeys, err } if len(uniqueKeys) == 0 { - return columns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out") + return columns, virtualColumns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out") } - columns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName) + columns, virtualColumns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName) if err != nil { - return columns, uniqueKeys, err + return columns, virtualColumns, uniqueKeys, err } - return columns, uniqueKeys, nil + return columns, virtualColumns, uniqueKeys, nil } func (this *Inspector) InspectOriginalTable() (err error) { - this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName) + this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName) if err != nil { return err } @@ -122,7 +122,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { return fmt.Errorf("It seems like table structure is not identical between master and replica. This scenario is not supported.") } - this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName()) + this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName()) if err != nil { return err } @@ -166,7 +166,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { } } - this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.ColumnRenameMap) + this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) // By fact that a non-empty unique key exists we also know the shared columns are non-empty @@ -692,7 +692,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [ } // getSharedColumns returns the intersection of two lists of columns in same order as the first list -func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) { +func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) { sharedColumnNames := []string{} for _, originalColumn := range originalColumns.Names() { isSharedColumn := false @@ -709,6 +709,16 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum isSharedColumn = false } } + for _, virtualColumn := range originalVirtualColumns.Names() { + if strings.EqualFold(originalColumn, virtualColumn) { + isSharedColumn = false + } + } + for _, virtualColumn := range ghostVirtualColumns.Names() { + if strings.EqualFold(originalColumn, virtualColumn) { + isSharedColumn = false + } + } if isSharedColumn { sharedColumnNames = append(sharedColumnNames, originalColumn) } diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4..67ed7f8 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -8,6 +8,7 @@ package mysql import ( gosql "database/sql" "fmt" + "strings" "sync" "time" @@ -178,7 +179,7 @@ func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { } // GetTableColumns reads column list from given table -func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, error) { +func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, *sql.ColumnList, error) { query := fmt.Sprintf(` show columns from %s.%s `, @@ -186,18 +187,24 @@ func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnL sql.EscapeName(tableName), ) columnNames := []string{} + virtualColumnNames := []string{} err := sqlutils.QueryRowsMap(db, query, func(rowMap sqlutils.RowMap) error { - columnNames = append(columnNames, rowMap.GetString("Field")) + columnName := rowMap.GetString("Field") + columnNames = append(columnNames, columnName) + if strings.Contains(rowMap.GetString("Extra"), " GENERATED") { + log.Debugf("%s is a generated column", columnName) + virtualColumnNames = append(virtualColumnNames, columnName) + } return nil }) if err != nil { - return nil, err + return nil, nil, err } if len(columnNames) == 0 { - return nil, log.Errorf("Found 0 columns on %s.%s. Bailing out", + return nil, nil, log.Errorf("Found 0 columns on %s.%s. Bailing out", sql.EscapeName(databaseName), sql.EscapeName(tableName), ) } - return sql.NewColumnList(columnNames), nil + return sql.NewColumnList(columnNames), sql.NewColumnList(virtualColumnNames), nil } diff --git a/localtests/generated-columns57/create.sql b/localtests/generated-columns57/create.sql new file mode 100644 index 0000000..e244ca3 --- /dev/null +++ b/localtests/generated-columns57/create.sql @@ -0,0 +1,30 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + sum_ab int as (a + b) virtual not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns57/ignore_versions b/localtests/generated-columns57/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/generated-columns57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/json57dml/create.sql b/localtests/json57dml/create.sql index ac20941..0e76b2e 100644 --- a/localtests/json57dml/create.sql +++ b/localtests/json57dml/create.sql @@ -19,7 +19,7 @@ create event gh_ost_test begin insert into gh_ost_test (id, i, j) values (null, 11, '"sometext"'); insert into gh_ost_test (id, i, j) values (null, 13, '{"key":"val"}'); - insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); + insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); insert into gh_ost_test (id, i, j) values (null, 19, '{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}'); update gh_ost_test set j = '{"updated": 11}', updated = 1 where i = 11 and updated = 0; From 03b8dadda691d9080075c6a71ad41cb734102fa6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 12:55:57 +0300 Subject: [PATCH 041/176] added generated column rename test --- .../generated-columns-rename57/create.sql | 30 +++++++++++++++++++ .../generated-columns-rename57/extra_args | 1 + .../ignore_versions | 1 + 3 files changed, 32 insertions(+) create mode 100644 localtests/generated-columns-rename57/create.sql create mode 100644 localtests/generated-columns-rename57/extra_args create mode 100644 localtests/generated-columns-rename57/ignore_versions diff --git a/localtests/generated-columns-rename57/create.sql b/localtests/generated-columns-rename57/create.sql new file mode 100644 index 0000000..e244ca3 --- /dev/null +++ b/localtests/generated-columns-rename57/create.sql @@ -0,0 +1,30 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + sum_ab int as (a + b) virtual not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns-rename57/extra_args b/localtests/generated-columns-rename57/extra_args new file mode 100644 index 0000000..6a19098 --- /dev/null +++ b/localtests/generated-columns-rename57/extra_args @@ -0,0 +1 @@ +--alter="change sum_ab total_ab int as (a + b) virtual not null" --approve-renamed-columns diff --git a/localtests/generated-columns-rename57/ignore_versions b/localtests/generated-columns-rename57/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/generated-columns-rename57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) From 5275ad93421d89d2bbb0c24a675e6ddca3923b8c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 13:05:12 +0300 Subject: [PATCH 042/176] updated docs --- doc/requirements-and-limitations.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 3d174a7..99e3ee3 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -26,10 +26,6 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Triggers are not supported. They may be supported in the future. -- MySQL 5.7 generated columns are not supported. They may be supported in the future. - -- MySQL 5.7 `POINT` column type is not supported. - - MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY` - The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md) @@ -53,4 +49,4 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles. -- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). \ No newline at end of file +- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). From 2c6b79c4e78916cc7cc0068a4e95dc5cf4709c42 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 14:09:48 +0300 Subject: [PATCH 043/176] test: add generated column --- localtests/generated-columns-add57/create.sql | 29 +++++++++++++++++++ localtests/generated-columns-add57/extra_args | 1 + .../generated-columns-add57/ghost_columns | 1 + .../generated-columns-add57/ignore_versions | 1 + localtests/generated-columns-add57/order_by | 1 + .../generated-columns-add57/orig_columns | 1 + 6 files changed, 34 insertions(+) create mode 100644 localtests/generated-columns-add57/create.sql create mode 100644 localtests/generated-columns-add57/extra_args create mode 100644 localtests/generated-columns-add57/ghost_columns create mode 100644 localtests/generated-columns-add57/ignore_versions create mode 100644 localtests/generated-columns-add57/order_by create mode 100644 localtests/generated-columns-add57/orig_columns diff --git a/localtests/generated-columns-add57/create.sql b/localtests/generated-columns-add57/create.sql new file mode 100644 index 0000000..ef795ba --- /dev/null +++ b/localtests/generated-columns-add57/create.sql @@ -0,0 +1,29 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns-add57/extra_args b/localtests/generated-columns-add57/extra_args new file mode 100644 index 0000000..b2bf5bc --- /dev/null +++ b/localtests/generated-columns-add57/extra_args @@ -0,0 +1 @@ +--alter="add column sum_ab int as (a + b) virtual not null" diff --git a/localtests/generated-columns-add57/ghost_columns b/localtests/generated-columns-add57/ghost_columns new file mode 100644 index 0000000..bd17155 --- /dev/null +++ b/localtests/generated-columns-add57/ghost_columns @@ -0,0 +1 @@ +id, a, b diff --git a/localtests/generated-columns-add57/ignore_versions b/localtests/generated-columns-add57/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/generated-columns-add57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/generated-columns-add57/order_by b/localtests/generated-columns-add57/order_by new file mode 100644 index 0000000..074d1ee --- /dev/null +++ b/localtests/generated-columns-add57/order_by @@ -0,0 +1 @@ +id diff --git a/localtests/generated-columns-add57/orig_columns b/localtests/generated-columns-add57/orig_columns new file mode 100644 index 0000000..bd17155 --- /dev/null +++ b/localtests/generated-columns-add57/orig_columns @@ -0,0 +1 @@ +id, a, b From 21d455013e9d8ce3b4892e44e3750aaf4fb58d51 Mon Sep 17 00:00:00 2001 From: MOON_CLJ Date: Fri, 27 Apr 2018 14:58:07 +0800 Subject: [PATCH 044/176] minor changes and typo correction --- go/binlog/binlog_entry.go | 4 ++-- go/binlog/gomysql_reader.go | 2 +- go/logic/applier.go | 10 +++++++--- go/logic/inspect.go | 3 +++ go/logic/migrator.go | 2 +- go/mysql/utils.go | 4 ---- go/sql/builder.go | 3 +-- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index bb70bc5..5650acc 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -26,7 +26,7 @@ func NewBinlogEntry(logFile string, logPos uint64) *BinlogEntry { return binlogEntry } -// NewBinlogEntry creates an empty, ready to go BinlogEntry object +// NewBinlogEntryAt creates an empty, ready to go BinlogEntry object func NewBinlogEntryAt(coordinates mysql.BinlogCoordinates) *BinlogEntry { binlogEntry := &BinlogEntry{ Coordinates: coordinates, @@ -41,7 +41,7 @@ func (this *BinlogEntry) Duplicate() *BinlogEntry { return binlogEntry } -// Duplicate creates and returns a new binlog entry, with some of the attributes pre-assigned +// String() returns a string representation of this binlog entry func (this *BinlogEntry) String() string { return fmt.Sprintf("[BinlogEntry at %+v; dml:%+v]", this.Coordinates, this.DmlEvent) } diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index a479c99..b86bd72 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -111,7 +111,7 @@ func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEven binlogEntry.DmlEvent.WhereColumnValues = sql.ToColumnValues(row) } } - // The channel will do the throttling. Whoever is reding from the channel + // The channel will do the throttling. Whoever is reading from the channel // decides whether action is taken synchronously (meaning we wait before // next iteration) or asynchronously (we keep pushing more events) // In reality, reads will be synchronous diff --git a/go/logic/applier.go b/go/logic/applier.go index afdc45f..0de5da4 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -126,7 +126,6 @@ func (this *Applier) readTableColumns() (err error) { // showTableStatus returns the output of `show table status like '...'` command func (this *Applier) showTableStatus(tableName string) (rowMap sqlutils.RowMap) { - rowMap = nil query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(this.migrationContext.DatabaseName), tableName) sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error { rowMap = m @@ -482,6 +481,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected if err != nil { return nil, err } + defer tx.Rollback() sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') @@ -1001,15 +1001,19 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { if err != nil { return err } + rollback := func(err error) error { + tx.Rollback() + return err + } sessionQuery := `SET SESSION time_zone = '+00:00', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') ` if _, err := tx.Exec(sessionQuery); err != nil { - return err + return rollback(err) } if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil { - return err + return rollback(err) } if err := tx.Commit(); err != nil { return err diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 2568afe..4d25ab1 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -699,14 +699,17 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum for _, ghostColumn := range ghostColumns.Names() { if strings.EqualFold(originalColumn, ghostColumn) { isSharedColumn = true + break } if strings.EqualFold(columnRenameMap[originalColumn], ghostColumn) { isSharedColumn = true + break } } for droppedColumn := range this.migrationContext.DroppedColumnsMap { if strings.EqualFold(originalColumn, droppedColumn) { isSharedColumn = false + break } } if isSharedColumn { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f52e2b8..f536de8 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -78,7 +78,7 @@ type Migrator struct { rowCopyCompleteFlag int64 // copyRowsQueue should not be buffered; if buffered some non-damaging but - // excessive work happens at the end of the iteration as new copy-jobs arrive befroe realizing the copy is complete + // excessive work happens at the end of the iteration as new copy-jobs arrive before realizing the copy is complete copyRowsQueue chan tableWriteFunc applyEventsQueue chan *applyEventStruct diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4..2ca8cc2 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -83,9 +83,6 @@ func GetMasterKeyFromSlaveStatus(connectionConfig *ConnectionConfig) (masterKey } defer db.Close() - if err != nil { - return nil, err - } err = sqlutils.QueryRowsMap(db, `show slave status`, func(rowMap sqlutils.RowMap) error { // We wish to recognize the case where the topology's master actually has replication configuration. // This can happen when a DBA issues a `RESET SLAVE` instead of `RESET SLAVE ALL`. @@ -98,7 +95,6 @@ func GetMasterKeyFromSlaveStatus(connectionConfig *ConnectionConfig) (masterKey slaveIORunning := rowMap.GetString("Slave_IO_Running") slaveSQLRunning := rowMap.GetString("Slave_SQL_Running") - // if slaveIORunning != "Yes" || slaveSQLRunning != "Yes" { return fmt.Errorf("Replication on %+v is broken: Slave_IO_Running: %s, Slave_SQL_Running: %s. Please make sure replication runs before using gh-ost.", connectionConfig.Key, diff --git a/go/sql/builder.go b/go/sql/builder.go index c3a6229..2c5a7ae 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -140,13 +140,12 @@ func BuildRangeComparison(columns []string, values []string, args []interface{}, comparisons := []string{} for i, column := range columns { - // value := values[i] rangeComparison, err := BuildValueComparison(column, value, comparisonSign) if err != nil { return "", explodedArgs, err } - if len(columns[0:i]) > 0 { + if i > 0 { equalitiesComparison, err := BuildEqualsComparison(columns[0:i], values[0:i]) if err != nil { return "", explodedArgs, err From 14eda7efe06f5eed297c0f243bd870aa6b6de8b4 Mon Sep 17 00:00:00 2001 From: MOON_CLJ Date: Sat, 28 Apr 2018 12:20:28 +0800 Subject: [PATCH 045/176] fix GetReplicationLag not used args --- go/logic/inspect.go | 3 +-- go/logic/throttler.go | 4 ++-- go/mysql/utils.go | 5 ++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 4d25ab1..db6254d 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -758,9 +758,8 @@ func (this *Inspector) getMasterConnectionConfig() (applierConfig *mysql.Connect } func (this *Inspector) getReplicationLag() (replicationLag time.Duration, err error) { - replicationLag, err = mysql.GetReplicationLag( + replicationLag, err = mysql.GetReplicationLagFromSlaveStatus( this.informationSchemaDb, - this.migrationContext.InspectorConnectionConfig, ) return replicationLag, err } diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 624956a..6f9f4bb 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -140,8 +140,8 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica { // when running on replica, the heartbeat injection is also done on the replica. // This means we will always get a good heartbeat value. - // When runnign on replica, we should instead check the `SHOW SLAVE STATUS` output. - if lag, err := mysql.GetReplicationLag(this.inspector.informationSchemaDb, this.inspector.connectionConfig); err != nil { + // When running on replica, we should instead check the `SHOW SLAVE STATUS` output. + if lag, err := mysql.GetReplicationLagFromSlaveStatus(this.inspector.informationSchemaDb); err != nil { return log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 2ca8cc2..c96aebd 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -57,9 +57,8 @@ func GetDB(migrationUuid string, mysql_uri string) (*gosql.DB, bool, error) { return knownDBs[cacheKey], exists, nil } -// GetReplicationLag returns replication lag for a given connection config; either by explicit query -// or via SHOW SLAVE STATUS -func GetReplicationLag(informationSchemaDb *gosql.DB, connectionConfig *ConnectionConfig) (replicationLag time.Duration, err error) { +// GetReplicationLagFromSlaveStatus returns replication lag for a given db; via SHOW SLAVE STATUS +func GetReplicationLagFromSlaveStatus(informationSchemaDb *gosql.DB) (replicationLag time.Duration, err error) { err = sqlutils.QueryRowsMap(informationSchemaDb, `show slave status`, func(m sqlutils.RowMap) error { slaveIORunning := m.GetString("Slave_IO_Running") slaveSQLRunning := m.GetString("Slave_SQL_Running") From d20903e2605672a34a3003aa0468e38fe97af613 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 27 May 2018 20:29:09 +0300 Subject: [PATCH 046/176] Package builds --- build.sh | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/build.sh b/build.sh index 4f38512..7f44a8a 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ # # -RELEASE_VERSION=$(cat RELEASE_VERSION) +RELEASE_VERSION= function build { osname=$1 @@ -28,14 +28,26 @@ function build { (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${timestamp}.tar.gz $target) } -buildpath=/tmp/gh-ost -target=gh-ost -timestamp=$(date "+%Y%m%d%H%M%S") -ldflags="-X main.AppVersion=${RELEASE_VERSION}" +main() { + if [ -z "${RELEASE_VERSION}" ] ; then + RELEASE_VERSION=$(git describe --abbrev=0 --tags | tr -d 'v') + fi + if [ -z "${RELEASE_VERSION}" ] ; then + RELEASE_VERSION=$(cat RELEASE_VERSION) + fi -mkdir -p ${buildpath} -build macOS osx darwin amd64 -build GNU/Linux linux linux amd64 -echo "Binaries found in:" -ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz + buildpath=/tmp/gh-ost + target=gh-ost + timestamp=$(date "+%Y%m%d%H%M%S") + ldflags="-X main.AppVersion=${RELEASE_VERSION}" + + mkdir -p ${buildpath} + build macOS osx darwin amd64 + build GNU/Linux linux linux amd64 + + echo "Binaries found in:" + ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz +} + +main "$@" From 460b15d827b98e8423e995eeceb214a06b295cb3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 27 May 2018 20:49:08 +0300 Subject: [PATCH 047/176] rpm and deb --- build.sh | 55 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/build.sh b/build.sh index 7f44a8a..3094123 100755 --- a/build.sh +++ b/build.sh @@ -3,29 +3,50 @@ # RELEASE_VERSION= +buildpath= +builddir= + +function setuptree() { + mkdir -p $buildpath + rm -rf ${buildpath:?}/* + b=$( mktemp -d $buildpath/gh-ostXXXXXX ) || return 1 + mkdir -p $b/gh-ost + mkdir -p $b/gh-ost/usr/bin + echo $b +} function build { - osname=$1 - osshort=$2 - GOOS=$3 - GOARCH=$4 + osname=$1 + osshort=$2 + GOOS=$3 + GOARCH=$4 - if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then - echo "go version is too low. Must use 1.9 or above" + builddir=$(setuptree) + + + if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then + echo "go version is too low. Must use 1.9 or above" + exit 1 + fi + + echo "Building ${osname} binary" + export GOOS + export GOARCH + go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go + + if [ $? -ne 0 ]; then + echo "Build failed for ${osname}" exit 1 - fi + fi - echo "Building ${osname} binary" - export GOOS - export GOARCH - go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go + (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${timestamp}.tar.gz $target) - if [ $? -ne 0 ]; then - echo "Build failed for ${osname}" - exit 1 - fi - - (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${timestamp}.tar.gz $target) + if [ "$GOOS" == "linux" ] ; then + echo "Creating Distro full packages" + cp $buildpath/$target $builddir/gh-ost/usr/bin + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . + fi } main() { From 33e5974549ef24cb38560f168862ba29dd012067 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 27 May 2018 20:53:18 +0300 Subject: [PATCH 048/176] working directory --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 3094123..cfc55b8 100755 --- a/build.sh +++ b/build.sh @@ -44,6 +44,7 @@ function build { if [ "$GOOS" == "linux" ] ; then echo "Creating Distro full packages" cp $buildpath/$target $builddir/gh-ost/usr/bin + cd $buildpath fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . fi From 7da0cbc270993327eb54a506968a5bbd5013fc7b Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 27 May 2018 20:59:58 +0300 Subject: [PATCH 049/176] /tmp/gh-ost-release --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index cfc55b8..029a946 100755 --- a/build.sh +++ b/build.sh @@ -59,7 +59,7 @@ main() { fi - buildpath=/tmp/gh-ost + buildpath=/tmp/gh-ost-release target=gh-ost timestamp=$(date "+%Y%m%d%H%M%S") ldflags="-X main.AppVersion=${RELEASE_VERSION}" From 0454c5ecb081b24b7a1ce6326ad4e1234ed9bb35 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 27 May 2018 21:02:00 +0300 Subject: [PATCH 050/176] do not delete during build --- build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 029a946..78b2481 100755 --- a/build.sh +++ b/build.sh @@ -7,8 +7,6 @@ buildpath= builddir= function setuptree() { - mkdir -p $buildpath - rm -rf ${buildpath:?}/* b=$( mktemp -d $buildpath/gh-ostXXXXXX ) || return 1 mkdir -p $b/gh-ost mkdir -p $b/gh-ost/usr/bin @@ -65,6 +63,7 @@ main() { ldflags="-X main.AppVersion=${RELEASE_VERSION}" mkdir -p ${buildpath} + rm -rf ${buildpath:?}/* build macOS osx darwin amd64 build GNU/Linux linux linux amd64 From 609890aec1d81781eec2684ddb4e8445fa39a1e9 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 27 May 2018 21:03:44 +0300 Subject: [PATCH 051/176] only create builddir for linux --- build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 78b2481..264620b 100755 --- a/build.sh +++ b/build.sh @@ -4,7 +4,6 @@ RELEASE_VERSION= buildpath= -builddir= function setuptree() { b=$( mktemp -d $buildpath/gh-ostXXXXXX ) || return 1 @@ -19,7 +18,6 @@ function build { GOOS=$3 GOARCH=$4 - builddir=$(setuptree) if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then @@ -41,6 +39,7 @@ function build { if [ "$GOOS" == "linux" ] ; then echo "Creating Distro full packages" + builddir=$(setuptree) cp $buildpath/$target $builddir/gh-ost/usr/bin cd $buildpath fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . From 6524a15ed343e6780c0d4a2e75042e6cb4121aa3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 28 May 2018 07:49:51 +0300 Subject: [PATCH 052/176] release 1.0.46 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 2fec751..3c92cd0 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.45 +1.0.46 From 441f6331fdd01c53dd8821098fb12a604f4f15a2 Mon Sep 17 00:00:00 2001 From: Eliot Alter Date: Mon, 25 Jun 2018 15:33:02 -0700 Subject: [PATCH 053/176] Fix a small typo in the documentation --- doc/requirements-and-limitations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 99e3ee3..4415a30 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -45,7 +45,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Master-master setup is only supported in active-passive setup. Active-active (where table is being written to on both masters concurrently) is unsupported. It may be supported in the future. -- If you have en `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520) +- If you have an `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520) - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles. From c7dff99a194cc3693654746fab29becb8fa44eeb Mon Sep 17 00:00:00 2001 From: wangfei Date: Mon, 23 Jul 2018 09:29:21 +0800 Subject: [PATCH 054/176] fix copyRowsFunc hangs bug --- go/logic/migrator.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f52e2b8..e9a8fa3 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1087,24 +1087,30 @@ func (this *Migrator) iterateChunks() error { log.Debugf("No rows found in table. Rowcopy will be implicitly empty") return terminateRowIteration(nil) } + + var hasNoFurtherRangeFlag int64 // Iterate per chunk: for { - if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 { + if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 { // Done // There's another such check down the line return nil } copyRowsFunc := func() error { - if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 { + if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 { // Done. // There's another such check down the line return nil } + + // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever + hasFurtherRange, err := this.applier.CalculateNextIterationRangeEndValues() if err != nil { return terminateRowIteration(err) } if !hasFurtherRange { + atomic.StoreInt64(&hasNoFurtherRangeFlag, 1) return terminateRowIteration(nil) } // Copy task: From ef686a4e7bcd0e723200a232e25a567815cf6bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=92=9F=E8=B6=8A?= Date: Fri, 31 Aug 2018 00:40:09 +0800 Subject: [PATCH 055/176] Support database with underscore --- go/logic/inspect.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index dc69b81..3616981 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -233,6 +233,9 @@ func (this *Inspector) validateGrants() error { if strings.Contains(grant, fmt.Sprintf("GRANT ALL PRIVILEGES ON `%s`.*", this.migrationContext.DatabaseName)) { foundDBAll = true } + if strings.Contains(grant, fmt.Sprintf("GRANT ALL PRIVILEGES ON `%s`.*", strings.Replace(this.migrationContext.DatabaseName, "_", "\\_", -1))) { + foundDBAll = true + } if base.StringContainsAll(grant, `ALTER`, `CREATE`, `DELETE`, `DROP`, `INDEX`, `INSERT`, `LOCK TABLES`, `SELECT`, `TRIGGER`, `UPDATE`, ` ON *.*`) { foundDBAll = true } From 32f1cf1dbe4d9ed3cc31a847e5b87608ede6d636 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 2 Sep 2018 09:51:40 +0300 Subject: [PATCH 056/176] Support for GCP (Google Cloud Platform) --- doc/command-line-flags.md | 8 ++++++++ doc/requirements-and-limitations.md | 3 ++- go/base/context.go | 1 + go/base/utils.go | 3 ++- go/binlog/gomysql_reader.go | 2 +- go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 2 +- go/logic/inspect.go | 2 +- 8 files changed, 17 insertions(+), 5 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 4c66add..3df55e0 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -2,6 +2,10 @@ A more in-depth discussion of various `gh-ost` command line flags: implementation, implication, use cases. +### aliyun-rds + +Add this flag when executing on Aliyun RDS. + ### allow-master-master See [`--assume-master-host`](#assume-master-host). @@ -103,6 +107,10 @@ While the ongoing estimated number of rows is still heuristic, it's almost exact Without this parameter, migration is a _noop_: testing table creation and validity of migration, but not touching data. +### gcp + +Add this flag when executing on a 1st generation Google Cloud Platform (GCP). + ### heartbeat-interval-millis Default 100. See [`subsecond-lag`](subsecond-lag.md) for details. diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 4415a30..628451d 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -39,7 +39,8 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - For example, you may not migrate `MyTable` if another table called `MYtable` exists in the same schema. - Amazon RDS works, but has it's own [limitations](rds.md). -- Google Cloud SQL is currently not supported +- Google Cloud SQL works, `--gcp` flag required. +- Aliyun RDS works, `--aliyun-rds` flag required. - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`) diff --git a/go/base/context.go b/go/base/context.go index 2357c54..0043d07 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -92,6 +92,7 @@ type MigrationContext struct { IsTungsten bool DiscardForeignKeys bool AliyunRDS bool + GoogleCloudPlatform bool config ContextConfig configMutex *sync.Mutex diff --git a/go/base/utils.go b/go/base/utils.go index fc1858d..99536f8 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -75,7 +75,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, // swallow this error. not all servers support extra_port } // AliyunRDS set users port to "NULL", replace it by gh-ost param - if migrationContext.AliyunRDS { + // GCP set users port to "NULL", replace it by gh-ost param + if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform { port = connectionConfig.Key.Port } else { portQuery := `select @@global.port` diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index a479c99..d2797bb 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -145,7 +145,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates.LogFile = string(rotateEvent.NextLogName) }() - log.Infof("rotate to next log name: %s", rotateEvent.NextLogName) + log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName) } else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok { if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil { return err diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 30ac436..3024702 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -68,6 +68,7 @@ func main() { flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys") flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that") flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") + flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index ae59f62..ee9e066 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -89,7 +89,7 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.validateAndReadTimeZone(); err != nil { return err } - if !this.migrationContext.AliyunRDS { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { diff --git a/go/logic/inspect.go b/go/logic/inspect.go index dc69b81..ce1fcfb 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,7 +53,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if !this.migrationContext.AliyunRDS { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { From 09a005d79ee0f52c038e92ee6d992b84554c009c Mon Sep 17 00:00:00 2001 From: Jacob Bednarz Date: Tue, 18 Sep 2018 14:39:30 +1000 Subject: [PATCH 057/176] Populate the env with dry run status Updates the `applyEnvironmentVariables` function to populate whether or not the current execution context is running as a dry run or not which can then be used in hooks. --- doc/hooks.md | 1 + go/logic/hooks.go | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/hooks.md b/doc/hooks.md index 43ddb7e..1b1958f 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -69,6 +69,7 @@ The following variables are available on all hooks: - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` - `GH_OST_HOOKS_HINT` - copy of `--hooks-hint` value +- `GH_OST_DRY_RUN` - whether or not the `gh-ost` run is a dry run The following variable are available on particular hooks: diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 1fdfd5c..6515d7f 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -64,6 +64,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) + env = append(env, fmt.Sprintf("GH_OST_DRY_RUN=%t", this.migrationContext.Noop)) for _, variable := range extraVariables { env = append(env, variable) From 7bd338faf75d6a501e7bcb3ede5df12c2640f415 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 20 Sep 2018 10:49:21 +0300 Subject: [PATCH 058/176] Fix to retry logic in row copy --- go/logic/migrator.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index e9a8fa3..4178f19 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1105,8 +1105,11 @@ func (this *Migrator) iterateChunks() error { // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever - hasFurtherRange, err := this.applier.CalculateNextIterationRangeEndValues() - if err != nil { + hasFurtherRange := false + if err := this.retryOperation(func() (e error) { + hasFurtherRange, e = this.applier.CalculateNextIterationRangeEndValues() + return e + }); err != nil { return terminateRowIteration(err) } if !hasFurtherRange { @@ -1128,7 +1131,7 @@ func (this *Migrator) iterateChunks() error { } _, rowsAffected, _, err := this.applier.ApplyIterationInsertQuery() if err != nil { - return terminateRowIteration(err) + return err // wrapping call will retry } atomic.AddInt64(&this.migrationContext.TotalRowsCopied, rowsAffected) atomic.AddInt64(&this.migrationContext.Iteration, 1) From c1222a24165efd2cb9afdc7c0f010a1c999ea943 Mon Sep 17 00:00:00 2001 From: Gillian Gunson Date: Tue, 9 Oct 2018 22:02:38 -0700 Subject: [PATCH 059/176] Draft rewrite --- doc/shared-key.md | 69 ++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/doc/shared-key.md b/doc/shared-key.md index 480ad97..c5af2dd 100644 --- a/doc/shared-key.md +++ b/doc/shared-key.md @@ -1,12 +1,12 @@ # Shared key -A requirement for a migration to run is that the two _before_ and _after_ tables have a shared unique key. This is to elaborate and illustrate on the matter. +gh-ost requires for every migration that both the _before_ and _after_ versions of the table share the same unique not-null key columns. This page illustrates this rule. ### Introduction -Consider a classic, simple migration. The table is any normal: +Consider a simple migration, with a normal table, -``` +```sql CREATE TABLE tbl ( id bigint unsigned not null auto_increment, data varchar(255), @@ -15,54 +15,73 @@ CREATE TABLE tbl ( ) ``` -And the migration is a simple `add column ts timestamp`. +and the migration `add column ts timestamp`. The _after_ table version would be: -In such migration there is no change in indexes, and in particular no change to any unique key, and specifically no change to the `PRIMARY KEY`. To run this migration, `gh-ost` would iterate the `tbl` table using the primary key, copy rows from `tbl` to the _ghost_ table `_tbl_gho` by order of `id`, and then apply binlog events onto `_tbl_gho`. +```sql +CREATE TABLE tbl ( + id bigint unsigned not null auto_increment, + data varchar(255), + more_data int, + ts timestamp, + PRIMARY KEY(id) +) +``` -Applying the binlog events assumes the existence of a shared unique key. For example, an `UPDATE` statement in the binary log translate to a `REPLACE` statement which `gh-ost` applies to the _ghost_ table. Such statement expects to add or replace an existing row based on given row data. In particular, it would _replace_ an existing row if a unique key violation is met. +(This is also the definition of the _ghost_ table, except that that table would be called `_tbl_gho`). -So `gh-ost` correlates `tbl` and `_tbl_gho` rows using a unique key. In the above example that would be the `PRIMARY KEY`. +In this migration, the _before_ and _after_ versions contain the same unique not-null key (the PRIMARY KEY). To run this migration, `gh-ost` would iterate through the `tbl` table using the primary key, copy rows from `tbl` to the _ghost_ table `_tbl_gho` in primary key order, while also applying the binlog event writes from `tble` onto `_tbl_gho`. -### Rules +The applying of the binlog events is what requires the shared unique key. For example, an `UPDATE` statement to `tbl` translates to a `REPLACE` statement which `gh-ost` applies to `_tbl_gho`. A REPLACE statement expects to insert or replace an existing row based on its row's values and the table's unique key constraints. In particular, if inserting that row would result in a unique key violation (e.g., a row with that primary key already exists), it would _replace_ that existing row with the new values. -There must be a shared set of not-null columns for which there is a unique constraint in both the original table and the migration (_ghost_) table. +So `gh-ost` correlates `tbl` and `_tbl_gho` rows one to one using a unique key. In the above example that would be the `PRIMARY KEY`. -### Interpreting the rules +### Interpreting the rule -The same columns must be covered by a unique key in both tables. This doesn't have to be the `PRIMARY KEY`. This doesn't have to be a key of the same name. +The _before_ and _after_ versions of the table share the same unique not-null key, but: +- the key doesn't have to be the PRIMARY KEY +- the key can have a different name between the _before_ and _after_ versions (e.g., renamed via DROP INDEX and ADD INDEX) so long as it contains the exact same column(s) -Upon migration, `gh-ost` inspects both the original and _ghost_ table and attempts to find at least one such unique key (or rather, a set of columns) that is shared between the two. Typically this would just be the `PRIMARY KEY`, but sometimes you may change the `PRIMARY KEY` itself, in which case `gh-ost` will look for other options. +At the start of the migration, `gh-ost` inspects both the original and _ghost_ table it created, and attempts to find at least one such unique key (or rather, a set of columns) that is shared between the two. Typically this would just be the `PRIMARY KEY`, but some tables don't have primary keys, or sometimes the primary key is being modified. In these cases `gh-ost` will look for other options. -`gh-ost` expects unique keys where no `NULL` values are found, i.e. all columns covered by the unique key are defined as `NOT NULL`. This is implicitly true for `PRIMARY KEY`s. If no such key can be found, `gh-ost` bails out. In the event there is no such key, but you happen to _know_ your columns have no `NULL` values even though they're `NULL`-able, you may take responsibility and pass the `--allow-nullable-unique-key`. The migration will run well as long as no `NULL` values are found in the unique key's columns. Any actual `NULL`s may corrupt the migration. +`gh-ost` expects unique keys where no `NULL` values are found, i.e. all columns contained in the unique key are defined as `NOT NULL`. This is implicitly true for primary keys. If no such key can be found, `gh-ost` bails out. + +If the table contains a unique key with nullable columns, but you know your columns contain no `NULL` values, use the `--allow-nullable-unique-key` option. The migration will run well as long as no `NULL` values are found in the unique key's columns. Any actual `NULL`s may corrupt the migration. ### Examples: allowed and not allowed ``` create table some_table ( - id int auto_increment, + id int not null auto_increment, ts timestamp, name varchar(128) not null, owner_id int not null, - loc_id int, + loc_id int not null, primary key(id), unique key name_uidx(name) ) ``` -Following are examples of migrations that are _good to run_: +Note the two unique, not-null indexes: the primary key and `name_uidx`. + +Allowed migrations: - `add column i int` -- `add key owner_idx(owner_id)` -- `add unique key owner_name_idx(owner_id, name)` - though you need to make sure to not write conflicting rows while this migration runs +- `add key owner_idx (owner_id)` +- `add unique key owner_name_idx (owner_id, name)` - be sure to not write conflicting rows while this migration runs - `drop key name_uidx` - `primary key` is shared between the tables -- `drop primary key, add primary key(owner_id, loc_id)` - `name_uidx` is shared between the tables and is used for migration -- `change id bigint unsigned` - the `'primary key` is used. The change of type still makes the `primary key` workable. -- `drop primary key, drop key name_uidx, create primary key(name), create unique key id_uidx(id)` - swapping the two keys. `gh-ost` is still happy because `id` is still unique in both tables. So is `name`. +- `drop primary key, add primary key(owner_id, loc_id)` - `name_uidx` is shared between the tables +- `change id bigint unsigned not null auto_increment` - the `'primary key` changes datatype but not value, and can be used +- `drop primary key, drop key name_uidx, add primary key(name), add unique key id_uidx(id)` - swapping the two keys. Either `id` or `name` could be used -Following are examples of migrations that _cannot run_: +Not allowed: -- `drop primary key, drop key name_uidx` - no unique key to _ghost_ table, so clearly cannot run -- `drop primary key, drop key name_uidx, create primary key(name, owner_id)` - no shared columns to both tables. Even though `name` exists in the _ghost_ table's `primary key`, it is only part of the key and in itself does not guarantee uniqueness in the _ghost_ table. +- `drop primary key, drop key name_uidx` - the _ghost_ table has no unique key +- `drop primary key, drop key name_uidx, create primary key(name, owner_id)` - no shared columns to the unique keys on both tables. Even though `name` exists in the _ghost_ table's `primary key`, it is only part of the key and in itself does not guarantee uniqueness in the _ghost_ table. -Also, you cannot run a migration on a table that doesn't have some form of `unique key` in the first place, such as `some_table (id int, ts timestamp)` + +### Workarounds + +If you need to change your primary key or only not-null unique index to use different columns, you will want to do it as two separate migrations: +1. `ADD UNIQUE KEY temp_pk (temp_pk_column,...)` +1. `DROP PRIMARY KEY, DROP KEY temp_pk, ADD PRIMARY KEY (temp_pk_column,...)` From eb529065bf82574841cde5ef50ef3a0c0c4d1b83 Mon Sep 17 00:00:00 2001 From: Evaldas Grudzinskas Date: Mon, 15 Oct 2018 11:51:02 +0300 Subject: [PATCH 060/176] Add sql snippets highlighting --- doc/shared-key.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/shared-key.md b/doc/shared-key.md index 480ad97..76a9522 100644 --- a/doc/shared-key.md +++ b/doc/shared-key.md @@ -6,7 +6,7 @@ A requirement for a migration to run is that the two _before_ and _after_ tables Consider a classic, simple migration. The table is any normal: -``` +```sql CREATE TABLE tbl ( id bigint unsigned not null auto_increment, data varchar(255), @@ -37,7 +37,7 @@ Upon migration, `gh-ost` inspects both the original and _ghost_ table and attemp ### Examples: allowed and not allowed -``` +```sql create table some_table ( id int auto_increment, ts timestamp, From 600faf62af50b7ee6f36a514cb6cd14587e94297 Mon Sep 17 00:00:00 2001 From: Gillian Gunson Date: Mon, 15 Oct 2018 20:47:37 -0700 Subject: [PATCH 061/176] added emphasis as requested by Shlomi --- doc/shared-key.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/shared-key.md b/doc/shared-key.md index c5af2dd..5e27983 100644 --- a/doc/shared-key.md +++ b/doc/shared-key.md @@ -45,7 +45,7 @@ At the start of the migration, `gh-ost` inspects both the original and _ghost_ t `gh-ost` expects unique keys where no `NULL` values are found, i.e. all columns contained in the unique key are defined as `NOT NULL`. This is implicitly true for primary keys. If no such key can be found, `gh-ost` bails out. -If the table contains a unique key with nullable columns, but you know your columns contain no `NULL` values, use the `--allow-nullable-unique-key` option. The migration will run well as long as no `NULL` values are found in the unique key's columns. Any actual `NULL`s may corrupt the migration. +If the table contains a unique key with nullable columns, but you know your columns contain no `NULL` values, use the `--allow-nullable-unique-key` option. The migration will run well as long as no `NULL` values are found in the unique key's columns. **Any actual `NULL`s may corrupt the migration.** ### Examples: allowed and not allowed @@ -67,7 +67,7 @@ Allowed migrations: - `add column i int` - `add key owner_idx (owner_id)` -- `add unique key owner_name_idx (owner_id, name)` - be sure to not write conflicting rows while this migration runs +- `add unique key owner_name_idx (owner_id, name)` - **be careful not to write conflicting rows while this migration runs** - `drop key name_uidx` - `primary key` is shared between the tables - `drop primary key, add primary key(owner_id, loc_id)` - `name_uidx` is shared between the tables - `change id bigint unsigned not null auto_increment` - the `'primary key` changes datatype but not value, and can be used From b3cd0233c9cfa63734e21c1591be581790ea3773 Mon Sep 17 00:00:00 2001 From: Gillian Gunson Date: Mon, 15 Oct 2018 21:15:13 -0700 Subject: [PATCH 062/176] quick syntax edits --- doc/shared-key.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/shared-key.md b/doc/shared-key.md index 5e27983..0fecf49 100644 --- a/doc/shared-key.md +++ b/doc/shared-key.md @@ -31,7 +31,7 @@ CREATE TABLE tbl ( In this migration, the _before_ and _after_ versions contain the same unique not-null key (the PRIMARY KEY). To run this migration, `gh-ost` would iterate through the `tbl` table using the primary key, copy rows from `tbl` to the _ghost_ table `_tbl_gho` in primary key order, while also applying the binlog event writes from `tble` onto `_tbl_gho`. -The applying of the binlog events is what requires the shared unique key. For example, an `UPDATE` statement to `tbl` translates to a `REPLACE` statement which `gh-ost` applies to `_tbl_gho`. A REPLACE statement expects to insert or replace an existing row based on its row's values and the table's unique key constraints. In particular, if inserting that row would result in a unique key violation (e.g., a row with that primary key already exists), it would _replace_ that existing row with the new values. +The applying of the binlog events is what requires the shared unique key. For example, an `UPDATE` statement to `tbl` translates to a `REPLACE` statement which `gh-ost` applies to `_tbl_gho`. A `REPLACE` statement expects to insert or replace an existing row based on its row's values and the table's unique key constraints. In particular, if inserting that row would result in a unique key violation (e.g., a row with that primary key already exists), it would _replace_ that existing row with the new values. So `gh-ost` correlates `tbl` and `_tbl_gho` rows one to one using a unique key. In the above example that would be the `PRIMARY KEY`. @@ -41,13 +41,13 @@ The _before_ and _after_ versions of the table share the same unique not-null ke - the key doesn't have to be the PRIMARY KEY - the key can have a different name between the _before_ and _after_ versions (e.g., renamed via DROP INDEX and ADD INDEX) so long as it contains the exact same column(s) -At the start of the migration, `gh-ost` inspects both the original and _ghost_ table it created, and attempts to find at least one such unique key (or rather, a set of columns) that is shared between the two. Typically this would just be the `PRIMARY KEY`, but some tables don't have primary keys, or sometimes the primary key is being modified. In these cases `gh-ost` will look for other options. +At the start of the migration, `gh-ost` inspects both the original and _ghost_ table it created, and attempts to find at least one such unique key (or rather, a set of columns) that is shared between the two. Typically this would just be the `PRIMARY KEY`, but some tables don't have primary keys, or sometimes it is the primary key that is being modified by the migration. In these cases `gh-ost` will look for other options. `gh-ost` expects unique keys where no `NULL` values are found, i.e. all columns contained in the unique key are defined as `NOT NULL`. This is implicitly true for primary keys. If no such key can be found, `gh-ost` bails out. If the table contains a unique key with nullable columns, but you know your columns contain no `NULL` values, use the `--allow-nullable-unique-key` option. The migration will run well as long as no `NULL` values are found in the unique key's columns. **Any actual `NULL`s may corrupt the migration.** -### Examples: allowed and not allowed +### Examples: Allowed and Not Allowed ``` create table some_table ( @@ -70,10 +70,9 @@ Allowed migrations: - `add unique key owner_name_idx (owner_id, name)` - **be careful not to write conflicting rows while this migration runs** - `drop key name_uidx` - `primary key` is shared between the tables - `drop primary key, add primary key(owner_id, loc_id)` - `name_uidx` is shared between the tables -- `change id bigint unsigned not null auto_increment` - the `'primary key` changes datatype but not value, and can be used +- `change id bigint unsigned not null auto_increment` - the `primary key` changes datatype but not value, and can be used - `drop primary key, drop key name_uidx, add primary key(name), add unique key id_uidx(id)` - swapping the two keys. Either `id` or `name` could be used - Not allowed: - `drop primary key, drop key name_uidx` - the _ghost_ table has no unique key From 789d247d83c77c0936a12dbdf5317b5fba144c6c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 16 Oct 2018 09:47:30 +0300 Subject: [PATCH 063/176] localtests support throttle flag file --- localtests/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/localtests/test.sh b/localtests/test.sh index 41c9822..9cc10ed 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -14,6 +14,7 @@ ghost_binary="" exec_command_file=/tmp/gh-ost-test.bash orig_content_output_file=/tmp/gh-ost-test.orig.content.csv ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv +throttle_flag_file=/tmp/gh-ost-test.ghost.throttle.flag master_host= master_port= @@ -108,7 +109,7 @@ test_single() { gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" fi - + gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql extra_args="" @@ -145,6 +146,7 @@ test_single() { --initially-drop-old-table \ --initially-drop-ghost-table \ --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \ + --throttle-flag-file=$throttle_flag_file \ --serve-socket-file=/tmp/gh-ost.test.sock \ --initially-drop-socket-file \ --test-on-replica \ From b8e02df0feeb5cea96d05794e301f23be352497f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 16 Oct 2018 10:22:59 +0300 Subject: [PATCH 064/176] reading original_sql_mode once --- localtests/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/localtests/test.sh b/localtests/test.sh index 9cc10ed..c526a6b 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -20,6 +20,7 @@ master_host= master_port= replica_host= replica_port= +original_sql_mode= OPTIND=1 while getopts "b:" OPTION @@ -46,6 +47,8 @@ verify_master_and_replica() { echo "Cannot enable event_scheduler on master" exit 1 fi + original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" + echo "sql_mode on master is ${original_sql_mode}" if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then echo "Cannot verify gh-ost-test-mysql-replica" @@ -88,7 +91,6 @@ start_replication() { test_single() { local test_name test_name="$1" - original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" if [ -f $tests_path/$test_name/ignore_versions ] ; then ignore_versions=$(cat $tests_path/$test_name/ignore_versions) From 9512a2de40f3faf061cc03fa7d18acd8844f97c2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 16 Oct 2018 10:30:19 +0300 Subject: [PATCH 065/176] forcinf UTC timezone --- localtests/datetime-1970/create.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/localtests/datetime-1970/create.sql b/localtests/datetime-1970/create.sql index 9fcf9ef..f95357b 100644 --- a/localtests/datetime-1970/create.sql +++ b/localtests/datetime-1970/create.sql @@ -1,3 +1,5 @@ +set session time_zone='+00:00'; + drop table if exists gh_ost_test; create table gh_ost_test ( id int auto_increment, From 02602302a05acee1555d1e5383d6a33ee34696f6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 16 Oct 2018 11:25:46 +0300 Subject: [PATCH 066/176] more attempts at session time zone --- localtests/datetime-1970/create.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/localtests/datetime-1970/create.sql b/localtests/datetime-1970/create.sql index f95357b..a1914c6 100644 --- a/localtests/datetime-1970/create.sql +++ b/localtests/datetime-1970/create.sql @@ -9,6 +9,7 @@ create table gh_ost_test ( primary key(id) ) auto_increment=1; +set session time_zone='+00:00'; insert into gh_ost_test values (1, '0000-00-00 00:00:00', now(), 0); drop event if exists gh_ost_test; @@ -21,5 +22,6 @@ create event gh_ost_test enable do begin + set session time_zone='+00:00'; update gh_ost_test set counter = counter + 1 where id = 1; end ;; From f41df8e6094c8571ab7c912c87cf324405a90b36 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 16 Oct 2018 11:50:37 +0300 Subject: [PATCH 067/176] v1.0.47 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 3c92cd0..bde91a2 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.46 +1.0.47 From 967baf9e3cfb041beb4a2573a06a2b8ea48950c1 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Sat, 20 Oct 2018 11:43:32 +0200 Subject: [PATCH 068/176] Bump Travis versions --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 92cb79c..b6b68f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,9 @@ # http://docs.travis-ci.com/user/languages/go/ language: go -go: 1.9 +go: + - "1.9" + - "1.10" os: - linux From 48e01ad39b950a04ca45202baed279890a396b23 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 29 Oct 2018 10:09:15 +0200 Subject: [PATCH 069/176] Adding binary/varbinary tests --- localtests/varbinary/create.sql | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 localtests/varbinary/create.sql diff --git a/localtests/varbinary/create.sql b/localtests/varbinary/create.sql new file mode 100644 index 0000000..9d85200 --- /dev/null +++ b/localtests/varbinary/create.sql @@ -0,0 +1,40 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id binary(16) NOT NULL, + info varchar(255) COLLATE utf8_unicode_ci NOT NULL, + data binary(8) NOT NULL, + primary key (id), + unique key info_uidx (info) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + replace into gh_ost_test (id, info, data) values (X'12ffffffffffffffffffffffffffff00', 'item 1a', X'12ffffffffffffff'); + replace into gh_ost_test (id, info, data) values (X'34ffffffffffffffffffffffffffffff', 'item 3a', X'34ffffffffffffff'); + replace into gh_ost_test (id, info, data) values (X'90ffffffffffffffffffffffffffffff', 'item 9a', X'90ffffffffffff00'); + + DELETE FROM gh_ost_test WHERE id = X'11ffffffffffffffffffffffffffff00'; + UPDATE gh_ost_test SET info = 'item 2++' WHERE id = X'22ffffffffffffffffffffffffffff00'; + UPDATE gh_ost_test SET info = 'item 3++', data = X'33ffffffffffff00' WHERE id = X'33ffffffffffffffffffffffffffffff'; + DELETE FROM gh_ost_test WHERE id = X'44ffffffffffffffffffffffffffffff'; + UPDATE gh_ost_test SET info = 'item 5++', data = X'55ffffffffffffee' WHERE id = X'55ffffffffffffffffffffffffffffff'; + INSERT INTO gh_ost_test (id, info, data) VALUES (X'66ffffffffffffffffffffffffffff00', 'item 6', X'66ffffffffffffff'); + INSERT INTO gh_ost_test (id, info, data) VALUES (X'77ffffffffffffffffffffffffffffff', 'item 7', X'77ffffffffffff00'); + INSERT INTO gh_ost_test (id, info, data) VALUES (X'88ffffffffffffffffffffffffffffff', 'item 8', X'88ffffffffffffff'); +end ;; + +INSERT INTO gh_ost_test (id, info, data) VALUES + (X'11ffffffffffffffffffffffffffff00', 'item 1', X'11ffffffffffffff'), -- id ends in 00 + (X'22ffffffffffffffffffffffffffff00', 'item 2', X'22ffffffffffffff'), -- id ends in 00 + (X'33ffffffffffffffffffffffffffffff', 'item 3', X'33ffffffffffffff'), + (X'44ffffffffffffffffffffffffffffff', 'item 4', X'44ffffffffffffff'), + (X'55ffffffffffffffffffffffffffffff', 'item 5', X'55ffffffffffffff'), + (X'99ffffffffffffffffffffffffffffff', 'item 9', X'99ffffffffffff00'); -- data ends in 00 From c32823c10a05864e974096653f55edd02c70f4dd Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 1 Nov 2018 15:27:28 +0200 Subject: [PATCH 070/176] BIT datatype tests --- localtests/bit-add/create.sql | 20 ++++++++++++++++++++ localtests/bit-add/extra_args | 1 + localtests/bit-add/ghost_columns | 1 + localtests/bit-add/orig_columns | 1 + localtests/bit-dml/create.sql | 24 ++++++++++++++++++++++++ localtests/bit-dml/extra_args | 1 + 6 files changed, 48 insertions(+) create mode 100644 localtests/bit-add/create.sql create mode 100644 localtests/bit-add/extra_args create mode 100644 localtests/bit-add/ghost_columns create mode 100644 localtests/bit-add/orig_columns create mode 100644 localtests/bit-dml/create.sql create mode 100644 localtests/bit-dml/extra_args diff --git a/localtests/bit-add/create.sql b/localtests/bit-add/create.sql new file mode 100644 index 0000000..d58934a --- /dev/null +++ b/localtests/bit-add/create.sql @@ -0,0 +1,20 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11); + insert into gh_ost_test values (null, 13); +end ;; diff --git a/localtests/bit-add/extra_args b/localtests/bit-add/extra_args new file mode 100644 index 0000000..9d26250 --- /dev/null +++ b/localtests/bit-add/extra_args @@ -0,0 +1 @@ +--alter="add column is_good bit null default 0" diff --git a/localtests/bit-add/ghost_columns b/localtests/bit-add/ghost_columns new file mode 100644 index 0000000..b464f06 --- /dev/null +++ b/localtests/bit-add/ghost_columns @@ -0,0 +1 @@ +id, i diff --git a/localtests/bit-add/orig_columns b/localtests/bit-add/orig_columns new file mode 100644 index 0000000..b464f06 --- /dev/null +++ b/localtests/bit-add/orig_columns @@ -0,0 +1 @@ +id, i diff --git a/localtests/bit-dml/create.sql b/localtests/bit-dml/create.sql new file mode 100644 index 0000000..c7c26af --- /dev/null +++ b/localtests/bit-dml/create.sql @@ -0,0 +1,24 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + is_good bit null default 0, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, 0); + insert into gh_ost_test values (null, 13, 1); + insert into gh_ost_test values (null, 17, 1); + + update gh_ost_test set is_good=0 where i=13 order by id desc limit 1; +end ;; diff --git a/localtests/bit-dml/extra_args b/localtests/bit-dml/extra_args new file mode 100644 index 0000000..a3abab2 --- /dev/null +++ b/localtests/bit-dml/extra_args @@ -0,0 +1 @@ +--alter="modify column is_good bit not null default 0" --approve-renamed-columns From dfc8aa28f46f5c3868e6dcd033fb4fe883a14e1f Mon Sep 17 00:00:00 2001 From: Eduardo Nunes Date: Thu, 1 Nov 2018 14:33:11 +0100 Subject: [PATCH 071/176] Ignore not available columns when applying column types The given `columnLists` may not contain all columns available in the given table. This patch prevents the code to fail. Before this patch the code was panicking whenever it was trying to set a value into a `nil` object, e.g. `columnList.GetColumn('non-existant').Type = SomeType`. For some reason the application was not completely failing but as a side-effect any column after the non-existant column would never get its Type properly set. --- go/logic/inspect.go | 53 +++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index ce1fcfb..e87a5fa 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -552,44 +552,35 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL err := sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error { columnName := m.GetString("COLUMN_NAME") columnType := m.GetString("COLUMN_TYPE") - if strings.Contains(columnType, "unsigned") { - for _, columnsList := range columnsLists { - columnsList.SetUnsigned(columnName) + for _, columnsList := range columnsLists { + column := columnsList.GetColumn(columnName) + if column == nil { + continue } - } - if strings.Contains(columnType, "mediumint") { - for _, columnsList := range columnsLists { - columnsList.GetColumn(columnName).Type = sql.MediumIntColumnType + + if strings.Contains(columnType, "unsigned") { + column.IsUnsigned = true } - } - if strings.Contains(columnType, "timestamp") { - for _, columnsList := range columnsLists { - columnsList.GetColumn(columnName).Type = sql.TimestampColumnType + if strings.Contains(columnType, "mediumint") { + column.Type = sql.MediumIntColumnType } - } - if strings.Contains(columnType, "datetime") { - for _, columnsList := range columnsLists { - columnsList.GetColumn(columnName).Type = sql.DateTimeColumnType + if strings.Contains(columnType, "timestamp") { + column.Type = sql.TimestampColumnType } - } - if strings.Contains(columnType, "json") { - for _, columnsList := range columnsLists { - columnsList.GetColumn(columnName).Type = sql.JSONColumnType + if strings.Contains(columnType, "datetime") { + column.Type = sql.DateTimeColumnType } - } - if strings.Contains(columnType, "float") { - for _, columnsList := range columnsLists { - columnsList.GetColumn(columnName).Type = sql.FloatColumnType + if strings.Contains(columnType, "json") { + column.Type = sql.JSONColumnType } - } - if strings.HasPrefix(columnType, "enum") { - for _, columnsList := range columnsLists { - columnsList.GetColumn(columnName).Type = sql.EnumColumnType + if strings.Contains(columnType, "float") { + column.Type = sql.FloatColumnType } - } - if charset := m.GetString("CHARACTER_SET_NAME"); charset != "" { - for _, columnsList := range columnsLists { - columnsList.SetCharset(columnName, charset) + if strings.HasPrefix(columnType, "enum") { + column.Type = sql.EnumColumnType + } + if charset := m.GetString("CHARACTER_SET_NAME"); charset != "" { + column.Charset = charset } } return nil From 5f15f8a5390fdd5eb6790d5d3f6940d3741cea4d Mon Sep 17 00:00:00 2001 From: Eduardo Nunes Date: Thu, 1 Nov 2018 14:43:57 +0100 Subject: [PATCH 072/176] Remove not needed call by grouping columnLists There is no need to call `applyColumnTypes` more than once for the same `databaseName` and `tableName`, it is just move the additional `columnList` to the first call. --- go/logic/inspect.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index e87a5fa..c9744a8 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -173,8 +173,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { // This additional step looks at which columns are unsigned. We could have merged this within // the `getTableColumns()` function, but it's a later patch and introduces some complexity; I feel // comfortable in doing this as a separate step. - this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns) - this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &this.migrationContext.UniqueKey.Columns) + this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, &this.migrationContext.UniqueKey.Columns) this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.GhostTableColumns, this.migrationContext.MappedSharedColumns) for i := range this.migrationContext.SharedColumns.Columns() { From f98ef182785fd66b7b3b2b6fd1b9193ee3db2149 Mon Sep 17 00:00:00 2001 From: John Morrison Date: Thu, 1 Nov 2018 21:42:56 -0700 Subject: [PATCH 073/176] Correct the go SHA used in script/ensure-go-installed --- script/ensure-go-installed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/ensure-go-installed b/script/ensure-go-installed index c9ec36b..a511e0e 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -4,7 +4,7 @@ PREFERRED_GO_VERSION=go1.9.2 SUPPORTED_GO_VERSIONS='go1.[89]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=73fd5840d55f5566d8db6c0ffdd187577e8ebe650c783f68bd27cbf95bde6743 +GO_PKG_DARWIN_SHA=8b4f6ae6deae1150d2e341d02c247fd18a99af387516540eeb84702ffd76d3a1 GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz GO_PKG_LINUX_SHA=de874549d9a8d8d8062be05808509c09a88a248e77ec14eb77453530829ac02b From 13b17b93c432cc9ae6f061d29ba32be1fd8dda61 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 20 Nov 2018 19:35:03 +0200 Subject: [PATCH 074/176] testing: convert to character set utf8mb4 --- localtests/convert-utf8mb4/create.sql | 25 +++++++++++++++++++++++++ localtests/convert-utf8mb4/extra_args | 1 + 2 files changed, 26 insertions(+) create mode 100644 localtests/convert-utf8mb4/create.sql create mode 100644 localtests/convert-utf8mb4/extra_args diff --git a/localtests/convert-utf8mb4/create.sql b/localtests/convert-utf8mb4/create.sql new file mode 100644 index 0000000..cf36d5f --- /dev/null +++ b/localtests/convert-utf8mb4/create.sql @@ -0,0 +1,25 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + t varchar(128) charset utf8 collate utf8_general_ci, + ta varchar(128) charset ascii not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, md5(rand()), 'a'); + insert into gh_ost_test values (null, 'novo proprietário', 'b'); + insert into gh_ost_test values (null, 'usuário', 'c'); + insert into gh_ost_test values (null, 'usuário', 'x'); + + delete from gh_ost_test where ta='x' order by id desc limit 1; +end ;; diff --git a/localtests/convert-utf8mb4/extra_args b/localtests/convert-utf8mb4/extra_args new file mode 100644 index 0000000..2b2f64d --- /dev/null +++ b/localtests/convert-utf8mb4/extra_args @@ -0,0 +1 @@ +--alter='convert to character set utf8mb4' From 5d4d1cd31bd8a09760e6972311fec987f58a3c56 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 20 Nov 2018 19:37:51 +0200 Subject: [PATCH 075/176] more interesting sample data --- localtests/convert-utf8mb4/create.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/localtests/convert-utf8mb4/create.sql b/localtests/convert-utf8mb4/create.sql index cf36d5f..083174a 100644 --- a/localtests/convert-utf8mb4/create.sql +++ b/localtests/convert-utf8mb4/create.sql @@ -6,6 +6,8 @@ create table gh_ost_test ( primary key(id) ) auto_increment=1; +insert into gh_ost_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'initial'); + drop event if exists gh_ost_test; delimiter ;; create event gh_ost_test @@ -18,7 +20,7 @@ create event gh_ost_test begin insert into gh_ost_test values (null, md5(rand()), 'a'); insert into gh_ost_test values (null, 'novo proprietário', 'b'); - insert into gh_ost_test values (null, 'usuário', 'c'); + insert into gh_ost_test values (null, '2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm', 'c'); insert into gh_ost_test values (null, 'usuário', 'x'); delete from gh_ost_test where ta='x' order by id desc limit 1; From a7aa4eb2c03f58e92e845818b97c358f9d14aee1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 20 Nov 2018 19:48:59 +0200 Subject: [PATCH 076/176] adding latin1 --- localtests/convert-utf8mb4/create.sql | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/localtests/convert-utf8mb4/create.sql b/localtests/convert-utf8mb4/create.sql index 083174a..05f1a13 100644 --- a/localtests/convert-utf8mb4/create.sql +++ b/localtests/convert-utf8mb4/create.sql @@ -2,11 +2,15 @@ drop table if exists gh_ost_test; create table gh_ost_test ( id int auto_increment, t varchar(128) charset utf8 collate utf8_general_ci, + tl varchar(128) charset latin1 not null, ta varchar(128) charset ascii not null, primary key(id) ) auto_increment=1; -insert into gh_ost_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'initial'); +insert into gh_ost_test values (null, 'átesting'); + + +insert into gh_ost_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'átesting0', 'initial'); drop event if exists gh_ost_test; delimiter ;; @@ -18,10 +22,10 @@ create event gh_ost_test enable do begin - insert into gh_ost_test values (null, md5(rand()), 'a'); - insert into gh_ost_test values (null, 'novo proprietário', 'b'); - insert into gh_ost_test values (null, '2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm', 'c'); - insert into gh_ost_test values (null, 'usuário', 'x'); + insert into gh_ost_test values (null, md5(rand()), 'átesting-a', 'a'); + insert into gh_ost_test values (null, 'novo proprietário', 'átesting-b', 'b'); + insert into gh_ost_test values (null, '2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm', 'átesting-c', 'c'); + insert into gh_ost_test values (null, 'usuário', 'átesting-x', 'x'); delete from gh_ost_test where ta='x' order by id desc limit 1; end ;; From 00f5044f4ff49099343930dd5e5bc3768a7aa4b3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 28 Nov 2018 10:19:28 +0200 Subject: [PATCH 077/176] Testing DECIMAL datatype --- localtests/decimal/create.sql | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 localtests/decimal/create.sql diff --git a/localtests/decimal/create.sql b/localtests/decimal/create.sql new file mode 100644 index 0000000..248c86a --- /dev/null +++ b/localtests/decimal/create.sql @@ -0,0 +1,23 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + dec0 decimal(65,30) unsigned NOT NULL DEFAULT '0.000000000000000000000000000000', + dec1 decimal(65,30) unsigned NOT NULL DEFAULT '1.000000000000000000000000000000', + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 0.0, 0.0); + insert into gh_ost_test values (null, 2.0, 4.0); + insert into gh_ost_test values (null, 99999999999999999999999999999999999.000, 6.0); + update gh_ost_test set dec1=4.5 where dec2=4.0 order by id desc limit 1; +end ;; From 96d00d470e9586cc9465324e36cdb067714400b9 Mon Sep 17 00:00:00 2001 From: ClareCat Date: Thu, 13 Dec 2018 19:31:36 -0800 Subject: [PATCH 078/176] Update throttle.md to describe --throttle-HTTP --- doc/throttle.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/throttle.md b/doc/throttle.md index bc4a315..3d9ecaa 100644 --- a/doc/throttle.md +++ b/doc/throttle.md @@ -46,6 +46,14 @@ Note that you may dynamically change both `--max-lag-millis` and the `throttle-c An example query could be: `--throttle-query="select hour(now()) between 8 and 17"` which implies throttling auto-starts `8:00am` and migration auto-resumes at `18:00pm`. +#### HTTP Throttle + +The `--throttle-HTTP` flag allows for throttling via HTTP. Every 100ms `gh-ost` issues a `HEAD` request to the provided URL. If the response status code is not `200` throttling will kick in until a `200` response status code is returned. + +If no URL is provided or the URL provided doesn't contain the scheme then the HTTP check will be disabled. For example `--throttle-HTTP="http://1.2.3.4:6789/throttle"` will enable the HTTP check/throttling, but `--throttle-HTTP=1.2.3.4:6789/throttle` will not. + +The URL can be queried and updated dynamically via [interactive interface](interactive-commands.md). + #### Manual control In addition to the above, you are able to take control and throttle the operation any time you like. From 2c48be107cbd00d68870b60be906ab2150970c2c Mon Sep 17 00:00:00 2001 From: ClareCat Date: Thu, 13 Dec 2018 20:25:04 -0800 Subject: [PATCH 079/176] Update command-line-flags --- doc/command-line-flags.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 3df55e0..804ba58 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -69,6 +69,10 @@ This is somewhat similar to a Nagios `n`-times test, where `n` in our case is al Optional. Default is `safe`. See more discussion in [`cut-over`](cut-over.md) +### cut-over-lock-timeout-seconds + +Default `3`. Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout). + ### discard-foreign-keys **Danger**: this flag will _silently_ discard any foreign keys existing on your table. @@ -107,6 +111,10 @@ While the ongoing estimated number of rows is still heuristic, it's almost exact Without this parameter, migration is a _noop_: testing table creation and validity of migration, but not touching data. +### force-table-names + +Table name prefix to be used on the temporary tables. + ### gcp Add this flag when executing on a 1st generation Google Cloud Platform (GCP). @@ -125,6 +133,10 @@ We think `gh-ost` should not take chances or make assumptions about the user's t See [`initially-drop-ghost-table`](#initially-drop-ghost-table) +### initially-drop-socket-file + +Default False. Should `gh-ost` forcibly delete an existing socket file. Be careful: this might drop the socket file of a running migration! + ### max-lag-millis On a replication topology, this is perhaps the most important migration throttling factor: the maximum lag allowed for migration to work. If lag exceeds this value, migration throttles. @@ -169,6 +181,10 @@ See [`approve-renamed-columns`](#approve-renamed-columns) Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) +### test-on-replica-skip-replica-stop + +Default `False`. When `--test-on-replica` is enabled, do not issue commands stop replication (requires `--test-on-replica`). + ### throttle-control-replicas Provide a command delimited list of replicas; `gh-ost` will throttle when any of the given replicas lag beyond [`--max-lag-millis`](#max-lag-millis). The list can be queried and updated dynamically via [interactive commands](interactive-commands.md) From d5aab8e03de6e95eb33aabfc732c0d18ad4ede23 Mon Sep 17 00:00:00 2001 From: ClareCat Date: Thu, 13 Dec 2018 20:26:51 -0800 Subject: [PATCH 080/176] Add missing quotes --- doc/throttle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/throttle.md b/doc/throttle.md index 3d9ecaa..bf5dcb0 100644 --- a/doc/throttle.md +++ b/doc/throttle.md @@ -50,7 +50,7 @@ An example query could be: `--throttle-query="select hour(now()) between 8 and 1 The `--throttle-HTTP` flag allows for throttling via HTTP. Every 100ms `gh-ost` issues a `HEAD` request to the provided URL. If the response status code is not `200` throttling will kick in until a `200` response status code is returned. -If no URL is provided or the URL provided doesn't contain the scheme then the HTTP check will be disabled. For example `--throttle-HTTP="http://1.2.3.4:6789/throttle"` will enable the HTTP check/throttling, but `--throttle-HTTP=1.2.3.4:6789/throttle` will not. +If no URL is provided or the URL provided doesn't contain the scheme then the HTTP check will be disabled. For example `--throttle-HTTP="http://1.2.3.4:6789/throttle"` will enable the HTTP check/throttling, but `--throttle-HTTP="1.2.3.4:6789/throttle"` will not. The URL can be queried and updated dynamically via [interactive interface](interactive-commands.md). From 432e83e1969f1d9b4145f2d056bad430c6bc5a96 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Dec 2018 07:48:20 +0200 Subject: [PATCH 081/176] lower case arg name --- doc/throttle.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/throttle.md b/doc/throttle.md index bf5dcb0..2ebc2ee 100644 --- a/doc/throttle.md +++ b/doc/throttle.md @@ -48,9 +48,9 @@ An example query could be: `--throttle-query="select hour(now()) between 8 and 1 #### HTTP Throttle -The `--throttle-HTTP` flag allows for throttling via HTTP. Every 100ms `gh-ost` issues a `HEAD` request to the provided URL. If the response status code is not `200` throttling will kick in until a `200` response status code is returned. +The `--throttle-http` flag allows for throttling via HTTP. Every 100ms `gh-ost` issues a `HEAD` request to the provided URL. If the response status code is not `200` throttling will kick in until a `200` response status code is returned. -If no URL is provided or the URL provided doesn't contain the scheme then the HTTP check will be disabled. For example `--throttle-HTTP="http://1.2.3.4:6789/throttle"` will enable the HTTP check/throttling, but `--throttle-HTTP="1.2.3.4:6789/throttle"` will not. +If no URL is provided or the URL provided doesn't contain the scheme then the HTTP check will be disabled. For example `--throttle-http="http://1.2.3.4:6789/throttle"` will enable the HTTP check/throttling, but `--throttle-http="1.2.3.4:6789/throttle"` will not. The URL can be queried and updated dynamically via [interactive interface](interactive-commands.md). From 255314927d10b4c16804f6efae1fe2b2839f1875 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 1 Jan 2019 10:57:46 +0200 Subject: [PATCH 082/176] Updated go-mysql library, now with fix to decimal. May break datetime --- go/binlog/gomysql_reader.go | 13 +- .../siddontang/go-mysql/.travis.yml | 26 +- .../github.com/siddontang/go-mysql/Makefile | 23 +- .../github.com/siddontang/go-mysql/README.md | 90 +- .../vendor/github.com/BurntSushi/toml/COPYING | 14 - .../github.com/BurntSushi/toml/decode.go | 492 ------- .../github.com/BurntSushi/toml/decode_meta.go | 122 -- .../vendor/github.com/BurntSushi/toml/doc.go | 27 - .../github.com/BurntSushi/toml/encode.go | 551 -------- .../BurntSushi/toml/encoding_types.go | 19 - .../BurntSushi/toml/encoding_types_1.1.go | 18 - .../vendor/github.com/BurntSushi/toml/lex.go | 874 ------------ .../github.com/BurntSushi/toml/parse.go | 498 ------- .../github.com/BurntSushi/toml/type_check.go | 91 -- .../github.com/BurntSushi/toml/type_fields.go | 241 ---- .../github.com/go-sql-driver/mysql/LICENSE | 373 ----- .../go-sql-driver/mysql/appengine.go | 19 - .../github.com/go-sql-driver/mysql/buffer.go | 147 -- .../go-sql-driver/mysql/collations.go | 250 ---- .../go-sql-driver/mysql/connection.go | 372 ----- .../github.com/go-sql-driver/mysql/const.go | 163 --- .../github.com/go-sql-driver/mysql/driver.go | 167 --- .../github.com/go-sql-driver/mysql/dsn.go | 513 ------- .../github.com/go-sql-driver/mysql/errors.go | 131 -- .../github.com/go-sql-driver/mysql/infile.go | 181 --- .../github.com/go-sql-driver/mysql/packets.go | 1243 ----------------- .../github.com/go-sql-driver/mysql/result.go | 22 - .../github.com/go-sql-driver/mysql/rows.go | 112 -- .../go-sql-driver/mysql/statement.go | 150 -- .../go-sql-driver/mysql/transaction.go | 31 - .../github.com/go-sql-driver/mysql/utils.go | 740 ---------- .../vendor/github.com/jmoiron/sqlx/LICENSE | 23 - .../vendor/github.com/jmoiron/sqlx/bind.go | 186 --- .../vendor/github.com/jmoiron/sqlx/doc.go | 12 - .../vendor/github.com/jmoiron/sqlx/named.go | 336 ----- .../jmoiron/sqlx/reflectx/reflect.go | 371 ----- .../vendor/github.com/jmoiron/sqlx/sqlx.go | 992 ------------- .../vendor/github.com/juju/errors/LICENSE | 191 --- .../vendor/github.com/juju/errors/doc.go | 81 -- .../vendor/github.com/juju/errors/error.go | 145 -- .../github.com/juju/errors/errortypes.go | 284 ---- .../github.com/juju/errors/functions.go | 330 ----- .../vendor/github.com/juju/errors/path.go | 38 - .../vendor/github.com/ngaut/log/LICENSE | 165 --- .../vendor/github.com/ngaut/log/crash_unix.go | 18 - .../vendor/github.com/ngaut/log/crash_win.go | 37 - .../vendor/github.com/ngaut/log/log.go | 380 ----- .../github.com/pingcap/check/benchmark.go | 187 --- .../vendor/github.com/pingcap/check/check.go | 954 ------------- .../github.com/pingcap/check/checkers.go | 458 ------ .../github.com/pingcap/check/checkers2.go | 131 -- .../github.com/pingcap/check/compare.go | 161 --- .../github.com/pingcap/check/helpers.go | 231 --- .../github.com/pingcap/check/printer.go | 168 --- .../vendor/github.com/pingcap/check/run.go | 175 --- .../vendor/github.com/satori/go.uuid/LICENSE | 20 - .../vendor/github.com/satori/go.uuid/uuid.go | 488 ------- .../vendor/github.com/siddontang/go/LICENSE | 20 - .../github.com/siddontang/go/hack/hack.go | 27 - .../siddontang/go/ioutil2/ioutil.go | 39 - .../siddontang/go/ioutil2/sectionwriter.go | 69 - .../github.com/siddontang/go/sync2/atomic.go | 146 -- .../siddontang/go/sync2/semaphore.go | 65 - .../_vendor/vendor/golang.org/x/net/LICENSE | 27 - .../_vendor/vendor/golang.org/x/net/PATENTS | 22 - .../golang.org/x/net/context/context.go | 447 ------ .../siddontang/go-mysql/canal/canal.go | 337 +++-- .../siddontang/go-mysql/canal/canal_test.go | 169 ++- .../siddontang/go-mysql/canal/config.go | 42 +- .../siddontang/go-mysql/canal/dump.go | 128 +- .../siddontang/go-mysql/canal/handler.go | 56 +- .../siddontang/go-mysql/canal/master.go | 113 +- .../siddontang/go-mysql/canal/rows.go | 48 +- .../siddontang/go-mysql/canal/sync.go | 226 ++- .../siddontang/go-mysql/client/auth.go | 176 ++- .../siddontang/go-mysql/client/client_test.go | 69 +- .../siddontang/go-mysql/client/conn.go | 21 +- .../siddontang/go-mysql/client/resp.go | 120 +- .../go-mysql/cmd/go-binlogparser/main.go | 2 +- .../siddontang/go-mysql/cmd/go-canal/main.go | 33 +- .../go-mysql/cmd/go-mysqlbinlog/main.go | 11 +- .../siddontang/go-mysql/driver/dirver_test.go | 8 +- .../siddontang/go-mysql/driver/driver.go | 3 +- .../siddontang/go-mysql/dump/dump.go | 71 +- .../siddontang/go-mysql/dump/dump_test.go | 30 +- .../siddontang/go-mysql/dump/parser.go | 10 +- .../go-mysql/failover/mariadb_gtid_handler.go | 6 +- .../siddontang/go-mysql/failover/server.go | 4 +- .../github.com/siddontang/go-mysql/glide.lock | 30 - .../github.com/siddontang/go-mysql/glide.yaml | 26 - .../siddontang/go-mysql/mysql/const.go | 17 +- .../siddontang/go-mysql/mysql/error.go | 7 + .../siddontang/go-mysql/mysql/field.go | 12 +- .../siddontang/go-mysql/mysql/gtid.go | 4 + .../siddontang/go-mysql/mysql/mariadb_gtid.go | 202 ++- .../siddontang/go-mysql/mysql/mysql_gtid.go | 46 +- .../siddontang/go-mysql/mysql/mysql_test.go | 43 +- .../siddontang/go-mysql/mysql/resultset.go | 7 +- .../go-mysql/mysql/resultset_helper.go | 73 +- .../siddontang/go-mysql/mysql/util.go | 103 +- .../siddontang/go-mysql/packet/conn.go | 131 +- .../siddontang/go-mysql/replication/backup.go | 5 +- .../go-mysql/replication/binlogstreamer.go | 36 +- .../go-mysql/replication/binlogsyncer.go | 250 +++- .../siddontang/go-mysql/replication/event.go | 61 +- .../go-mysql/replication/json_binary.go | 16 +- .../siddontang/go-mysql/replication/parser.go | 207 ++- .../go-mysql/replication/parser_test.go | 2 +- .../go-mysql/replication/replication_test.go | 64 +- .../go-mysql/replication/row_event.go | 158 ++- .../go-mysql/replication/row_event_test.go | 610 +++++--- .../siddontang/go-mysql/schema/schema.go | 207 ++- .../siddontang/go-mysql/schema/schema_test.go | 34 +- .../siddontang/go-mysql/server/auth.go | 253 ++-- .../siddontang/go-mysql/server/command.go | 17 +- .../siddontang/go-mysql/server/conn.go | 76 +- .../siddontang/go-mysql/server/resp.go | 53 + .../siddontang/go-mysql/server/server_test.go | 254 ++-- .../siddontang/go-mysql/server/stmt.go | 10 +- 119 files changed, 3615 insertions(+), 16219 deletions(-) delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/COPYING delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode_meta.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/doc.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encode.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/lex.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/parse.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_check.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_fields.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/LICENSE delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/appengine.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/buffer.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/collations.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/connection.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/const.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/driver.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/dsn.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/errors.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/infile.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/packets.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/result.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/rows.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/statement.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/transaction.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/utils.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/LICENSE delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/bind.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/doc.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/named.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/sqlx.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/LICENSE delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/doc.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/error.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/errortypes.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/functions.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/path.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/LICENSE delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_unix.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_win.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/log.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/benchmark.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/check.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers2.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/compare.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/helpers.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/printer.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/run.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/LICENSE delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/uuid.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/LICENSE delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/hack/hack.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/ioutil.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/sectionwriter.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/atomic.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/semaphore.go delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/LICENSE delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/PATENTS delete mode 100644 vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/context/context.go mode change 100644 => 100755 vendor/github.com/siddontang/go-mysql/canal/canal_test.go delete mode 100644 vendor/github.com/siddontang/go-mysql/glide.lock delete mode 100644 vendor/github.com/siddontang/go-mysql/glide.yaml diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index d2797bb..106f73d 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -40,12 +40,13 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go serverId := uint32(migrationContext.ReplicaServerId) binlogSyncerConfig := replication.BinlogSyncerConfig{ - ServerID: serverId, - Flavor: "mysql", - Host: binlogReader.connectionConfig.Key.Hostname, - Port: uint16(binlogReader.connectionConfig.Key.Port), - User: binlogReader.connectionConfig.User, - Password: binlogReader.connectionConfig.Password, + ServerID: serverId, + Flavor: "mysql", + Host: binlogReader.connectionConfig.Key.Hostname, + Port: uint16(binlogReader.connectionConfig.Key.Port), + User: binlogReader.connectionConfig.User, + Password: binlogReader.connectionConfig.Password, + UseDecimal: true, } binlogReader.binlogSyncer = replication.NewBinlogSyncer(binlogSyncerConfig) diff --git a/vendor/github.com/siddontang/go-mysql/.travis.yml b/vendor/github.com/siddontang/go-mysql/.travis.yml index cc0db3c..8f8eafd 100644 --- a/vendor/github.com/siddontang/go-mysql/.travis.yml +++ b/vendor/github.com/siddontang/go-mysql/.travis.yml @@ -1,32 +1,34 @@ language: go go: - - 1.6 - - 1.7 + - "1.9" + - "1.10" -dist: trusty -sudo: required addons: apt: + sources: + - mysql-5.7-trusty packages: - - mysql-server-5.6 - - mysql-client-core-5.6 - - mysql-client-5.6 + - mysql-server + - mysql-client + +before_install: + - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;" + - sudo mysql_upgrade -before_script: # stop mysql and use row-based format binlog - - "sudo /etc/init.d/mysql stop || true" + - "sudo service mysql stop || true" - "echo '[mysqld]' | sudo tee /etc/mysql/conf.d/replication.cnf" - "echo 'server-id=1' | sudo tee -a /etc/mysql/conf.d/replication.cnf" - - "echo 'log-bin=mysql' | sudo tee -a /etc/mysql/conf.d/replication.cnf" + - "echo 'log-bin=mysql' | sudo tee -a /etc/mysql/conf.d/replication.cnf" - "echo 'binlog-format = row' | sudo tee -a /etc/mysql/conf.d/replication.cnf" # Start mysql (avoid errors to have logs) - - "sudo /etc/init.d/mysql start || true" + - "sudo service mysql start || true" - "sudo tail -1000 /var/log/syslog" - mysql -e "CREATE DATABASE IF NOT EXISTS test;" -uroot script: - - make test \ No newline at end of file + - make test diff --git a/vendor/github.com/siddontang/go-mysql/Makefile b/vendor/github.com/siddontang/go-mysql/Makefile index 92744b1..3decd6c 100644 --- a/vendor/github.com/siddontang/go-mysql/Makefile +++ b/vendor/github.com/siddontang/go-mysql/Makefile @@ -1,33 +1,14 @@ all: build build: - rm -rf vendor && ln -s _vendor/vendor vendor go build -o bin/go-mysqlbinlog cmd/go-mysqlbinlog/main.go go build -o bin/go-mysqldump cmd/go-mysqldump/main.go go build -o bin/go-canal cmd/go-canal/main.go go build -o bin/go-binlogparser cmd/go-binlogparser/main.go - rm -rf vendor - + test: - rm -rf vendor && ln -s _vendor/vendor vendor go test --race -timeout 2m ./... - rm -rf vendor clean: go clean -i ./... - @rm -rf ./bin - -update_vendor: - which glide >/dev/null || curl https://glide.sh/get | sh - which glide-vc || go get -v -u github.com/sgotti/glide-vc - rm -r vendor && mv _vendor/vendor vendor || true - rm -rf _vendor -ifdef PKG - glide get --strip-vendor --skip-test ${PKG} -else - glide update --strip-vendor --skip-test -endif - @echo "removing test files" - glide vc --only-code --no-tests - mkdir -p _vendor - mv vendor _vendor/vendor + @rm -rf ./bin \ No newline at end of file diff --git a/vendor/github.com/siddontang/go-mysql/README.md b/vendor/github.com/siddontang/go-mysql/README.md index b30f6fd..0b958c7 100644 --- a/vendor/github.com/siddontang/go-mysql/README.md +++ b/vendor/github.com/siddontang/go-mysql/README.md @@ -15,7 +15,7 @@ import ( "github.com/siddontang/go-mysql/replication" "os" ) -// Create a binlog syncer with a unique server id, the server id must be different from other MySQL's. +// Create a binlog syncer with a unique server id, the server id must be different from other MySQL's. // flavor is mysql or mariadb cfg := replication.BinlogSyncerConfig { ServerID: 100, @@ -27,7 +27,7 @@ cfg := replication.BinlogSyncerConfig { } syncer := replication.NewBinlogSyncer(cfg) -// Start sync with sepcified binlog file and position +// Start sync with specified binlog file and position streamer, _ := syncer.StartSync(mysql.Position{binlogFile, binlogPos}) // or you can start a gtid replication like @@ -44,7 +44,7 @@ for { // or we can use a timeout context for { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - e, _ := s.GetEvent(ctx) + ev, err := s.GetEvent(ctx) cancel() if err == context.DeadlineExceeded { @@ -85,13 +85,13 @@ Schema: test Query: DROP TABLE IF EXISTS `test_replication` /* generated by server */ ``` -## Canal +## Canal -Canal is a package that can sync your MySQL into everywhere, like Redis, Elasticsearch. +Canal is a package that can sync your MySQL into everywhere, like Redis, Elasticsearch. -First, canal will dump your MySQL data then sync changed data using binlog incrementally. +First, canal will dump your MySQL data then sync changed data using binlog incrementally. -You must use ROW format for binlog, full binlog row image is preferred, because we may meet some errors when primary key changed in update for minimal or noblob row image. +You must use ROW format for binlog, full binlog row image is preferred, because we may meet some errors when primary key changed in update for minimal or noblob row image. A simple example: @@ -105,30 +105,31 @@ cfg.Dump.Tables = []string{"canal_test"} c, err := NewCanal(cfg) -type myRowsEventHandler struct { +type MyEventHandler struct { + DummyEventHandler } -func (h *myRowsEventHandler) Do(e *RowsEvent) error { +func (h *MyEventHandler) OnRow(e *RowsEvent) error { log.Infof("%s %v\n", e.Action, e.Rows) return nil } -func (h *myRowsEventHandler) String() string { - return "myRowsEventHandler" +func (h *MyEventHandler) String() string { + return "MyEventHandler" } // Register a handler to handle RowsEvent -c.RegRowsEventHandler(&MyRowsEventHandler{}) +c.SetEventHandler(&MyEventHandler{}) // Start canal c.Start() ``` -You can see [go-mysql-elasticsearch](https://github.com/siddontang/go-mysql-elasticsearch) for how to sync MySQL data into Elasticsearch. +You can see [go-mysql-elasticsearch](https://github.com/siddontang/go-mysql-elasticsearch) for how to sync MySQL data into Elasticsearch. ## Client -Client package supports a simple MySQL connection driver which you can use it to communicate with MySQL server. +Client package supports a simple MySQL connection driver which you can use it to communicate with MySQL server. ### Example @@ -137,9 +138,16 @@ import ( "github.com/siddontang/go-mysql/client" ) -// Connect MySQL at 127.0.0.1:3306, with user root, an empty passowrd and database test +// Connect MySQL at 127.0.0.1:3306, with user root, an empty password and database test conn, _ := client.Connect("127.0.0.1:3306", "root", "", "test") +// Or to use SSL/TLS connection if MySQL server supports TLS +//conn, _ := client.Connect("127.0.0.1:3306", "root", "", "test", func(c *Conn) {c.UseSSL(true)}) + +// or to set your own client-side certificates for identity verification for security +//tlsConfig := NewClientTLSConfig(caPem, certPem, keyPem, false, "your-server-name") +//conn, _ := client.Connect("127.0.0.1:3306", "root", "", "test", func(c *Conn) {c.SetTLSConfig(tlsConfig)}) + conn.Ping() // Insert @@ -153,13 +161,20 @@ r, _ := conn.Execute(`select id, name from table where id = 1`) // Handle resultset v, _ := r.GetInt(0, 0) -v, _ = r.GetIntByName(0, "id") +v, _ = r.GetIntByName(0, "id") ``` +Tested MySQL versions for the client include: +- 5.5.x +- 5.6.x +- 5.7.x +- 8.0.x + ## Server -Server package supplies a framework to implement a simple MySQL server which can handle the packets from the MySQL client. -You can use it to build your own MySQL proxy. +Server package supplies a framework to implement a simple MySQL server which can handle the packets from the MySQL client. +You can use it to build your own MySQL proxy. The server connection is compatible with MySQL 5.5, 5.6, 5.7, and 8.0 versions, +so that most MySQL clients should be able to connect to the Server without modifications. ### Example @@ -173,42 +188,53 @@ l, _ := net.Listen("tcp", "127.0.0.1:4000") c, _ := l.Accept() -// Create a connection with user root and an empty passowrd -// We only an empty handler to handle command too +// Create a connection with user root and an empty password. +// You can use your own handler to handle command here. conn, _ := server.NewConn(c, "root", "", server.EmptyHandler{}) for { conn.HandleCommand() } -``` +``` Another shell ``` -mysql -h127.0.0.1 -P4000 -uroot -p -//Becuase empty handler does nothing, so here the MySQL client can only connect the proxy server. :-) +mysql -h127.0.0.1 -P4000 -uroot -p +//Becuase empty handler does nothing, so here the MySQL client can only connect the proxy server. :-) ``` +> ```NewConn()``` will use default server configurations: +> 1. automatically generate default server certificates and enable TLS/SSL support. +> 2. support three mainstream authentication methods **'mysql_native_password'**, **'caching_sha2_password'**, and **'sha256_password'** +> and use **'mysql_native_password'** as default. +> 3. use an in-memory user credential provider to store user and password. +> +> To customize server configurations, use ```NewServer()``` and create connection via ```NewCustomizedConn()```. + + ## Failover Failover supports to promote a new master and let other slaves replicate from it automatically when the old master was down. Failover supports MySQL >= 5.6.9 with GTID mode, if you use lower version, e.g, MySQL 5.0 - 5.5, please use [MHA](http://code.google.com/p/mysql-master-ha/) or [orchestrator](https://github.com/outbrain/orchestrator). -At the same time, Failover supports MariaDB >= 10.0.9 with GTID mode too. +At the same time, Failover supports MariaDB >= 10.0.9 with GTID mode too. -Why only GTID? Supporting failover with no GTID mode is very hard, because slave can not find the proper binlog filename and position with the new master. -Although there are many companies use MySQL 5.0 - 5.5, I think upgrade MySQL to 5.6 or higher is easy. +Why only GTID? Supporting failover with no GTID mode is very hard, because slave can not find the proper binlog filename and position with the new master. +Although there are many companies use MySQL 5.0 - 5.5, I think upgrade MySQL to 5.6 or higher is easy. ## Driver Driver is the package that you can use go-mysql with go database/sql like other drivers. A simple example: ``` +package main + import ( "database/sql" - - "github.com/siddontang/go-mysql/driver" + _ "github.com/siddontang/go-mysql/driver" ) func main() { @@ -221,9 +247,17 @@ func main() { We pass all tests in https://github.com/bradfitz/go-sql-test using go-mysql driver. :-) +## Donate + +If you like the project and want to buy me a cola, you can through: + +|PayPal|微信| +|------|---| +|[![](https://www.paypalobjects.com/webstatic/paypalme/images/pp_logo_small.png)](https://paypal.me/siddontang)|[![](https://github.com/siddontang/blog/blob/master/donate/weixin.png)| + ## Feedback -go-mysql is still in development, your feedback is very welcome. +go-mysql is still in development, your feedback is very welcome. Gmail: siddontang@gmail.com diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/COPYING deleted file mode 100644 index 5a8e332..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode.go deleted file mode 100644 index 6c7d398..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode.go +++ /dev/null @@ -1,492 +0,0 @@ -package toml - -import ( - "fmt" - "io" - "io/ioutil" - "math" - "reflect" - "strings" - "time" -) - -var e = fmt.Errorf - -// Unmarshaler is the interface implemented by objects that can unmarshal a -// TOML description of themselves. -type Unmarshaler interface { - UnmarshalTOML(interface{}) error -} - -// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`. -func Unmarshal(p []byte, v interface{}) error { - _, err := Decode(string(p), v) - return err -} - -// Primitive is a TOML value that hasn't been decoded into a Go value. -// When using the various `Decode*` functions, the type `Primitive` may -// be given to any value, and its decoding will be delayed. -// -// A `Primitive` value can be decoded using the `PrimitiveDecode` function. -// -// The underlying representation of a `Primitive` value is subject to change. -// Do not rely on it. -// -// N.B. Primitive values are still parsed, so using them will only avoid -// the overhead of reflection. They can be useful when you don't know the -// exact type of TOML data until run time. -type Primitive struct { - undecoded interface{} - context Key -} - -// DEPRECATED! -// -// Use MetaData.PrimitiveDecode instead. -func PrimitiveDecode(primValue Primitive, v interface{}) error { - md := MetaData{decoded: make(map[string]bool)} - return md.unify(primValue.undecoded, rvalue(v)) -} - -// PrimitiveDecode is just like the other `Decode*` functions, except it -// decodes a TOML value that has already been parsed. Valid primitive values -// can *only* be obtained from values filled by the decoder functions, -// including this method. (i.e., `v` may contain more `Primitive` -// values.) -// -// Meta data for primitive values is included in the meta data returned by -// the `Decode*` functions with one exception: keys returned by the Undecoded -// method will only reflect keys that were decoded. Namely, any keys hidden -// behind a Primitive will be considered undecoded. Executing this method will -// update the undecoded keys in the meta data. (See the example.) -func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { - md.context = primValue.context - defer func() { md.context = nil }() - return md.unify(primValue.undecoded, rvalue(v)) -} - -// Decode will decode the contents of `data` in TOML format into a pointer -// `v`. -// -// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be -// used interchangeably.) -// -// TOML arrays of tables correspond to either a slice of structs or a slice -// of maps. -// -// TOML datetimes correspond to Go `time.Time` values. -// -// All other TOML types (float, string, int, bool and array) correspond -// to the obvious Go types. -// -// An exception to the above rules is if a type implements the -// encoding.TextUnmarshaler interface. In this case, any primitive TOML value -// (floats, strings, integers, booleans and datetimes) will be converted to -// a byte string and given to the value's UnmarshalText method. See the -// Unmarshaler example for a demonstration with time duration strings. -// -// Key mapping -// -// TOML keys can map to either keys in a Go map or field names in a Go -// struct. The special `toml` struct tag may be used to map TOML keys to -// struct fields that don't match the key name exactly. (See the example.) -// A case insensitive match to struct names will be tried if an exact match -// can't be found. -// -// The mapping between TOML values and Go values is loose. That is, there -// may exist TOML values that cannot be placed into your representation, and -// there may be parts of your representation that do not correspond to -// TOML values. This loose mapping can be made stricter by using the IsDefined -// and/or Undecoded methods on the MetaData returned. -// -// This decoder will not handle cyclic types. If a cyclic type is passed, -// `Decode` will not terminate. -func Decode(data string, v interface{}) (MetaData, error) { - p, err := parse(data) - if err != nil { - return MetaData{}, err - } - md := MetaData{ - p.mapping, p.types, p.ordered, - make(map[string]bool, len(p.ordered)), nil, - } - return md, md.unify(p.mapping, rvalue(v)) -} - -// DecodeFile is just like Decode, except it will automatically read the -// contents of the file at `fpath` and decode it for you. -func DecodeFile(fpath string, v interface{}) (MetaData, error) { - bs, err := ioutil.ReadFile(fpath) - if err != nil { - return MetaData{}, err - } - return Decode(string(bs), v) -} - -// DecodeReader is just like Decode, except it will consume all bytes -// from the reader and decode it for you. -func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { - bs, err := ioutil.ReadAll(r) - if err != nil { - return MetaData{}, err - } - return Decode(string(bs), v) -} - -// unify performs a sort of type unification based on the structure of `rv`, -// which is the client representation. -// -// Any type mismatch produces an error. Finding a type that we don't know -// how to handle produces an unsupported type error. -func (md *MetaData) unify(data interface{}, rv reflect.Value) error { - - // Special case. Look for a `Primitive` value. - if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { - // Save the undecoded data and the key context into the primitive - // value. - context := make(Key, len(md.context)) - copy(context, md.context) - rv.Set(reflect.ValueOf(Primitive{ - undecoded: data, - context: context, - })) - return nil - } - - // Special case. Unmarshaler Interface support. - if rv.CanAddr() { - if v, ok := rv.Addr().Interface().(Unmarshaler); ok { - return v.UnmarshalTOML(data) - } - } - - // Special case. Handle time.Time values specifically. - // TODO: Remove this code when we decide to drop support for Go 1.1. - // This isn't necessary in Go 1.2 because time.Time satisfies the encoding - // interfaces. - if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { - return md.unifyDatetime(data, rv) - } - - // Special case. Look for a value satisfying the TextUnmarshaler interface. - if v, ok := rv.Interface().(TextUnmarshaler); ok { - return md.unifyText(data, v) - } - // BUG(burntsushi) - // The behavior here is incorrect whenever a Go type satisfies the - // encoding.TextUnmarshaler interface but also corresponds to a TOML - // hash or array. In particular, the unmarshaler should only be applied - // to primitive TOML values. But at this point, it will be applied to - // all kinds of values and produce an incorrect error whenever those values - // are hashes or arrays (including arrays of tables). - - k := rv.Kind() - - // laziness - if k >= reflect.Int && k <= reflect.Uint64 { - return md.unifyInt(data, rv) - } - switch k { - case reflect.Ptr: - elem := reflect.New(rv.Type().Elem()) - err := md.unify(data, reflect.Indirect(elem)) - if err != nil { - return err - } - rv.Set(elem) - return nil - case reflect.Struct: - return md.unifyStruct(data, rv) - case reflect.Map: - return md.unifyMap(data, rv) - case reflect.Array: - return md.unifyArray(data, rv) - case reflect.Slice: - return md.unifySlice(data, rv) - case reflect.String: - return md.unifyString(data, rv) - case reflect.Bool: - return md.unifyBool(data, rv) - case reflect.Interface: - // we only support empty interfaces. - if rv.NumMethod() > 0 { - return e("Unsupported type '%s'.", rv.Kind()) - } - return md.unifyAnything(data, rv) - case reflect.Float32: - fallthrough - case reflect.Float64: - return md.unifyFloat64(data, rv) - } - return e("Unsupported type '%s'.", rv.Kind()) -} - -func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { - tmap, ok := mapping.(map[string]interface{}) - if !ok { - return mismatch(rv, "map", mapping) - } - - for key, datum := range tmap { - var f *field - fields := cachedTypeFields(rv.Type()) - for i := range fields { - ff := &fields[i] - if ff.name == key { - f = ff - break - } - if f == nil && strings.EqualFold(ff.name, key) { - f = ff - } - } - if f != nil { - subv := rv - for _, i := range f.index { - subv = indirect(subv.Field(i)) - } - if isUnifiable(subv) { - md.decoded[md.context.add(key).String()] = true - md.context = append(md.context, key) - if err := md.unify(datum, subv); err != nil { - return e("Type mismatch for '%s.%s': %s", - rv.Type().String(), f.name, err) - } - md.context = md.context[0 : len(md.context)-1] - } else if f.name != "" { - // Bad user! No soup for you! - return e("Field '%s.%s' is unexported, and therefore cannot "+ - "be loaded with reflection.", rv.Type().String(), f.name) - } - } - } - return nil -} - -func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { - tmap, ok := mapping.(map[string]interface{}) - if !ok { - return badtype("map", mapping) - } - if rv.IsNil() { - rv.Set(reflect.MakeMap(rv.Type())) - } - for k, v := range tmap { - md.decoded[md.context.add(k).String()] = true - md.context = append(md.context, k) - - rvkey := indirect(reflect.New(rv.Type().Key())) - rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) - if err := md.unify(v, rvval); err != nil { - return err - } - md.context = md.context[0 : len(md.context)-1] - - rvkey.SetString(k) - rv.SetMapIndex(rvkey, rvval) - } - return nil -} - -func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { - datav := reflect.ValueOf(data) - if datav.Kind() != reflect.Slice { - return badtype("slice", data) - } - sliceLen := datav.Len() - if sliceLen != rv.Len() { - return e("expected array length %d; got TOML array of length %d", - rv.Len(), sliceLen) - } - return md.unifySliceArray(datav, rv) -} - -func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { - datav := reflect.ValueOf(data) - if datav.Kind() != reflect.Slice { - return badtype("slice", data) - } - sliceLen := datav.Len() - if rv.IsNil() { - rv.Set(reflect.MakeSlice(rv.Type(), sliceLen, sliceLen)) - } - return md.unifySliceArray(datav, rv) -} - -func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { - sliceLen := data.Len() - for i := 0; i < sliceLen; i++ { - v := data.Index(i).Interface() - sliceval := indirect(rv.Index(i)) - if err := md.unify(v, sliceval); err != nil { - return err - } - } - return nil -} - -func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { - if _, ok := data.(time.Time); ok { - rv.Set(reflect.ValueOf(data)) - return nil - } - return badtype("time.Time", data) -} - -func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { - if s, ok := data.(string); ok { - rv.SetString(s) - return nil - } - return badtype("string", data) -} - -func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { - if num, ok := data.(float64); ok { - switch rv.Kind() { - case reflect.Float32: - fallthrough - case reflect.Float64: - rv.SetFloat(num) - default: - panic("bug") - } - return nil - } - return badtype("float", data) -} - -func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { - if num, ok := data.(int64); ok { - if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 { - switch rv.Kind() { - case reflect.Int, reflect.Int64: - // No bounds checking necessary. - case reflect.Int8: - if num < math.MinInt8 || num > math.MaxInt8 { - return e("Value '%d' is out of range for int8.", num) - } - case reflect.Int16: - if num < math.MinInt16 || num > math.MaxInt16 { - return e("Value '%d' is out of range for int16.", num) - } - case reflect.Int32: - if num < math.MinInt32 || num > math.MaxInt32 { - return e("Value '%d' is out of range for int32.", num) - } - } - rv.SetInt(num) - } else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 { - unum := uint64(num) - switch rv.Kind() { - case reflect.Uint, reflect.Uint64: - // No bounds checking necessary. - case reflect.Uint8: - if num < 0 || unum > math.MaxUint8 { - return e("Value '%d' is out of range for uint8.", num) - } - case reflect.Uint16: - if num < 0 || unum > math.MaxUint16 { - return e("Value '%d' is out of range for uint16.", num) - } - case reflect.Uint32: - if num < 0 || unum > math.MaxUint32 { - return e("Value '%d' is out of range for uint32.", num) - } - } - rv.SetUint(unum) - } else { - panic("unreachable") - } - return nil - } - return badtype("integer", data) -} - -func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { - if b, ok := data.(bool); ok { - rv.SetBool(b) - return nil - } - return badtype("boolean", data) -} - -func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { - rv.Set(reflect.ValueOf(data)) - return nil -} - -func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error { - var s string - switch sdata := data.(type) { - case TextMarshaler: - text, err := sdata.MarshalText() - if err != nil { - return err - } - s = string(text) - case fmt.Stringer: - s = sdata.String() - case string: - s = sdata - case bool: - s = fmt.Sprintf("%v", sdata) - case int64: - s = fmt.Sprintf("%d", sdata) - case float64: - s = fmt.Sprintf("%f", sdata) - default: - return badtype("primitive (string-like)", data) - } - if err := v.UnmarshalText([]byte(s)); err != nil { - return err - } - return nil -} - -// rvalue returns a reflect.Value of `v`. All pointers are resolved. -func rvalue(v interface{}) reflect.Value { - return indirect(reflect.ValueOf(v)) -} - -// indirect returns the value pointed to by a pointer. -// Pointers are followed until the value is not a pointer. -// New values are allocated for each nil pointer. -// -// An exception to this rule is if the value satisfies an interface of -// interest to us (like encoding.TextUnmarshaler). -func indirect(v reflect.Value) reflect.Value { - if v.Kind() != reflect.Ptr { - if v.CanAddr() { - pv := v.Addr() - if _, ok := pv.Interface().(TextUnmarshaler); ok { - return pv - } - } - return v - } - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - return indirect(reflect.Indirect(v)) -} - -func isUnifiable(rv reflect.Value) bool { - if rv.CanSet() { - return true - } - if _, ok := rv.Interface().(TextUnmarshaler); ok { - return true - } - return false -} - -func badtype(expected string, data interface{}) error { - return e("Expected %s but found '%T'.", expected, data) -} - -func mismatch(user reflect.Value, expected string, data interface{}) error { - return e("Type mismatch for %s. Expected %s but found '%T'.", - user.Type().String(), expected, data) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode_meta.go deleted file mode 100644 index ef6f545..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/decode_meta.go +++ /dev/null @@ -1,122 +0,0 @@ -package toml - -import "strings" - -// MetaData allows access to meta information about TOML data that may not -// be inferrable via reflection. In particular, whether a key has been defined -// and the TOML type of a key. -type MetaData struct { - mapping map[string]interface{} - types map[string]tomlType - keys []Key - decoded map[string]bool - context Key // Used only during decoding. -} - -// IsDefined returns true if the key given exists in the TOML data. The key -// should be specified hierarchially. e.g., -// -// // access the TOML key 'a.b.c' -// IsDefined("a", "b", "c") -// -// IsDefined will return false if an empty key given. Keys are case sensitive. -func (md *MetaData) IsDefined(key ...string) bool { - if len(key) == 0 { - return false - } - - var hash map[string]interface{} - var ok bool - var hashOrVal interface{} = md.mapping - for _, k := range key { - if hash, ok = hashOrVal.(map[string]interface{}); !ok { - return false - } - if hashOrVal, ok = hash[k]; !ok { - return false - } - } - return true -} - -// Type returns a string representation of the type of the key specified. -// -// Type will return the empty string if given an empty key or a key that -// does not exist. Keys are case sensitive. -func (md *MetaData) Type(key ...string) string { - fullkey := strings.Join(key, ".") - if typ, ok := md.types[fullkey]; ok { - return typ.typeString() - } - return "" -} - -// Key is the type of any TOML key, including key groups. Use (MetaData).Keys -// to get values of this type. -type Key []string - -func (k Key) String() string { - return strings.Join(k, ".") -} - -func (k Key) maybeQuotedAll() string { - var ss []string - for i := range k { - ss = append(ss, k.maybeQuoted(i)) - } - return strings.Join(ss, ".") -} - -func (k Key) maybeQuoted(i int) string { - quote := false - for _, c := range k[i] { - if !isBareKeyChar(c) { - quote = true - break - } - } - if quote { - return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" - } else { - return k[i] - } -} - -func (k Key) add(piece string) Key { - newKey := make(Key, len(k)+1) - copy(newKey, k) - newKey[len(k)] = piece - return newKey -} - -// Keys returns a slice of every key in the TOML data, including key groups. -// Each key is itself a slice, where the first element is the top of the -// hierarchy and the last is the most specific. -// -// The list will have the same order as the keys appeared in the TOML data. -// -// All keys returned are non-empty. -func (md *MetaData) Keys() []Key { - return md.keys -} - -// Undecoded returns all keys that have not been decoded in the order in which -// they appear in the original TOML document. -// -// This includes keys that haven't been decoded because of a Primitive value. -// Once the Primitive value is decoded, the keys will be considered decoded. -// -// Also note that decoding into an empty interface will result in no decoding, -// and so no keys will be considered decoded. -// -// In this sense, the Undecoded keys correspond to keys in the TOML document -// that do not have a concrete type in your representation. -func (md *MetaData) Undecoded() []Key { - undecoded := make([]Key, 0, len(md.keys)) - for _, key := range md.keys { - if !md.decoded[key.String()] { - undecoded = append(undecoded, key) - } - } - return undecoded -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/doc.go deleted file mode 100644 index fe26800..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/doc.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Package toml provides facilities for decoding and encoding TOML configuration -files via reflection. There is also support for delaying decoding with -the Primitive type, and querying the set of keys in a TOML document with the -MetaData type. - -The specification implemented: https://github.com/mojombo/toml - -The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify -whether a file is a valid TOML document. It can also be used to print the -type of each key in a TOML document. - -Testing - -There are two important types of tests used for this package. The first is -contained inside '*_test.go' files and uses the standard Go unit testing -framework. These tests are primarily devoted to holistically testing the -decoder and encoder. - -The second type of testing is used to verify the implementation's adherence -to the TOML specification. These tests have been factored into their own -project: https://github.com/BurntSushi/toml-test - -The reason the tests are in a separate project is so that they can be used by -any implementation of TOML. Namely, it is language agnostic. -*/ -package toml diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encode.go deleted file mode 100644 index c7e227c..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encode.go +++ /dev/null @@ -1,551 +0,0 @@ -package toml - -import ( - "bufio" - "errors" - "fmt" - "io" - "reflect" - "sort" - "strconv" - "strings" - "time" -) - -type tomlEncodeError struct{ error } - -var ( - errArrayMixedElementTypes = errors.New( - "can't encode array with mixed element types") - errArrayNilElement = errors.New( - "can't encode array with nil element") - errNonString = errors.New( - "can't encode a map with non-string key type") - errAnonNonStruct = errors.New( - "can't encode an anonymous field that is not a struct") - errArrayNoTable = errors.New( - "TOML array element can't contain a table") - errNoKey = errors.New( - "top-level values must be a Go map or struct") - errAnything = errors.New("") // used in testing -) - -var quotedReplacer = strings.NewReplacer( - "\t", "\\t", - "\n", "\\n", - "\r", "\\r", - "\"", "\\\"", - "\\", "\\\\", -) - -// Encoder controls the encoding of Go values to a TOML document to some -// io.Writer. -// -// The indentation level can be controlled with the Indent field. -type Encoder struct { - // A single indentation level. By default it is two spaces. - Indent string - - // hasWritten is whether we have written any output to w yet. - hasWritten bool - w *bufio.Writer -} - -// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer -// given. By default, a single indentation level is 2 spaces. -func NewEncoder(w io.Writer) *Encoder { - return &Encoder{ - w: bufio.NewWriter(w), - Indent: " ", - } -} - -// Encode writes a TOML representation of the Go value to the underlying -// io.Writer. If the value given cannot be encoded to a valid TOML document, -// then an error is returned. -// -// The mapping between Go values and TOML values should be precisely the same -// as for the Decode* functions. Similarly, the TextMarshaler interface is -// supported by encoding the resulting bytes as strings. (If you want to write -// arbitrary binary data then you will need to use something like base64 since -// TOML does not have any binary types.) -// -// When encoding TOML hashes (i.e., Go maps or structs), keys without any -// sub-hashes are encoded first. -// -// If a Go map is encoded, then its keys are sorted alphabetically for -// deterministic output. More control over this behavior may be provided if -// there is demand for it. -// -// Encoding Go values without a corresponding TOML representation---like map -// types with non-string keys---will cause an error to be returned. Similarly -// for mixed arrays/slices, arrays/slices with nil elements, embedded -// non-struct types and nested slices containing maps or structs. -// (e.g., [][]map[string]string is not allowed but []map[string]string is OK -// and so is []map[string][]string.) -func (enc *Encoder) Encode(v interface{}) error { - rv := eindirect(reflect.ValueOf(v)) - if err := enc.safeEncode(Key([]string{}), rv); err != nil { - return err - } - return enc.w.Flush() -} - -func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { - defer func() { - if r := recover(); r != nil { - if terr, ok := r.(tomlEncodeError); ok { - err = terr.error - return - } - panic(r) - } - }() - enc.encode(key, rv) - return nil -} - -func (enc *Encoder) encode(key Key, rv reflect.Value) { - // Special case. Time needs to be in ISO8601 format. - // Special case. If we can marshal the type to text, then we used that. - // Basically, this prevents the encoder for handling these types as - // generic structs (or whatever the underlying type of a TextMarshaler is). - switch rv.Interface().(type) { - case time.Time, TextMarshaler: - enc.keyEqElement(key, rv) - return - } - - k := rv.Kind() - switch k { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, - reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, - reflect.Uint64, - reflect.Float32, reflect.Float64, reflect.String, reflect.Bool: - enc.keyEqElement(key, rv) - case reflect.Array, reflect.Slice: - if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) { - enc.eArrayOfTables(key, rv) - } else { - enc.keyEqElement(key, rv) - } - case reflect.Interface: - if rv.IsNil() { - return - } - enc.encode(key, rv.Elem()) - case reflect.Map: - if rv.IsNil() { - return - } - enc.eTable(key, rv) - case reflect.Ptr: - if rv.IsNil() { - return - } - enc.encode(key, rv.Elem()) - case reflect.Struct: - enc.eTable(key, rv) - default: - panic(e("Unsupported type for key '%s': %s", key, k)) - } -} - -// eElement encodes any value that can be an array element (primitives and -// arrays). -func (enc *Encoder) eElement(rv reflect.Value) { - switch v := rv.Interface().(type) { - case time.Time: - // Special case time.Time as a primitive. Has to come before - // TextMarshaler below because time.Time implements - // encoding.TextMarshaler, but we need to always use UTC. - enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z")) - return - case TextMarshaler: - // Special case. Use text marshaler if it's available for this value. - if s, err := v.MarshalText(); err != nil { - encPanic(err) - } else { - enc.writeQuoted(string(s)) - } - return - } - switch rv.Kind() { - case reflect.Bool: - enc.wf(strconv.FormatBool(rv.Bool())) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, - reflect.Int64: - enc.wf(strconv.FormatInt(rv.Int(), 10)) - case reflect.Uint, reflect.Uint8, reflect.Uint16, - reflect.Uint32, reflect.Uint64: - enc.wf(strconv.FormatUint(rv.Uint(), 10)) - case reflect.Float32: - enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) - case reflect.Float64: - enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) - case reflect.Array, reflect.Slice: - enc.eArrayOrSliceElement(rv) - case reflect.Interface: - enc.eElement(rv.Elem()) - case reflect.String: - enc.writeQuoted(rv.String()) - default: - panic(e("Unexpected primitive type: %s", rv.Kind())) - } -} - -// By the TOML spec, all floats must have a decimal with at least one -// number on either side. -func floatAddDecimal(fstr string) string { - if !strings.Contains(fstr, ".") { - return fstr + ".0" - } - return fstr -} - -func (enc *Encoder) writeQuoted(s string) { - enc.wf("\"%s\"", quotedReplacer.Replace(s)) -} - -func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { - length := rv.Len() - enc.wf("[") - for i := 0; i < length; i++ { - elem := rv.Index(i) - enc.eElement(elem) - if i != length-1 { - enc.wf(", ") - } - } - enc.wf("]") -} - -func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { - if len(key) == 0 { - encPanic(errNoKey) - } - for i := 0; i < rv.Len(); i++ { - trv := rv.Index(i) - if isNil(trv) { - continue - } - panicIfInvalidKey(key) - enc.newline() - enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll()) - enc.newline() - enc.eMapOrStruct(key, trv) - } -} - -func (enc *Encoder) eTable(key Key, rv reflect.Value) { - panicIfInvalidKey(key) - if len(key) == 1 { - // Output an extra new line between top-level tables. - // (The newline isn't written if nothing else has been written though.) - enc.newline() - } - if len(key) > 0 { - enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll()) - enc.newline() - } - enc.eMapOrStruct(key, rv) -} - -func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) { - switch rv := eindirect(rv); rv.Kind() { - case reflect.Map: - enc.eMap(key, rv) - case reflect.Struct: - enc.eStruct(key, rv) - default: - panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) - } -} - -func (enc *Encoder) eMap(key Key, rv reflect.Value) { - rt := rv.Type() - if rt.Key().Kind() != reflect.String { - encPanic(errNonString) - } - - // Sort keys so that we have deterministic output. And write keys directly - // underneath this key first, before writing sub-structs or sub-maps. - var mapKeysDirect, mapKeysSub []string - for _, mapKey := range rv.MapKeys() { - k := mapKey.String() - if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { - mapKeysSub = append(mapKeysSub, k) - } else { - mapKeysDirect = append(mapKeysDirect, k) - } - } - - var writeMapKeys = func(mapKeys []string) { - sort.Strings(mapKeys) - for _, mapKey := range mapKeys { - mrv := rv.MapIndex(reflect.ValueOf(mapKey)) - if isNil(mrv) { - // Don't write anything for nil fields. - continue - } - enc.encode(key.add(mapKey), mrv) - } - } - writeMapKeys(mapKeysDirect) - writeMapKeys(mapKeysSub) -} - -func (enc *Encoder) eStruct(key Key, rv reflect.Value) { - // Write keys for fields directly under this key first, because if we write - // a field that creates a new table, then all keys under it will be in that - // table (not the one we're writing here). - rt := rv.Type() - var fieldsDirect, fieldsSub [][]int - var addFields func(rt reflect.Type, rv reflect.Value, start []int) - addFields = func(rt reflect.Type, rv reflect.Value, start []int) { - for i := 0; i < rt.NumField(); i++ { - f := rt.Field(i) - // skip unexporded fields - if f.PkgPath != "" { - continue - } - frv := rv.Field(i) - if f.Anonymous { - frv := eindirect(frv) - t := frv.Type() - if t.Kind() != reflect.Struct { - encPanic(errAnonNonStruct) - } - addFields(t, frv, f.Index) - } else if typeIsHash(tomlTypeOfGo(frv)) { - fieldsSub = append(fieldsSub, append(start, f.Index...)) - } else { - fieldsDirect = append(fieldsDirect, append(start, f.Index...)) - } - } - } - addFields(rt, rv, nil) - - var writeFields = func(fields [][]int) { - for _, fieldIndex := range fields { - sft := rt.FieldByIndex(fieldIndex) - sf := rv.FieldByIndex(fieldIndex) - if isNil(sf) { - // Don't write anything for nil fields. - continue - } - - keyName := sft.Tag.Get("toml") - if keyName == "-" { - continue - } - if keyName == "" { - keyName = sft.Name - } - - keyName, opts := getOptions(keyName) - if _, ok := opts["omitempty"]; ok && isEmpty(sf) { - continue - } else if _, ok := opts["omitzero"]; ok && isZero(sf) { - continue - } - - enc.encode(key.add(keyName), sf) - } - } - writeFields(fieldsDirect) - writeFields(fieldsSub) -} - -// tomlTypeName returns the TOML type name of the Go value's type. It is -// used to determine whether the types of array elements are mixed (which is -// forbidden). If the Go value is nil, then it is illegal for it to be an array -// element, and valueIsNil is returned as true. - -// Returns the TOML type of a Go value. The type may be `nil`, which means -// no concrete TOML type could be found. -func tomlTypeOfGo(rv reflect.Value) tomlType { - if isNil(rv) || !rv.IsValid() { - return nil - } - switch rv.Kind() { - case reflect.Bool: - return tomlBool - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, - reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, - reflect.Uint64: - return tomlInteger - case reflect.Float32, reflect.Float64: - return tomlFloat - case reflect.Array, reflect.Slice: - if typeEqual(tomlHash, tomlArrayType(rv)) { - return tomlArrayHash - } else { - return tomlArray - } - case reflect.Ptr, reflect.Interface: - return tomlTypeOfGo(rv.Elem()) - case reflect.String: - return tomlString - case reflect.Map: - return tomlHash - case reflect.Struct: - switch rv.Interface().(type) { - case time.Time: - return tomlDatetime - case TextMarshaler: - return tomlString - default: - return tomlHash - } - default: - panic("unexpected reflect.Kind: " + rv.Kind().String()) - } -} - -// tomlArrayType returns the element type of a TOML array. The type returned -// may be nil if it cannot be determined (e.g., a nil slice or a zero length -// slize). This function may also panic if it finds a type that cannot be -// expressed in TOML (such as nil elements, heterogeneous arrays or directly -// nested arrays of tables). -func tomlArrayType(rv reflect.Value) tomlType { - if isNil(rv) || !rv.IsValid() || rv.Len() == 0 { - return nil - } - firstType := tomlTypeOfGo(rv.Index(0)) - if firstType == nil { - encPanic(errArrayNilElement) - } - - rvlen := rv.Len() - for i := 1; i < rvlen; i++ { - elem := rv.Index(i) - switch elemType := tomlTypeOfGo(elem); { - case elemType == nil: - encPanic(errArrayNilElement) - case !typeEqual(firstType, elemType): - encPanic(errArrayMixedElementTypes) - } - } - // If we have a nested array, then we must make sure that the nested - // array contains ONLY primitives. - // This checks arbitrarily nested arrays. - if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) { - nest := tomlArrayType(eindirect(rv.Index(0))) - if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) { - encPanic(errArrayNoTable) - } - } - return firstType -} - -func getOptions(keyName string) (string, map[string]struct{}) { - opts := make(map[string]struct{}) - ss := strings.Split(keyName, ",") - name := ss[0] - if len(ss) > 1 { - for _, opt := range ss { - opts[opt] = struct{}{} - } - } - - return name, opts -} - -func isZero(rv reflect.Value) bool { - switch rv.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if rv.Int() == 0 { - return true - } - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - if rv.Uint() == 0 { - return true - } - case reflect.Float32, reflect.Float64: - if rv.Float() == 0.0 { - return true - } - } - - return false -} - -func isEmpty(rv reflect.Value) bool { - switch rv.Kind() { - case reflect.String: - if len(strings.TrimSpace(rv.String())) == 0 { - return true - } - case reflect.Array, reflect.Slice, reflect.Map: - if rv.Len() == 0 { - return true - } - } - - return false -} - -func (enc *Encoder) newline() { - if enc.hasWritten { - enc.wf("\n") - } -} - -func (enc *Encoder) keyEqElement(key Key, val reflect.Value) { - if len(key) == 0 { - encPanic(errNoKey) - } - panicIfInvalidKey(key) - enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1)) - enc.eElement(val) - enc.newline() -} - -func (enc *Encoder) wf(format string, v ...interface{}) { - if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { - encPanic(err) - } - enc.hasWritten = true -} - -func (enc *Encoder) indentStr(key Key) string { - return strings.Repeat(enc.Indent, len(key)-1) -} - -func encPanic(err error) { - panic(tomlEncodeError{err}) -} - -func eindirect(v reflect.Value) reflect.Value { - switch v.Kind() { - case reflect.Ptr, reflect.Interface: - return eindirect(v.Elem()) - default: - return v - } -} - -func isNil(rv reflect.Value) bool { - switch rv.Kind() { - case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: - return rv.IsNil() - default: - return false - } -} - -func panicIfInvalidKey(key Key) { - for _, k := range key { - if len(k) == 0 { - encPanic(e("Key '%s' is not a valid table name. Key names "+ - "cannot be empty.", key.maybeQuotedAll())) - } - } -} - -func isValidKeyName(s string) bool { - return len(s) != 0 -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types.go deleted file mode 100644 index d36e1dd..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build go1.2 - -package toml - -// In order to support Go 1.1, we define our own TextMarshaler and -// TextUnmarshaler types. For Go 1.2+, we just alias them with the -// standard library interfaces. - -import ( - "encoding" -) - -// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here -// so that Go 1.1 can be supported. -type TextMarshaler encoding.TextMarshaler - -// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined -// here so that Go 1.1 can be supported. -type TextUnmarshaler encoding.TextUnmarshaler diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go deleted file mode 100644 index e8d503d..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build !go1.2 - -package toml - -// These interfaces were introduced in Go 1.2, so we add them manually when -// compiling for Go 1.1. - -// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here -// so that Go 1.1 can be supported. -type TextMarshaler interface { - MarshalText() (text []byte, err error) -} - -// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined -// here so that Go 1.1 can be supported. -type TextUnmarshaler interface { - UnmarshalText(text []byte) error -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/lex.go deleted file mode 100644 index 2191228..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/lex.go +++ /dev/null @@ -1,874 +0,0 @@ -package toml - -import ( - "fmt" - "strings" - "unicode/utf8" -) - -type itemType int - -const ( - itemError itemType = iota - itemNIL // used in the parser to indicate no type - itemEOF - itemText - itemString - itemRawString - itemMultilineString - itemRawMultilineString - itemBool - itemInteger - itemFloat - itemDatetime - itemArray // the start of an array - itemArrayEnd - itemTableStart - itemTableEnd - itemArrayTableStart - itemArrayTableEnd - itemKeyStart - itemCommentStart -) - -const ( - eof = 0 - tableStart = '[' - tableEnd = ']' - arrayTableStart = '[' - arrayTableEnd = ']' - tableSep = '.' - keySep = '=' - arrayStart = '[' - arrayEnd = ']' - arrayValTerm = ',' - commentStart = '#' - stringStart = '"' - stringEnd = '"' - rawStringStart = '\'' - rawStringEnd = '\'' -) - -type stateFn func(lx *lexer) stateFn - -type lexer struct { - input string - start int - pos int - width int - line int - state stateFn - items chan item - - // A stack of state functions used to maintain context. - // The idea is to reuse parts of the state machine in various places. - // For example, values can appear at the top level or within arbitrarily - // nested arrays. The last state on the stack is used after a value has - // been lexed. Similarly for comments. - stack []stateFn -} - -type item struct { - typ itemType - val string - line int -} - -func (lx *lexer) nextItem() item { - for { - select { - case item := <-lx.items: - return item - default: - lx.state = lx.state(lx) - } - } -} - -func lex(input string) *lexer { - lx := &lexer{ - input: input + "\n", - state: lexTop, - line: 1, - items: make(chan item, 10), - stack: make([]stateFn, 0, 10), - } - return lx -} - -func (lx *lexer) push(state stateFn) { - lx.stack = append(lx.stack, state) -} - -func (lx *lexer) pop() stateFn { - if len(lx.stack) == 0 { - return lx.errorf("BUG in lexer: no states to pop.") - } - last := lx.stack[len(lx.stack)-1] - lx.stack = lx.stack[0 : len(lx.stack)-1] - return last -} - -func (lx *lexer) current() string { - return lx.input[lx.start:lx.pos] -} - -func (lx *lexer) emit(typ itemType) { - lx.items <- item{typ, lx.current(), lx.line} - lx.start = lx.pos -} - -func (lx *lexer) emitTrim(typ itemType) { - lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} - lx.start = lx.pos -} - -func (lx *lexer) next() (r rune) { - if lx.pos >= len(lx.input) { - lx.width = 0 - return eof - } - - if lx.input[lx.pos] == '\n' { - lx.line++ - } - r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) - lx.pos += lx.width - return r -} - -// ignore skips over the pending input before this point. -func (lx *lexer) ignore() { - lx.start = lx.pos -} - -// backup steps back one rune. Can be called only once per call of next. -func (lx *lexer) backup() { - lx.pos -= lx.width - if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { - lx.line-- - } -} - -// accept consumes the next rune if it's equal to `valid`. -func (lx *lexer) accept(valid rune) bool { - if lx.next() == valid { - return true - } - lx.backup() - return false -} - -// peek returns but does not consume the next rune in the input. -func (lx *lexer) peek() rune { - r := lx.next() - lx.backup() - return r -} - -// errorf stops all lexing by emitting an error and returning `nil`. -// Note that any value that is a character is escaped if it's a special -// character (new lines, tabs, etc.). -func (lx *lexer) errorf(format string, values ...interface{}) stateFn { - lx.items <- item{ - itemError, - fmt.Sprintf(format, values...), - lx.line, - } - return nil -} - -// lexTop consumes elements at the top level of TOML data. -func lexTop(lx *lexer) stateFn { - r := lx.next() - if isWhitespace(r) || isNL(r) { - return lexSkip(lx, lexTop) - } - - switch r { - case commentStart: - lx.push(lexTop) - return lexCommentStart - case tableStart: - return lexTableStart - case eof: - if lx.pos > lx.start { - return lx.errorf("Unexpected EOF.") - } - lx.emit(itemEOF) - return nil - } - - // At this point, the only valid item can be a key, so we back up - // and let the key lexer do the rest. - lx.backup() - lx.push(lexTopEnd) - return lexKeyStart -} - -// lexTopEnd is entered whenever a top-level item has been consumed. (A value -// or a table.) It must see only whitespace, and will turn back to lexTop -// upon a new line. If it sees EOF, it will quit the lexer successfully. -func lexTopEnd(lx *lexer) stateFn { - r := lx.next() - switch { - case r == commentStart: - // a comment will read to a new line for us. - lx.push(lexTop) - return lexCommentStart - case isWhitespace(r): - return lexTopEnd - case isNL(r): - lx.ignore() - return lexTop - case r == eof: - lx.ignore() - return lexTop - } - return lx.errorf("Expected a top-level item to end with a new line, "+ - "comment or EOF, but got %q instead.", r) -} - -// lexTable lexes the beginning of a table. Namely, it makes sure that -// it starts with a character other than '.' and ']'. -// It assumes that '[' has already been consumed. -// It also handles the case that this is an item in an array of tables. -// e.g., '[[name]]'. -func lexTableStart(lx *lexer) stateFn { - if lx.peek() == arrayTableStart { - lx.next() - lx.emit(itemArrayTableStart) - lx.push(lexArrayTableEnd) - } else { - lx.emit(itemTableStart) - lx.push(lexTableEnd) - } - return lexTableNameStart -} - -func lexTableEnd(lx *lexer) stateFn { - lx.emit(itemTableEnd) - return lexTopEnd -} - -func lexArrayTableEnd(lx *lexer) stateFn { - if r := lx.next(); r != arrayTableEnd { - return lx.errorf("Expected end of table array name delimiter %q, "+ - "but got %q instead.", arrayTableEnd, r) - } - lx.emit(itemArrayTableEnd) - return lexTopEnd -} - -func lexTableNameStart(lx *lexer) stateFn { - switch r := lx.peek(); { - case r == tableEnd || r == eof: - return lx.errorf("Unexpected end of table name. (Table names cannot " + - "be empty.)") - case r == tableSep: - return lx.errorf("Unexpected table separator. (Table names cannot " + - "be empty.)") - case r == stringStart || r == rawStringStart: - lx.ignore() - lx.push(lexTableNameEnd) - return lexValue // reuse string lexing - case isWhitespace(r): - return lexTableNameStart - default: - return lexBareTableName - } -} - -// lexTableName lexes the name of a table. It assumes that at least one -// valid character for the table has already been read. -func lexBareTableName(lx *lexer) stateFn { - switch r := lx.next(); { - case isBareKeyChar(r): - return lexBareTableName - case r == tableSep || r == tableEnd: - lx.backup() - lx.emitTrim(itemText) - return lexTableNameEnd - default: - return lx.errorf("Bare keys cannot contain %q.", r) - } -} - -// lexTableNameEnd reads the end of a piece of a table name, optionally -// consuming whitespace. -func lexTableNameEnd(lx *lexer) stateFn { - switch r := lx.next(); { - case isWhitespace(r): - return lexTableNameEnd - case r == tableSep: - lx.ignore() - return lexTableNameStart - case r == tableEnd: - return lx.pop() - default: - return lx.errorf("Expected '.' or ']' to end table name, but got %q "+ - "instead.", r) - } -} - -// lexKeyStart consumes a key name up until the first non-whitespace character. -// lexKeyStart will ignore whitespace. -func lexKeyStart(lx *lexer) stateFn { - r := lx.peek() - switch { - case r == keySep: - return lx.errorf("Unexpected key separator %q.", keySep) - case isWhitespace(r) || isNL(r): - lx.next() - return lexSkip(lx, lexKeyStart) - case r == stringStart || r == rawStringStart: - lx.ignore() - lx.emit(itemKeyStart) - lx.push(lexKeyEnd) - return lexValue // reuse string lexing - default: - lx.ignore() - lx.emit(itemKeyStart) - return lexBareKey - } -} - -// lexBareKey consumes the text of a bare key. Assumes that the first character -// (which is not whitespace) has not yet been consumed. -func lexBareKey(lx *lexer) stateFn { - switch r := lx.next(); { - case isBareKeyChar(r): - return lexBareKey - case isWhitespace(r): - lx.emitTrim(itemText) - return lexKeyEnd - case r == keySep: - lx.backup() - lx.emitTrim(itemText) - return lexKeyEnd - default: - return lx.errorf("Bare keys cannot contain %q.", r) - } -} - -// lexKeyEnd consumes the end of a key and trims whitespace (up to the key -// separator). -func lexKeyEnd(lx *lexer) stateFn { - switch r := lx.next(); { - case r == keySep: - return lexSkip(lx, lexValue) - case isWhitespace(r): - return lexSkip(lx, lexKeyEnd) - default: - return lx.errorf("Expected key separator %q, but got %q instead.", - keySep, r) - } -} - -// lexValue starts the consumption of a value anywhere a value is expected. -// lexValue will ignore whitespace. -// After a value is lexed, the last state on the next is popped and returned. -func lexValue(lx *lexer) stateFn { - // We allow whitespace to precede a value, but NOT new lines. - // In array syntax, the array states are responsible for ignoring new - // lines. - r := lx.next() - if isWhitespace(r) { - return lexSkip(lx, lexValue) - } - - switch { - case r == arrayStart: - lx.ignore() - lx.emit(itemArray) - return lexArrayValue - case r == stringStart: - if lx.accept(stringStart) { - if lx.accept(stringStart) { - lx.ignore() // Ignore """ - return lexMultilineString - } - lx.backup() - } - lx.ignore() // ignore the '"' - return lexString - case r == rawStringStart: - if lx.accept(rawStringStart) { - if lx.accept(rawStringStart) { - lx.ignore() // Ignore """ - return lexMultilineRawString - } - lx.backup() - } - lx.ignore() // ignore the "'" - return lexRawString - case r == 't': - return lexTrue - case r == 'f': - return lexFalse - case r == '-': - return lexNumberStart - case isDigit(r): - lx.backup() // avoid an extra state and use the same as above - return lexNumberOrDateStart - case r == '.': // special error case, be kind to users - return lx.errorf("Floats must start with a digit, not '.'.") - } - return lx.errorf("Expected value but found %q instead.", r) -} - -// lexArrayValue consumes one value in an array. It assumes that '[' or ',' -// have already been consumed. All whitespace and new lines are ignored. -func lexArrayValue(lx *lexer) stateFn { - r := lx.next() - switch { - case isWhitespace(r) || isNL(r): - return lexSkip(lx, lexArrayValue) - case r == commentStart: - lx.push(lexArrayValue) - return lexCommentStart - case r == arrayValTerm: - return lx.errorf("Unexpected array value terminator %q.", - arrayValTerm) - case r == arrayEnd: - return lexArrayEnd - } - - lx.backup() - lx.push(lexArrayValueEnd) - return lexValue -} - -// lexArrayValueEnd consumes the cruft between values of an array. Namely, -// it ignores whitespace and expects either a ',' or a ']'. -func lexArrayValueEnd(lx *lexer) stateFn { - r := lx.next() - switch { - case isWhitespace(r) || isNL(r): - return lexSkip(lx, lexArrayValueEnd) - case r == commentStart: - lx.push(lexArrayValueEnd) - return lexCommentStart - case r == arrayValTerm: - lx.ignore() - return lexArrayValue // move on to the next value - case r == arrayEnd: - return lexArrayEnd - } - return lx.errorf("Expected an array value terminator %q or an array "+ - "terminator %q, but got %q instead.", arrayValTerm, arrayEnd, r) -} - -// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has -// just been consumed. -func lexArrayEnd(lx *lexer) stateFn { - lx.ignore() - lx.emit(itemArrayEnd) - return lx.pop() -} - -// lexString consumes the inner contents of a string. It assumes that the -// beginning '"' has already been consumed and ignored. -func lexString(lx *lexer) stateFn { - r := lx.next() - switch { - case isNL(r): - return lx.errorf("Strings cannot contain new lines.") - case r == '\\': - lx.push(lexString) - return lexStringEscape - case r == stringEnd: - lx.backup() - lx.emit(itemString) - lx.next() - lx.ignore() - return lx.pop() - } - return lexString -} - -// lexMultilineString consumes the inner contents of a string. It assumes that -// the beginning '"""' has already been consumed and ignored. -func lexMultilineString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '\\': - return lexMultilineStringEscape - case r == stringEnd: - if lx.accept(stringEnd) { - if lx.accept(stringEnd) { - lx.backup() - lx.backup() - lx.backup() - lx.emit(itemMultilineString) - lx.next() - lx.next() - lx.next() - lx.ignore() - return lx.pop() - } - lx.backup() - } - } - return lexMultilineString -} - -// lexRawString consumes a raw string. Nothing can be escaped in such a string. -// It assumes that the beginning "'" has already been consumed and ignored. -func lexRawString(lx *lexer) stateFn { - r := lx.next() - switch { - case isNL(r): - return lx.errorf("Strings cannot contain new lines.") - case r == rawStringEnd: - lx.backup() - lx.emit(itemRawString) - lx.next() - lx.ignore() - return lx.pop() - } - return lexRawString -} - -// lexMultilineRawString consumes a raw string. Nothing can be escaped in such -// a string. It assumes that the beginning "'" has already been consumed and -// ignored. -func lexMultilineRawString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == rawStringEnd: - if lx.accept(rawStringEnd) { - if lx.accept(rawStringEnd) { - lx.backup() - lx.backup() - lx.backup() - lx.emit(itemRawMultilineString) - lx.next() - lx.next() - lx.next() - lx.ignore() - return lx.pop() - } - lx.backup() - } - } - return lexMultilineRawString -} - -// lexMultilineStringEscape consumes an escaped character. It assumes that the -// preceding '\\' has already been consumed. -func lexMultilineStringEscape(lx *lexer) stateFn { - // Handle the special case first: - if isNL(lx.next()) { - lx.next() - return lexMultilineString - } else { - lx.backup() - lx.push(lexMultilineString) - return lexStringEscape(lx) - } -} - -func lexStringEscape(lx *lexer) stateFn { - r := lx.next() - switch r { - case 'b': - fallthrough - case 't': - fallthrough - case 'n': - fallthrough - case 'f': - fallthrough - case 'r': - fallthrough - case '"': - fallthrough - case '\\': - return lx.pop() - case 'u': - return lexShortUnicodeEscape - case 'U': - return lexLongUnicodeEscape - } - return lx.errorf("Invalid escape character %q. Only the following "+ - "escape characters are allowed: "+ - "\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, "+ - "\\uXXXX and \\UXXXXXXXX.", r) -} - -func lexShortUnicodeEscape(lx *lexer) stateFn { - var r rune - for i := 0; i < 4; i++ { - r = lx.next() - if !isHexadecimal(r) { - return lx.errorf("Expected four hexadecimal digits after '\\u', "+ - "but got '%s' instead.", lx.current()) - } - } - return lx.pop() -} - -func lexLongUnicodeEscape(lx *lexer) stateFn { - var r rune - for i := 0; i < 8; i++ { - r = lx.next() - if !isHexadecimal(r) { - return lx.errorf("Expected eight hexadecimal digits after '\\U', "+ - "but got '%s' instead.", lx.current()) - } - } - return lx.pop() -} - -// lexNumberOrDateStart consumes either a (positive) integer, float or -// datetime. It assumes that NO negative sign has been consumed. -func lexNumberOrDateStart(lx *lexer) stateFn { - r := lx.next() - if !isDigit(r) { - if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) - } - } - return lexNumberOrDate -} - -// lexNumberOrDate consumes either a (positive) integer, float or datetime. -func lexNumberOrDate(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '-': - if lx.pos-lx.start != 5 { - return lx.errorf("All ISO8601 dates must be in full Zulu form.") - } - return lexDateAfterYear - case isDigit(r): - return lexNumberOrDate - case r == '.': - return lexFloatStart - } - - lx.backup() - lx.emit(itemInteger) - return lx.pop() -} - -// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. -// It assumes that "YYYY-" has already been consumed. -func lexDateAfterYear(lx *lexer) stateFn { - formats := []rune{ - // digits are '0'. - // everything else is direct equality. - '0', '0', '-', '0', '0', - 'T', - '0', '0', ':', '0', '0', ':', '0', '0', - 'Z', - } - for _, f := range formats { - r := lx.next() - if f == '0' { - if !isDigit(r) { - return lx.errorf("Expected digit in ISO8601 datetime, "+ - "but found %q instead.", r) - } - } else if f != r { - return lx.errorf("Expected %q in ISO8601 datetime, "+ - "but found %q instead.", f, r) - } - } - lx.emit(itemDatetime) - return lx.pop() -} - -// lexNumberStart consumes either an integer or a float. It assumes that -// a negative sign has already been read, but that *no* digits have been -// consumed. lexNumberStart will move to the appropriate integer or float -// states. -func lexNumberStart(lx *lexer) stateFn { - // we MUST see a digit. Even floats have to start with a digit. - r := lx.next() - if !isDigit(r) { - if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) - } - } - return lexNumber -} - -// lexNumber consumes an integer or a float after seeing the first digit. -func lexNumber(lx *lexer) stateFn { - r := lx.next() - switch { - case isDigit(r): - return lexNumber - case r == '.': - return lexFloatStart - } - - lx.backup() - lx.emit(itemInteger) - return lx.pop() -} - -// lexFloatStart starts the consumption of digits of a float after a '.'. -// Namely, at least one digit is required. -func lexFloatStart(lx *lexer) stateFn { - r := lx.next() - if !isDigit(r) { - return lx.errorf("Floats must have a digit after the '.', but got "+ - "%q instead.", r) - } - return lexFloat -} - -// lexFloat consumes the digits of a float after a '.'. -// Assumes that one digit has been consumed after a '.' already. -func lexFloat(lx *lexer) stateFn { - r := lx.next() - if isDigit(r) { - return lexFloat - } - - lx.backup() - lx.emit(itemFloat) - return lx.pop() -} - -// lexConst consumes the s[1:] in s. It assumes that s[0] has already been -// consumed. -func lexConst(lx *lexer, s string) stateFn { - for i := range s[1:] { - if r := lx.next(); r != rune(s[i+1]) { - return lx.errorf("Expected %q, but found %q instead.", s[:i+1], - s[:i]+string(r)) - } - } - return nil -} - -// lexTrue consumes the "rue" in "true". It assumes that 't' has already -// been consumed. -func lexTrue(lx *lexer) stateFn { - if fn := lexConst(lx, "true"); fn != nil { - return fn - } - lx.emit(itemBool) - return lx.pop() -} - -// lexFalse consumes the "alse" in "false". It assumes that 'f' has already -// been consumed. -func lexFalse(lx *lexer) stateFn { - if fn := lexConst(lx, "false"); fn != nil { - return fn - } - lx.emit(itemBool) - return lx.pop() -} - -// lexCommentStart begins the lexing of a comment. It will emit -// itemCommentStart and consume no characters, passing control to lexComment. -func lexCommentStart(lx *lexer) stateFn { - lx.ignore() - lx.emit(itemCommentStart) - return lexComment -} - -// lexComment lexes an entire comment. It assumes that '#' has been consumed. -// It will consume *up to* the first new line character, and pass control -// back to the last state on the stack. -func lexComment(lx *lexer) stateFn { - r := lx.peek() - if isNL(r) || r == eof { - lx.emit(itemText) - return lx.pop() - } - lx.next() - return lexComment -} - -// lexSkip ignores all slurped input and moves on to the next state. -func lexSkip(lx *lexer, nextState stateFn) stateFn { - return func(lx *lexer) stateFn { - lx.ignore() - return nextState - } -} - -// isWhitespace returns true if `r` is a whitespace character according -// to the spec. -func isWhitespace(r rune) bool { - return r == '\t' || r == ' ' -} - -func isNL(r rune) bool { - return r == '\n' || r == '\r' -} - -func isDigit(r rune) bool { - return r >= '0' && r <= '9' -} - -func isHexadecimal(r rune) bool { - return (r >= '0' && r <= '9') || - (r >= 'a' && r <= 'f') || - (r >= 'A' && r <= 'F') -} - -func isBareKeyChar(r rune) bool { - return (r >= 'A' && r <= 'Z') || - (r >= 'a' && r <= 'z') || - (r >= '0' && r <= '9') || - r == '_' || - r == '-' -} - -func (itype itemType) String() string { - switch itype { - case itemError: - return "Error" - case itemNIL: - return "NIL" - case itemEOF: - return "EOF" - case itemText: - return "Text" - case itemString: - return "String" - case itemRawString: - return "String" - case itemMultilineString: - return "String" - case itemRawMultilineString: - return "String" - case itemBool: - return "Bool" - case itemInteger: - return "Integer" - case itemFloat: - return "Float" - case itemDatetime: - return "DateTime" - case itemTableStart: - return "TableStart" - case itemTableEnd: - return "TableEnd" - case itemKeyStart: - return "KeyStart" - case itemArray: - return "Array" - case itemArrayEnd: - return "ArrayEnd" - case itemCommentStart: - return "CommentStart" - } - panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype))) -} - -func (item item) String() string { - return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/parse.go deleted file mode 100644 index c6069be..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/parse.go +++ /dev/null @@ -1,498 +0,0 @@ -package toml - -import ( - "fmt" - "log" - "strconv" - "strings" - "time" - "unicode" - "unicode/utf8" -) - -type parser struct { - mapping map[string]interface{} - types map[string]tomlType - lx *lexer - - // A list of keys in the order that they appear in the TOML data. - ordered []Key - - // the full key for the current hash in scope - context Key - - // the base key name for everything except hashes - currentKey string - - // rough approximation of line number - approxLine int - - // A map of 'key.group.names' to whether they were created implicitly. - implicits map[string]bool -} - -type parseError string - -func (pe parseError) Error() string { - return string(pe) -} - -func parse(data string) (p *parser, err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - if err, ok = r.(parseError); ok { - return - } - panic(r) - } - }() - - p = &parser{ - mapping: make(map[string]interface{}), - types: make(map[string]tomlType), - lx: lex(data), - ordered: make([]Key, 0), - implicits: make(map[string]bool), - } - for { - item := p.next() - if item.typ == itemEOF { - break - } - p.topLevel(item) - } - - return p, nil -} - -func (p *parser) panicf(format string, v ...interface{}) { - msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s", - p.approxLine, p.current(), fmt.Sprintf(format, v...)) - panic(parseError(msg)) -} - -func (p *parser) next() item { - it := p.lx.nextItem() - if it.typ == itemError { - p.panicf("%s", it.val) - } - return it -} - -func (p *parser) bug(format string, v ...interface{}) { - log.Fatalf("BUG: %s\n\n", fmt.Sprintf(format, v...)) -} - -func (p *parser) expect(typ itemType) item { - it := p.next() - p.assertEqual(typ, it.typ) - return it -} - -func (p *parser) assertEqual(expected, got itemType) { - if expected != got { - p.bug("Expected '%s' but got '%s'.", expected, got) - } -} - -func (p *parser) topLevel(item item) { - switch item.typ { - case itemCommentStart: - p.approxLine = item.line - p.expect(itemText) - case itemTableStart: - kg := p.next() - p.approxLine = kg.line - - var key Key - for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() { - key = append(key, p.keyString(kg)) - } - p.assertEqual(itemTableEnd, kg.typ) - - p.establishContext(key, false) - p.setType("", tomlHash) - p.ordered = append(p.ordered, key) - case itemArrayTableStart: - kg := p.next() - p.approxLine = kg.line - - var key Key - for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() { - key = append(key, p.keyString(kg)) - } - p.assertEqual(itemArrayTableEnd, kg.typ) - - p.establishContext(key, true) - p.setType("", tomlArrayHash) - p.ordered = append(p.ordered, key) - case itemKeyStart: - kname := p.next() - p.approxLine = kname.line - p.currentKey = p.keyString(kname) - - val, typ := p.value(p.next()) - p.setValue(p.currentKey, val) - p.setType(p.currentKey, typ) - p.ordered = append(p.ordered, p.context.add(p.currentKey)) - p.currentKey = "" - default: - p.bug("Unexpected type at top level: %s", item.typ) - } -} - -// Gets a string for a key (or part of a key in a table name). -func (p *parser) keyString(it item) string { - switch it.typ { - case itemText: - return it.val - case itemString, itemMultilineString, - itemRawString, itemRawMultilineString: - s, _ := p.value(it) - return s.(string) - default: - p.bug("Unexpected key type: %s", it.typ) - panic("unreachable") - } -} - -// value translates an expected value from the lexer into a Go value wrapped -// as an empty interface. -func (p *parser) value(it item) (interface{}, tomlType) { - switch it.typ { - case itemString: - return p.replaceEscapes(it.val), p.typeOfPrimitive(it) - case itemMultilineString: - trimmed := stripFirstNewline(stripEscapedWhitespace(it.val)) - return p.replaceEscapes(trimmed), p.typeOfPrimitive(it) - case itemRawString: - return it.val, p.typeOfPrimitive(it) - case itemRawMultilineString: - return stripFirstNewline(it.val), p.typeOfPrimitive(it) - case itemBool: - switch it.val { - case "true": - return true, p.typeOfPrimitive(it) - case "false": - return false, p.typeOfPrimitive(it) - } - p.bug("Expected boolean value, but got '%s'.", it.val) - case itemInteger: - num, err := strconv.ParseInt(it.val, 10, 64) - if err != nil { - // See comment below for floats describing why we make a - // distinction between a bug and a user error. - if e, ok := err.(*strconv.NumError); ok && - e.Err == strconv.ErrRange { - - p.panicf("Integer '%s' is out of the range of 64-bit "+ - "signed integers.", it.val) - } else { - p.bug("Expected integer value, but got '%s'.", it.val) - } - } - return num, p.typeOfPrimitive(it) - case itemFloat: - num, err := strconv.ParseFloat(it.val, 64) - if err != nil { - // Distinguish float values. Normally, it'd be a bug if the lexer - // provides an invalid float, but it's possible that the float is - // out of range of valid values (which the lexer cannot determine). - // So mark the former as a bug but the latter as a legitimate user - // error. - // - // This is also true for integers. - if e, ok := err.(*strconv.NumError); ok && - e.Err == strconv.ErrRange { - - p.panicf("Float '%s' is out of the range of 64-bit "+ - "IEEE-754 floating-point numbers.", it.val) - } else { - p.bug("Expected float value, but got '%s'.", it.val) - } - } - return num, p.typeOfPrimitive(it) - case itemDatetime: - t, err := time.Parse("2006-01-02T15:04:05Z", it.val) - if err != nil { - p.bug("Expected Zulu formatted DateTime, but got '%s'.", it.val) - } - return t, p.typeOfPrimitive(it) - case itemArray: - array := make([]interface{}, 0) - types := make([]tomlType, 0) - - for it = p.next(); it.typ != itemArrayEnd; it = p.next() { - if it.typ == itemCommentStart { - p.expect(itemText) - continue - } - - val, typ := p.value(it) - array = append(array, val) - types = append(types, typ) - } - return array, p.typeOfArray(types) - } - p.bug("Unexpected value type: %s", it.typ) - panic("unreachable") -} - -// establishContext sets the current context of the parser, -// where the context is either a hash or an array of hashes. Which one is -// set depends on the value of the `array` parameter. -// -// Establishing the context also makes sure that the key isn't a duplicate, and -// will create implicit hashes automatically. -func (p *parser) establishContext(key Key, array bool) { - var ok bool - - // Always start at the top level and drill down for our context. - hashContext := p.mapping - keyContext := make(Key, 0) - - // We only need implicit hashes for key[0:-1] - for _, k := range key[0 : len(key)-1] { - _, ok = hashContext[k] - keyContext = append(keyContext, k) - - // No key? Make an implicit hash and move on. - if !ok { - p.addImplicit(keyContext) - hashContext[k] = make(map[string]interface{}) - } - - // If the hash context is actually an array of tables, then set - // the hash context to the last element in that array. - // - // Otherwise, it better be a table, since this MUST be a key group (by - // virtue of it not being the last element in a key). - switch t := hashContext[k].(type) { - case []map[string]interface{}: - hashContext = t[len(t)-1] - case map[string]interface{}: - hashContext = t - default: - p.panicf("Key '%s' was already created as a hash.", keyContext) - } - } - - p.context = keyContext - if array { - // If this is the first element for this array, then allocate a new - // list of tables for it. - k := key[len(key)-1] - if _, ok := hashContext[k]; !ok { - hashContext[k] = make([]map[string]interface{}, 0, 5) - } - - // Add a new table. But make sure the key hasn't already been used - // for something else. - if hash, ok := hashContext[k].([]map[string]interface{}); ok { - hashContext[k] = append(hash, make(map[string]interface{})) - } else { - p.panicf("Key '%s' was already created and cannot be used as "+ - "an array.", keyContext) - } - } else { - p.setValue(key[len(key)-1], make(map[string]interface{})) - } - p.context = append(p.context, key[len(key)-1]) -} - -// setValue sets the given key to the given value in the current context. -// It will make sure that the key hasn't already been defined, account for -// implicit key groups. -func (p *parser) setValue(key string, value interface{}) { - var tmpHash interface{} - var ok bool - - hash := p.mapping - keyContext := make(Key, 0) - for _, k := range p.context { - keyContext = append(keyContext, k) - if tmpHash, ok = hash[k]; !ok { - p.bug("Context for key '%s' has not been established.", keyContext) - } - switch t := tmpHash.(type) { - case []map[string]interface{}: - // The context is a table of hashes. Pick the most recent table - // defined as the current hash. - hash = t[len(t)-1] - case map[string]interface{}: - hash = t - default: - p.bug("Expected hash to have type 'map[string]interface{}', but "+ - "it has '%T' instead.", tmpHash) - } - } - keyContext = append(keyContext, key) - - if _, ok := hash[key]; ok { - // Typically, if the given key has already been set, then we have - // to raise an error since duplicate keys are disallowed. However, - // it's possible that a key was previously defined implicitly. In this - // case, it is allowed to be redefined concretely. (See the - // `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.) - // - // But we have to make sure to stop marking it as an implicit. (So that - // another redefinition provokes an error.) - // - // Note that since it has already been defined (as a hash), we don't - // want to overwrite it. So our business is done. - if p.isImplicit(keyContext) { - p.removeImplicit(keyContext) - return - } - - // Otherwise, we have a concrete key trying to override a previous - // key, which is *always* wrong. - p.panicf("Key '%s' has already been defined.", keyContext) - } - hash[key] = value -} - -// setType sets the type of a particular value at a given key. -// It should be called immediately AFTER setValue. -// -// Note that if `key` is empty, then the type given will be applied to the -// current context (which is either a table or an array of tables). -func (p *parser) setType(key string, typ tomlType) { - keyContext := make(Key, 0, len(p.context)+1) - for _, k := range p.context { - keyContext = append(keyContext, k) - } - if len(key) > 0 { // allow type setting for hashes - keyContext = append(keyContext, key) - } - p.types[keyContext.String()] = typ -} - -// addImplicit sets the given Key as having been created implicitly. -func (p *parser) addImplicit(key Key) { - p.implicits[key.String()] = true -} - -// removeImplicit stops tagging the given key as having been implicitly -// created. -func (p *parser) removeImplicit(key Key) { - p.implicits[key.String()] = false -} - -// isImplicit returns true if the key group pointed to by the key was created -// implicitly. -func (p *parser) isImplicit(key Key) bool { - return p.implicits[key.String()] -} - -// current returns the full key name of the current context. -func (p *parser) current() string { - if len(p.currentKey) == 0 { - return p.context.String() - } - if len(p.context) == 0 { - return p.currentKey - } - return fmt.Sprintf("%s.%s", p.context, p.currentKey) -} - -func stripFirstNewline(s string) string { - if len(s) == 0 || s[0] != '\n' { - return s - } - return s[1:len(s)] -} - -func stripEscapedWhitespace(s string) string { - esc := strings.Split(s, "\\\n") - if len(esc) > 1 { - for i := 1; i < len(esc); i++ { - esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace) - } - } - return strings.Join(esc, "") -} - -func (p *parser) replaceEscapes(str string) string { - var replaced []rune - s := []byte(str) - r := 0 - for r < len(s) { - if s[r] != '\\' { - c, size := utf8.DecodeRune(s[r:]) - r += size - replaced = append(replaced, c) - continue - } - r += 1 - if r >= len(s) { - p.bug("Escape sequence at end of string.") - return "" - } - switch s[r] { - default: - p.bug("Expected valid escape code after \\, but got %q.", s[r]) - return "" - case 'b': - replaced = append(replaced, rune(0x0008)) - r += 1 - case 't': - replaced = append(replaced, rune(0x0009)) - r += 1 - case 'n': - replaced = append(replaced, rune(0x000A)) - r += 1 - case 'f': - replaced = append(replaced, rune(0x000C)) - r += 1 - case 'r': - replaced = append(replaced, rune(0x000D)) - r += 1 - case '"': - replaced = append(replaced, rune(0x0022)) - r += 1 - case '\\': - replaced = append(replaced, rune(0x005C)) - r += 1 - case 'u': - // At this point, we know we have a Unicode escape of the form - // `uXXXX` at [r, r+5). (Because the lexer guarantees this - // for us.) - escaped := p.asciiEscapeToUnicode(s[r+1 : r+5]) - replaced = append(replaced, escaped) - r += 5 - case 'U': - // At this point, we know we have a Unicode escape of the form - // `uXXXX` at [r, r+9). (Because the lexer guarantees this - // for us.) - escaped := p.asciiEscapeToUnicode(s[r+1 : r+9]) - replaced = append(replaced, escaped) - r += 9 - } - } - return string(replaced) -} - -func (p *parser) asciiEscapeToUnicode(bs []byte) rune { - s := string(bs) - hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) - if err != nil { - p.bug("Could not parse '%s' as a hexadecimal number, but the "+ - "lexer claims it's OK: %s", s, err) - } - - // BUG(burntsushi) - // I honestly don't understand how this works. I can't seem - // to find a way to make this fail. I figured this would fail on invalid - // UTF-8 characters like U+DCFF, but it doesn't. - if !utf8.ValidString(string(rune(hex))) { - p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s) - } - return rune(hex) -} - -func isStringType(ty itemType) bool { - return ty == itemString || ty == itemMultilineString || - ty == itemRawString || ty == itemRawMultilineString -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_check.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_check.go deleted file mode 100644 index c73f8af..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_check.go +++ /dev/null @@ -1,91 +0,0 @@ -package toml - -// tomlType represents any Go type that corresponds to a TOML type. -// While the first draft of the TOML spec has a simplistic type system that -// probably doesn't need this level of sophistication, we seem to be militating -// toward adding real composite types. -type tomlType interface { - typeString() string -} - -// typeEqual accepts any two types and returns true if they are equal. -func typeEqual(t1, t2 tomlType) bool { - if t1 == nil || t2 == nil { - return false - } - return t1.typeString() == t2.typeString() -} - -func typeIsHash(t tomlType) bool { - return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) -} - -type tomlBaseType string - -func (btype tomlBaseType) typeString() string { - return string(btype) -} - -func (btype tomlBaseType) String() string { - return btype.typeString() -} - -var ( - tomlInteger tomlBaseType = "Integer" - tomlFloat tomlBaseType = "Float" - tomlDatetime tomlBaseType = "Datetime" - tomlString tomlBaseType = "String" - tomlBool tomlBaseType = "Bool" - tomlArray tomlBaseType = "Array" - tomlHash tomlBaseType = "Hash" - tomlArrayHash tomlBaseType = "ArrayHash" -) - -// typeOfPrimitive returns a tomlType of any primitive value in TOML. -// Primitive values are: Integer, Float, Datetime, String and Bool. -// -// Passing a lexer item other than the following will cause a BUG message -// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime. -func (p *parser) typeOfPrimitive(lexItem item) tomlType { - switch lexItem.typ { - case itemInteger: - return tomlInteger - case itemFloat: - return tomlFloat - case itemDatetime: - return tomlDatetime - case itemString: - return tomlString - case itemMultilineString: - return tomlString - case itemRawString: - return tomlString - case itemRawMultilineString: - return tomlString - case itemBool: - return tomlBool - } - p.bug("Cannot infer primitive type of lex item '%s'.", lexItem) - panic("unreachable") -} - -// typeOfArray returns a tomlType for an array given a list of types of its -// values. -// -// In the current spec, if an array is homogeneous, then its type is always -// "Array". If the array is not homogeneous, an error is generated. -func (p *parser) typeOfArray(types []tomlType) tomlType { - // Empty arrays are cool. - if len(types) == 0 { - return tomlArray - } - - theType := types[0] - for _, t := range types[1:] { - if !typeEqual(theType, t) { - p.panicf("Array contains values of type '%s' and '%s', but "+ - "arrays must be homogeneous.", theType, t) - } - } - return tomlArray -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_fields.go deleted file mode 100644 index 7592f87..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/BurntSushi/toml/type_fields.go +++ /dev/null @@ -1,241 +0,0 @@ -package toml - -// Struct field handling is adapted from code in encoding/json: -// -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the Go distribution. - -import ( - "reflect" - "sort" - "sync" -) - -// A field represents a single field found in a struct. -type field struct { - name string // the name of the field (`toml` tag included) - tag bool // whether field has a `toml` tag - index []int // represents the depth of an anonymous field - typ reflect.Type // the type of the field -} - -// byName sorts field by name, breaking ties with depth, -// then breaking ties with "name came from toml tag", then -// breaking ties with index sequence. -type byName []field - -func (x byName) Len() int { return len(x) } - -func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byName) Less(i, j int) bool { - if x[i].name != x[j].name { - return x[i].name < x[j].name - } - if len(x[i].index) != len(x[j].index) { - return len(x[i].index) < len(x[j].index) - } - if x[i].tag != x[j].tag { - return x[i].tag - } - return byIndex(x).Less(i, j) -} - -// byIndex sorts field by index sequence. -type byIndex []field - -func (x byIndex) Len() int { return len(x) } - -func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byIndex) Less(i, j int) bool { - for k, xik := range x[i].index { - if k >= len(x[j].index) { - return false - } - if xik != x[j].index[k] { - return xik < x[j].index[k] - } - } - return len(x[i].index) < len(x[j].index) -} - -// typeFields returns a list of fields that TOML should recognize for the given -// type. The algorithm is breadth-first search over the set of structs to -// include - the top struct and then any reachable anonymous structs. -func typeFields(t reflect.Type) []field { - // Anonymous fields to explore at the current level and the next. - current := []field{} - next := []field{{typ: t}} - - // Count of queued names for current level and the next. - count := map[reflect.Type]int{} - nextCount := map[reflect.Type]int{} - - // Types already visited at an earlier level. - visited := map[reflect.Type]bool{} - - // Fields found. - var fields []field - - for len(next) > 0 { - current, next = next, current[:0] - count, nextCount = nextCount, map[reflect.Type]int{} - - for _, f := range current { - if visited[f.typ] { - continue - } - visited[f.typ] = true - - // Scan f.typ for fields to include. - for i := 0; i < f.typ.NumField(); i++ { - sf := f.typ.Field(i) - if sf.PkgPath != "" { // unexported - continue - } - name := sf.Tag.Get("toml") - if name == "-" { - continue - } - index := make([]int, len(f.index)+1) - copy(index, f.index) - index[len(f.index)] = i - - ft := sf.Type - if ft.Name() == "" && ft.Kind() == reflect.Ptr { - // Follow pointer. - ft = ft.Elem() - } - - // Record found field and index sequence. - if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { - tagged := name != "" - if name == "" { - name = sf.Name - } - fields = append(fields, field{name, tagged, index, ft}) - if count[f.typ] > 1 { - // If there were multiple instances, add a second, - // so that the annihilation code will see a duplicate. - // It only cares about the distinction between 1 or 2, - // so don't bother generating any more copies. - fields = append(fields, fields[len(fields)-1]) - } - continue - } - - // Record new anonymous struct to explore in next round. - nextCount[ft]++ - if nextCount[ft] == 1 { - f := field{name: ft.Name(), index: index, typ: ft} - next = append(next, f) - } - } - } - } - - sort.Sort(byName(fields)) - - // Delete all fields that are hidden by the Go rules for embedded fields, - // except that fields with TOML tags are promoted. - - // The fields are sorted in primary order of name, secondary order - // of field index length. Loop over names; for each name, delete - // hidden fields by choosing the one dominant field that survives. - out := fields[:0] - for advance, i := 0, 0; i < len(fields); i += advance { - // One iteration per name. - // Find the sequence of fields with the name of this first field. - fi := fields[i] - name := fi.name - for advance = 1; i+advance < len(fields); advance++ { - fj := fields[i+advance] - if fj.name != name { - break - } - } - if advance == 1 { // Only one field with this name - out = append(out, fi) - continue - } - dominant, ok := dominantField(fields[i : i+advance]) - if ok { - out = append(out, dominant) - } - } - - fields = out - sort.Sort(byIndex(fields)) - - return fields -} - -// dominantField looks through the fields, all of which are known to -// have the same name, to find the single field that dominates the -// others using Go's embedding rules, modified by the presence of -// TOML tags. If there are multiple top-level fields, the boolean -// will be false: This condition is an error in Go and we skip all -// the fields. -func dominantField(fields []field) (field, bool) { - // The fields are sorted in increasing index-length order. The winner - // must therefore be one with the shortest index length. Drop all - // longer entries, which is easy: just truncate the slice. - length := len(fields[0].index) - tagged := -1 // Index of first tagged field. - for i, f := range fields { - if len(f.index) > length { - fields = fields[:i] - break - } - if f.tag { - if tagged >= 0 { - // Multiple tagged fields at the same level: conflict. - // Return no field. - return field{}, false - } - tagged = i - } - } - if tagged >= 0 { - return fields[tagged], true - } - // All remaining fields have the same length. If there's more than one, - // we have a conflict (two fields named "X" at the same level) and we - // return no field. - if len(fields) > 1 { - return field{}, false - } - return fields[0], true -} - -var fieldCache struct { - sync.RWMutex - m map[reflect.Type][]field -} - -// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. -func cachedTypeFields(t reflect.Type) []field { - fieldCache.RLock() - f := fieldCache.m[t] - fieldCache.RUnlock() - if f != nil { - return f - } - - // Compute fields without lock. - // Might duplicate effort but won't hold other computations back. - f = typeFields(t) - if f == nil { - f = []field{} - } - - fieldCache.Lock() - if fieldCache.m == nil { - fieldCache.m = map[reflect.Type][]field{} - } - fieldCache.m[t] = f - fieldCache.Unlock() - return f -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/LICENSE b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/LICENSE deleted file mode 100644 index 14e2f77..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/LICENSE +++ /dev/null @@ -1,373 +0,0 @@ -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" - means each individual or legal entity that creates, contributes to - the creation of, or owns Covered Software. - -1.2. "Contributor Version" - means the combination of the Contributions of others (if any) used - by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" - means Covered Software of a particular Contributor. - -1.4. "Covered Software" - means Source Code Form to which the initial Contributor has attached - the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case - including portions thereof. - -1.5. "Incompatible With Secondary Licenses" - means - - (a) that the initial Contributor has attached the notice described - in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" - means any form of the work other than Source Code Form. - -1.7. "Larger Work" - means a work that combines Covered Software with other material, in - a separate file or files, that is not Covered Software. - -1.8. "License" - means this document. - -1.9. "Licensable" - means having the right to grant, to the maximum extent possible, - whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" - means any of the following: - - (a) any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered - Software; or - - (b) any new file in Source Code Form that contains any Covered - Software. - -1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the - License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its - Contributor Version. - -1.12. "Secondary License" - means either the GNU General Public License, Version 2.0, the GNU - Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those - licenses. - -1.13. "Source Code Form" - means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") - means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For - purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, - whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial - ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, -non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer - for sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution -become effective for each Contribution on the date the Contributor first -distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under -this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. -Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; - or - -(b) for infringements caused by: (i) Your and any other third party's - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of - its Contributions. - -This License does not grant any rights in the trademarks, service marks, -or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to -distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if -permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its -Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under -applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted -in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source -Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not -attempt to alter or restrict the recipients' rights in the Source Code -Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code - Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code - Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter - the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, -provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered -Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this -License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of -the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary -License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices -(including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of -the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, -indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on -behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by -You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, -indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this -License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with -the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must -be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute -or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically -if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular -Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an -ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have -come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License -from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version -directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section -2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all -end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License -prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the -courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that -jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring -cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject -matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent -necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter -shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section -10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a -distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version -of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license -steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to -create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove -any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary -Licenses - -If You choose to distribute Source Code Form that is Incompatible With -Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public - License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular -file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look -for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/appengine.go deleted file mode 100644 index 565614e..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/appengine.go +++ /dev/null @@ -1,19 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build appengine - -package mysql - -import ( - "appengine/cloudsql" -) - -func init() { - RegisterDial("cloudsql", cloudsql.Dial) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/buffer.go deleted file mode 100644 index 2001fea..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/buffer.go +++ /dev/null @@ -1,147 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "io" - "net" - "time" -) - -const defaultBufSize = 4096 - -// A buffer which is used for both reading and writing. -// This is possible since communication on each connection is synchronous. -// In other words, we can't write and read simultaneously on the same connection. -// The buffer is similar to bufio.Reader / Writer but zero-copy-ish -// Also highly optimized for this particular use case. -type buffer struct { - buf []byte - nc net.Conn - idx int - length int - timeout time.Duration -} - -func newBuffer(nc net.Conn) buffer { - var b [defaultBufSize]byte - return buffer{ - buf: b[:], - nc: nc, - } -} - -// fill reads into the buffer until at least _need_ bytes are in it -func (b *buffer) fill(need int) error { - n := b.length - - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) - } - - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { - // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf - } - - b.idx = 0 - - for { - if b.timeout > 0 { - if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil { - return err - } - } - - nn, err := b.nc.Read(b.buf[n:]) - n += nn - - switch err { - case nil: - if n < need { - continue - } - b.length = n - return nil - - case io.EOF: - if n >= need { - b.length = n - return nil - } - return io.ErrUnexpectedEOF - - default: - return err - } - } -} - -// returns next N bytes from buffer. -// The returned slice is only guaranteed to be valid until the next read -func (b *buffer) readNext(need int) ([]byte, error) { - if b.length < need { - // refill - if err := b.fill(need); err != nil { - return nil, err - } - } - - offset := b.idx - b.idx += need - b.length -= need - return b.buf[offset:b.idx], nil -} - -// returns a buffer with the requested size. -// If possible, a slice from the existing buffer is returned. -// Otherwise a bigger buffer is made. -// Only one buffer (total) can be used at a time. -func (b *buffer) takeBuffer(length int) []byte { - if b.length > 0 { - return nil - } - - // test (cheap) general case first - if length <= defaultBufSize || length <= cap(b.buf) { - return b.buf[:length] - } - - if length < maxPacketSize { - b.buf = make([]byte, length) - return b.buf - } - return make([]byte, length) -} - -// shortcut which can be used if the requested buffer is guaranteed to be -// smaller than defaultBufSize -// Only one buffer (total) can be used at a time. -func (b *buffer) takeSmallBuffer(length int) []byte { - if b.length == 0 { - return b.buf[:length] - } - return nil -} - -// takeCompleteBuffer returns the complete existing buffer. -// This can be used if the necessary buffer size is unknown. -// Only one buffer (total) can be used at a time. -func (b *buffer) takeCompleteBuffer() []byte { - if b.length == 0 { - return b.buf - } - return nil -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/collations.go deleted file mode 100644 index 82079cf..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/collations.go +++ /dev/null @@ -1,250 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2014 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -const defaultCollation = "utf8_general_ci" - -// A list of available collations mapped to the internal ID. -// To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS -var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, - "utf8_unicode_ci": 192, - "utf8_icelandic_ci": 193, - "utf8_latvian_ci": 194, - "utf8_romanian_ci": 195, - "utf8_slovenian_ci": 196, - "utf8_polish_ci": 197, - "utf8_estonian_ci": 198, - "utf8_spanish_ci": 199, - "utf8_swedish_ci": 200, - "utf8_turkish_ci": 201, - "utf8_czech_ci": 202, - "utf8_danish_ci": 203, - "utf8_lithuanian_ci": 204, - "utf8_slovak_ci": 205, - "utf8_spanish2_ci": 206, - "utf8_roman_ci": 207, - "utf8_persian_ci": 208, - "utf8_esperanto_ci": 209, - "utf8_hungarian_ci": 210, - "utf8_sinhala_ci": 211, - "utf8_german2_ci": 212, - "utf8_croatian_ci": 213, - "utf8_unicode_520_ci": 214, - "utf8_vietnamese_ci": 215, - "utf8_general_mysql500_ci": 223, - "utf8mb4_unicode_ci": 224, - "utf8mb4_icelandic_ci": 225, - "utf8mb4_latvian_ci": 226, - "utf8mb4_romanian_ci": 227, - "utf8mb4_slovenian_ci": 228, - "utf8mb4_polish_ci": 229, - "utf8mb4_estonian_ci": 230, - "utf8mb4_spanish_ci": 231, - "utf8mb4_swedish_ci": 232, - "utf8mb4_turkish_ci": 233, - "utf8mb4_czech_ci": 234, - "utf8mb4_danish_ci": 235, - "utf8mb4_lithuanian_ci": 236, - "utf8mb4_slovak_ci": 237, - "utf8mb4_spanish2_ci": 238, - "utf8mb4_roman_ci": 239, - "utf8mb4_persian_ci": 240, - "utf8mb4_esperanto_ci": 241, - "utf8mb4_hungarian_ci": 242, - "utf8mb4_sinhala_ci": 243, - "utf8mb4_german2_ci": 244, - "utf8mb4_croatian_ci": 245, - "utf8mb4_unicode_520_ci": 246, - "utf8mb4_vietnamese_ci": 247, -} - -// A blacklist of collations which is unsafe to interpolate parameters. -// These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. -var unsafeCollations = map[string]bool{ - "big5_chinese_ci": true, - "sjis_japanese_ci": true, - "gbk_chinese_ci": true, - "big5_bin": true, - "gb2312_bin": true, - "gbk_bin": true, - "sjis_bin": true, - "cp932_japanese_ci": true, - "cp932_bin": true, -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/connection.go deleted file mode 100644 index c3899de..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/connection.go +++ /dev/null @@ -1,372 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "database/sql/driver" - "net" - "strconv" - "strings" - "time" -) - -type mysqlConn struct { - buf buffer - netConn net.Conn - affectedRows uint64 - insertId uint64 - cfg *Config - maxPacketAllowed int - maxWriteSize int - writeTimeout time.Duration - flags clientFlag - status statusFlag - sequence uint8 - parseTime bool - strict bool -} - -// Handles parameters set in DSN after the connection is established -func (mc *mysqlConn) handleParams() (err error) { - for param, val := range mc.cfg.Params { - switch param { - // Charset - case "charset": - charsets := strings.Split(val, ",") - for i := range charsets { - // ignore errors here - a charset may not exist - err = mc.exec("SET NAMES " + charsets[i]) - if err == nil { - break - } - } - if err != nil { - return - } - - // System Vars - default: - err = mc.exec("SET " + param + "=" + val + "") - if err != nil { - return - } - } - } - - return -} - -func (mc *mysqlConn) Begin() (driver.Tx, error) { - if mc.netConn == nil { - errLog.Print(ErrInvalidConn) - return nil, driver.ErrBadConn - } - err := mc.exec("START TRANSACTION") - if err == nil { - return &mysqlTx{mc}, err - } - - return nil, err -} - -func (mc *mysqlConn) Close() (err error) { - // Makes Close idempotent - if mc.netConn != nil { - err = mc.writeCommandPacket(comQuit) - } - - mc.cleanup() - - return -} - -// Closes the network connection and unsets internal variables. Do not call this -// function after successfully authentication, call Close instead. This function -// is called before auth or on auth failure because MySQL will have already -// closed the network connection. -func (mc *mysqlConn) cleanup() { - // Makes cleanup idempotent - if mc.netConn != nil { - if err := mc.netConn.Close(); err != nil { - errLog.Print(err) - } - mc.netConn = nil - } - mc.cfg = nil - mc.buf.nc = nil -} - -func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { - if mc.netConn == nil { - errLog.Print(ErrInvalidConn) - return nil, driver.ErrBadConn - } - // Send command - err := mc.writeCommandPacketStr(comStmtPrepare, query) - if err != nil { - return nil, err - } - - stmt := &mysqlStmt{ - mc: mc, - } - - // Read Result - columnCount, err := stmt.readPrepareResultPacket() - if err == nil { - if stmt.paramCount > 0 { - if err = mc.readUntilEOF(); err != nil { - return nil, err - } - } - - if columnCount > 0 { - err = mc.readUntilEOF() - } - } - - return stmt, err -} - -func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) { - buf := mc.buf.takeCompleteBuffer() - if buf == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return "", driver.ErrBadConn - } - buf = buf[:0] - argPos := 0 - - for i := 0; i < len(query); i++ { - q := strings.IndexByte(query[i:], '?') - if q == -1 { - buf = append(buf, query[i:]...) - break - } - buf = append(buf, query[i:i+q]...) - i += q - - arg := args[argPos] - argPos++ - - if arg == nil { - buf = append(buf, "NULL"...) - continue - } - - switch v := arg.(type) { - case int64: - buf = strconv.AppendInt(buf, v, 10) - case float64: - buf = strconv.AppendFloat(buf, v, 'g', -1, 64) - case bool: - if v { - buf = append(buf, '1') - } else { - buf = append(buf, '0') - } - case time.Time: - if v.IsZero() { - buf = append(buf, "'0000-00-00'"...) - } else { - v := v.In(mc.cfg.Loc) - v = v.Add(time.Nanosecond * 500) // To round under microsecond - year := v.Year() - year100 := year / 100 - year1 := year % 100 - month := v.Month() - day := v.Day() - hour := v.Hour() - minute := v.Minute() - second := v.Second() - micro := v.Nanosecond() / 1000 - - buf = append(buf, []byte{ - '\'', - digits10[year100], digits01[year100], - digits10[year1], digits01[year1], - '-', - digits10[month], digits01[month], - '-', - digits10[day], digits01[day], - ' ', - digits10[hour], digits01[hour], - ':', - digits10[minute], digits01[minute], - ':', - digits10[second], digits01[second], - }...) - - if micro != 0 { - micro10000 := micro / 10000 - micro100 := micro / 100 % 100 - micro1 := micro % 100 - buf = append(buf, []byte{ - '.', - digits10[micro10000], digits01[micro10000], - digits10[micro100], digits01[micro100], - digits10[micro1], digits01[micro1], - }...) - } - buf = append(buf, '\'') - } - case []byte: - if v == nil { - buf = append(buf, "NULL"...) - } else { - buf = append(buf, "_binary'"...) - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeBytesBackslash(buf, v) - } else { - buf = escapeBytesQuotes(buf, v) - } - buf = append(buf, '\'') - } - case string: - buf = append(buf, '\'') - if mc.status&statusNoBackslashEscapes == 0 { - buf = escapeStringBackslash(buf, v) - } else { - buf = escapeStringQuotes(buf, v) - } - buf = append(buf, '\'') - default: - return "", driver.ErrSkip - } - - if len(buf)+4 > mc.maxPacketAllowed { - return "", driver.ErrSkip - } - } - if argPos != len(args) { - return "", driver.ErrSkip - } - return string(buf), nil -} - -func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) { - if mc.netConn == nil { - errLog.Print(ErrInvalidConn) - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.InterpolateParams { - return nil, driver.ErrSkip - } - // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - args = nil - } - mc.affectedRows = 0 - mc.insertId = 0 - - err := mc.exec(query) - if err == nil { - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, err - } - return nil, err -} - -// Internal function to execute commands -func (mc *mysqlConn) exec(query string) error { - // Send command - err := mc.writeCommandPacketStr(comQuery, query) - if err != nil { - return err - } - - // Read Result - resLen, err := mc.readResultSetHeaderPacket() - if err == nil && resLen > 0 { - if err = mc.readUntilEOF(); err != nil { - return err - } - - err = mc.readUntilEOF() - } - - return err -} - -func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) { - if mc.netConn == nil { - errLog.Print(ErrInvalidConn) - return nil, driver.ErrBadConn - } - if len(args) != 0 { - if !mc.cfg.InterpolateParams { - return nil, driver.ErrSkip - } - // try client-side prepare to reduce roundtrip - prepared, err := mc.interpolateParams(query, args) - if err != nil { - return nil, err - } - query = prepared - args = nil - } - // Send command - err := mc.writeCommandPacketStr(comQuery, query) - if err == nil { - // Read Result - var resLen int - resLen, err = mc.readResultSetHeaderPacket() - if err == nil { - rows := new(textRows) - rows.mc = mc - - if resLen == 0 { - // no columns, no more data - return emptyRows{}, nil - } - // Columns - rows.columns, err = mc.readColumns(resLen) - return rows, err - } - } - return nil, err -} - -// Gets the value of the given MySQL System Variable -// The returned byte slice is only valid until the next read -func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { - // Send command - if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil { - return nil, err - } - - // Read Result - resLen, err := mc.readResultSetHeaderPacket() - if err == nil { - rows := new(textRows) - rows.mc = mc - rows.columns = []mysqlField{{fieldType: fieldTypeVarChar}} - - if resLen > 0 { - // Columns - if err := mc.readUntilEOF(); err != nil { - return nil, err - } - } - - dest := make([]driver.Value, resLen) - if err = rows.readRow(dest); err == nil { - return dest[0].([]byte), mc.readUntilEOF() - } - } - return nil, err -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/const.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/const.go deleted file mode 100644 index 88cfff3..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/const.go +++ /dev/null @@ -1,163 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -const ( - minProtocolVersion byte = 10 - maxPacketSize = 1<<24 - 1 - timeFormat = "2006-01-02 15:04:05.999999" -) - -// MySQL constants documentation: -// http://dev.mysql.com/doc/internals/en/client-server-protocol.html - -const ( - iOK byte = 0x00 - iLocalInFile byte = 0xfb - iEOF byte = 0xfe - iERR byte = 0xff -) - -// https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags -type clientFlag uint32 - -const ( - clientLongPassword clientFlag = 1 << iota - clientFoundRows - clientLongFlag - clientConnectWithDB - clientNoSchema - clientCompress - clientODBC - clientLocalFiles - clientIgnoreSpace - clientProtocol41 - clientInteractive - clientSSL - clientIgnoreSIGPIPE - clientTransactions - clientReserved - clientSecureConn - clientMultiStatements - clientMultiResults - clientPSMultiResults - clientPluginAuth - clientConnectAttrs - clientPluginAuthLenEncClientData - clientCanHandleExpiredPasswords - clientSessionTrack - clientDeprecateEOF -) - -const ( - comQuit byte = iota + 1 - comInitDB - comQuery - comFieldList - comCreateDB - comDropDB - comRefresh - comShutdown - comStatistics - comProcessInfo - comConnect - comProcessKill - comDebug - comPing - comTime - comDelayedInsert - comChangeUser - comBinlogDump - comTableDump - comConnectOut - comRegisterSlave - comStmtPrepare - comStmtExecute - comStmtSendLongData - comStmtClose - comStmtReset - comSetOption - comStmtFetch -) - -// https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnType -const ( - fieldTypeDecimal byte = iota - fieldTypeTiny - fieldTypeShort - fieldTypeLong - fieldTypeFloat - fieldTypeDouble - fieldTypeNULL - fieldTypeTimestamp - fieldTypeLongLong - fieldTypeInt24 - fieldTypeDate - fieldTypeTime - fieldTypeDateTime - fieldTypeYear - fieldTypeNewDate - fieldTypeVarChar - fieldTypeBit -) -const ( - fieldTypeJSON byte = iota + 0xf5 - fieldTypeNewDecimal - fieldTypeEnum - fieldTypeSet - fieldTypeTinyBLOB - fieldTypeMediumBLOB - fieldTypeLongBLOB - fieldTypeBLOB - fieldTypeVarString - fieldTypeString - fieldTypeGeometry -) - -type fieldFlag uint16 - -const ( - flagNotNULL fieldFlag = 1 << iota - flagPriKey - flagUniqueKey - flagMultipleKey - flagBLOB - flagUnsigned - flagZeroFill - flagBinary - flagEnum - flagAutoIncrement - flagTimestamp - flagSet - flagUnknown1 - flagUnknown2 - flagUnknown3 - flagUnknown4 -) - -// http://dev.mysql.com/doc/internals/en/status-flags.html -type statusFlag uint16 - -const ( - statusInTrans statusFlag = 1 << iota - statusInAutocommit - statusReserved // Not in documentation - statusMoreResultsExists - statusNoGoodIndexUsed - statusNoIndexUsed - statusCursorExists - statusLastRowSent - statusDbDropped - statusNoBackslashEscapes - statusMetadataChanged - statusQueryWasSlow - statusPsOutParams - statusInTransReadonly - statusSessionStateChanged -) diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/driver.go deleted file mode 100644 index 899f955..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/driver.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// Package mysql provides a MySQL driver for Go's database/sql package -// -// The driver should be used via the database/sql package: -// -// import "database/sql" -// import _ "github.com/go-sql-driver/mysql" -// -// db, err := sql.Open("mysql", "user:password@/dbname") -// -// See https://github.com/go-sql-driver/mysql#usage for details -package mysql - -import ( - "database/sql" - "database/sql/driver" - "net" -) - -// MySQLDriver is exported to make the driver directly accessible. -// In general the driver is used via the database/sql package. -type MySQLDriver struct{} - -// DialFunc is a function which can be used to establish the network connection. -// Custom dial functions must be registered with RegisterDial -type DialFunc func(addr string) (net.Conn, error) - -var dials map[string]DialFunc - -// RegisterDial registers a custom dial function. It can then be used by the -// network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { - if dials == nil { - dials = make(map[string]DialFunc) - } - dials[net] = dial -} - -// Open new Connection. -// See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how -// the DSN string is formated -func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxPacketAllowed: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - } - mc.cfg, err = ParseDSN(dsn) - if err != nil { - return nil, err - } - mc.parseTime = mc.cfg.ParseTime - mc.strict = mc.cfg.Strict - - // Connect to Server - if dial, ok := dials[mc.cfg.Net]; ok { - mc.netConn, err = dial(mc.cfg.Addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.Timeout} - mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) - } - if err != nil { - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - mc.buf = newBuffer(mc.netConn) - - // Set I/O timeouts - mc.buf.timeout = mc.cfg.ReadTimeout - mc.writeTimeout = mc.cfg.WriteTimeout - - // Reading Handshake Initialization Packet - cipher, err := mc.readInitPacket() - if err != nil { - mc.cleanup() - return nil, err - } - - // Send Client Authentication Packet - if err = mc.writeAuthPacket(cipher); err != nil { - mc.cleanup() - return nil, err - } - - // Handle response to auth packet, switch methods if possible - if err = handleAuthResult(mc, cipher); err != nil { - // Authentication failed and MySQL has already closed the connection - // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). - // Do not send COM_QUIT, just cleanup and return the error. - mc.cleanup() - return nil, err - } - - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxPacketAllowed = stringToInt(maxap) - 1 - if mc.maxPacketAllowed < maxPacketSize { - mc.maxWriteSize = mc.maxPacketAllowed - } - - // Handle DSN Params - err = mc.handleParams() - if err != nil { - mc.Close() - return nil, err - } - - return mc, nil -} - -func handleAuthResult(mc *mysqlConn, cipher []byte) error { - // Read Result Packet - err := mc.readResultOK() - if err == nil { - return nil // auth successful - } - - if mc.cfg == nil { - return err // auth failed and retry not possible - } - - // Retry auth if configured to do so. - if mc.cfg.AllowOldPasswords && err == ErrOldPassword { - // Retry with old authentication method. Note: there are edge cases - // where this should work but doesn't; this is currently "wontfix": - // https://github.com/go-sql-driver/mysql/issues/184 - if err = mc.writeOldAuthPacket(cipher); err != nil { - return err - } - err = mc.readResultOK() - } else if mc.cfg.AllowCleartextPasswords && err == ErrCleartextPassword { - // Retry with clear text password for - // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html - // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html - if err = mc.writeClearAuthPacket(); err != nil { - return err - } - err = mc.readResultOK() - } - return err -} - -func init() { - sql.Register("mysql", &MySQLDriver{}) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/dsn.go deleted file mode 100644 index 73138bc..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/dsn.go +++ /dev/null @@ -1,513 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "bytes" - "crypto/tls" - "errors" - "fmt" - "net" - "net/url" - "strings" - "time" -) - -var ( - errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") - errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") - errInvalidDSNUnsafeCollation = errors.New("invalid DSN: interpolateParams can not be used with unsafe collations") -) - -// Config is a configuration parsed from a DSN string -type Config struct { - User string // Username - Passwd string // Password (requires User) - Net string // Network type - Addr string // Network address (requires Net) - DBName string // Database name - Params map[string]string // Connection parameters - Collation string // Connection collation - Loc *time.Location // Location for time.Time values - TLSConfig string // TLS configuration name - tls *tls.Config // TLS configuration - Timeout time.Duration // Dial timeout - ReadTimeout time.Duration // I/O read timeout - WriteTimeout time.Duration // I/O write timeout - - AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE - AllowCleartextPasswords bool // Allows the cleartext client side plugin - AllowOldPasswords bool // Allows the old insecure password method - ClientFoundRows bool // Return number of matching rows instead of rows changed - ColumnsWithAlias bool // Prepend table alias to column names - InterpolateParams bool // Interpolate placeholders into query string - MultiStatements bool // Allow multiple statements in one query - ParseTime bool // Parse time values to time.Time - Strict bool // Return warnings as errors -} - -// FormatDSN formats the given Config into a DSN string which can be passed to -// the driver. -func (cfg *Config) FormatDSN() string { - var buf bytes.Buffer - - // [username[:password]@] - if len(cfg.User) > 0 { - buf.WriteString(cfg.User) - if len(cfg.Passwd) > 0 { - buf.WriteByte(':') - buf.WriteString(cfg.Passwd) - } - buf.WriteByte('@') - } - - // [protocol[(address)]] - if len(cfg.Net) > 0 { - buf.WriteString(cfg.Net) - if len(cfg.Addr) > 0 { - buf.WriteByte('(') - buf.WriteString(cfg.Addr) - buf.WriteByte(')') - } - } - - // /dbname - buf.WriteByte('/') - buf.WriteString(cfg.DBName) - - // [?param1=value1&...¶mN=valueN] - hasParam := false - - if cfg.AllowAllFiles { - hasParam = true - buf.WriteString("?allowAllFiles=true") - } - - if cfg.AllowCleartextPasswords { - if hasParam { - buf.WriteString("&allowCleartextPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowCleartextPasswords=true") - } - } - - if cfg.AllowOldPasswords { - if hasParam { - buf.WriteString("&allowOldPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowOldPasswords=true") - } - } - - if cfg.ClientFoundRows { - if hasParam { - buf.WriteString("&clientFoundRows=true") - } else { - hasParam = true - buf.WriteString("?clientFoundRows=true") - } - } - - if col := cfg.Collation; col != defaultCollation && len(col) > 0 { - if hasParam { - buf.WriteString("&collation=") - } else { - hasParam = true - buf.WriteString("?collation=") - } - buf.WriteString(col) - } - - if cfg.ColumnsWithAlias { - if hasParam { - buf.WriteString("&columnsWithAlias=true") - } else { - hasParam = true - buf.WriteString("?columnsWithAlias=true") - } - } - - if cfg.InterpolateParams { - if hasParam { - buf.WriteString("&interpolateParams=true") - } else { - hasParam = true - buf.WriteString("?interpolateParams=true") - } - } - - if cfg.Loc != time.UTC && cfg.Loc != nil { - if hasParam { - buf.WriteString("&loc=") - } else { - hasParam = true - buf.WriteString("?loc=") - } - buf.WriteString(url.QueryEscape(cfg.Loc.String())) - } - - if cfg.MultiStatements { - if hasParam { - buf.WriteString("&multiStatements=true") - } else { - hasParam = true - buf.WriteString("?multiStatements=true") - } - } - - if cfg.ParseTime { - if hasParam { - buf.WriteString("&parseTime=true") - } else { - hasParam = true - buf.WriteString("?parseTime=true") - } - } - - if cfg.ReadTimeout > 0 { - if hasParam { - buf.WriteString("&readTimeout=") - } else { - hasParam = true - buf.WriteString("?readTimeout=") - } - buf.WriteString(cfg.ReadTimeout.String()) - } - - if cfg.Strict { - if hasParam { - buf.WriteString("&strict=true") - } else { - hasParam = true - buf.WriteString("?strict=true") - } - } - - if cfg.Timeout > 0 { - if hasParam { - buf.WriteString("&timeout=") - } else { - hasParam = true - buf.WriteString("?timeout=") - } - buf.WriteString(cfg.Timeout.String()) - } - - if len(cfg.TLSConfig) > 0 { - if hasParam { - buf.WriteString("&tls=") - } else { - hasParam = true - buf.WriteString("?tls=") - } - buf.WriteString(url.QueryEscape(cfg.TLSConfig)) - } - - if cfg.WriteTimeout > 0 { - if hasParam { - buf.WriteString("&writeTimeout=") - } else { - hasParam = true - buf.WriteString("?writeTimeout=") - } - buf.WriteString(cfg.WriteTimeout.String()) - } - - // other params - if cfg.Params != nil { - for param, value := range cfg.Params { - if hasParam { - buf.WriteByte('&') - } else { - hasParam = true - buf.WriteByte('?') - } - - buf.WriteString(param) - buf.WriteByte('=') - buf.WriteString(url.QueryEscape(value)) - } - } - - return buf.String() -} - -// ParseDSN parses the DSN string to a Config -func ParseDSN(dsn string) (cfg *Config, err error) { - // New config with some default values - cfg = &Config{ - Loc: time.UTC, - Collation: defaultCollation, - } - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.Passwd = dsn[k+1 : j] - break - } - } - cfg.User = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - cfg.Addr = dsn[k+1 : i-1] - break - } - } - cfg.Net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.DBName = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { - return nil, errInvalidDSNUnsafeCollation - } - - // Set default network if empty - if cfg.Net == "" { - cfg.Net = "tcp" - } - - // Set default address if empty - if cfg.Addr == "" { - switch cfg.Net { - case "tcp": - cfg.Addr = "127.0.0.1:3306" - case "unix": - cfg.Addr = "/tmp/mysql.sock" - default: - return nil, errors.New("default addr for network '" + cfg.Net + "' unknown") - } - - } - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *Config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - - // Disable INFILE whitelist / enable all files - case "allowAllFiles": - var isBool bool - cfg.AllowAllFiles, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Use cleartext authentication mode (MySQL 5.5.10+) - case "allowCleartextPasswords": - var isBool bool - cfg.AllowCleartextPasswords, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Use old authentication mode (pre MySQL 4.1) - case "allowOldPasswords": - var isBool bool - cfg.AllowOldPasswords, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Switch "rowsAffected" mode - case "clientFoundRows": - var isBool bool - cfg.ClientFoundRows, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Collation - case "collation": - cfg.Collation = value - break - - case "columnsWithAlias": - var isBool bool - cfg.ColumnsWithAlias, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Compression - case "compress": - return errors.New("compression not implemented yet") - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.InterpolateParams, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.Loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // multiple statements in one query - case "multiStatements": - var isBool bool - cfg.MultiStatements, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // time.Time parsing - case "parseTime": - var isBool bool - cfg.ParseTime, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // I/O read Timeout - case "readTimeout": - cfg.ReadTimeout, err = time.ParseDuration(value) - if err != nil { - return - } - - // Strict mode - case "strict": - var isBool bool - cfg.Strict, isBool = readBool(value) - if !isBool { - return errors.New("invalid bool value: " + value) - } - - // Dial Timeout - case "timeout": - cfg.Timeout, err = time.ParseDuration(value) - if err != nil { - return - } - - // TLS-Encryption - case "tls": - boolValue, isBool := readBool(value) - if isBool { - if boolValue { - cfg.TLSConfig = "true" - cfg.tls = &tls.Config{} - } else { - cfg.TLSConfig = "false" - } - } else if vl := strings.ToLower(value); vl == "skip-verify" { - cfg.TLSConfig = vl - cfg.tls = &tls.Config{InsecureSkipVerify: true} - } else { - name, err := url.QueryUnescape(value) - if err != nil { - return fmt.Errorf("invalid value for TLS config name: %v", err) - } - - if tlsConfig, ok := tlsConfigRegister[name]; ok { - if len(tlsConfig.ServerName) == 0 && !tlsConfig.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.Addr) - if err == nil { - tlsConfig.ServerName = host - } - } - - cfg.TLSConfig = name - cfg.tls = tlsConfig - } else { - return errors.New("invalid value / unknown config name: " + name) - } - } - - // I/O write Timeout - case "writeTimeout": - cfg.WriteTimeout, err = time.ParseDuration(value) - if err != nil { - return - } - - default: - // lazy init - if cfg.Params == nil { - cfg.Params = make(map[string]string) - } - - if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - - return -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/errors.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/errors.go deleted file mode 100644 index 1543a80..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/errors.go +++ /dev/null @@ -1,131 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "database/sql/driver" - "errors" - "fmt" - "io" - "log" - "os" -) - -// Various errors the driver might return. Can change between driver versions. -var ( - ErrInvalidConn = errors.New("invalid connection") - ErrMalformPkt = errors.New("malformed packet") - ErrNoTLS = errors.New("TLS requested but server does not support TLS") - ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") - ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") - ErrUnknownPlugin = errors.New("this authentication plugin is not supported") - ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") - ErrPktSync = errors.New("commands out of sync. You can't run this command now") - ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") - ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") - ErrBusyBuffer = errors.New("busy buffer") -) - -var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) - -// Logger is used to log critical error messages. -type Logger interface { - Print(v ...interface{}) -} - -// SetLogger is used to set the logger for critical errors. -// The initial logger is os.Stderr. -func SetLogger(logger Logger) error { - if logger == nil { - return errors.New("logger is nil") - } - errLog = logger - return nil -} - -// MySQLError is an error type which represents a single MySQL error -type MySQLError struct { - Number uint16 - Message string -} - -func (me *MySQLError) Error() string { - return fmt.Sprintf("Error %d: %s", me.Number, me.Message) -} - -// MySQLWarnings is an error type which represents a group of one or more MySQL -// warnings -type MySQLWarnings []MySQLWarning - -func (mws MySQLWarnings) Error() string { - var msg string - for i, warning := range mws { - if i > 0 { - msg += "\r\n" - } - msg += fmt.Sprintf( - "%s %s: %s", - warning.Level, - warning.Code, - warning.Message, - ) - } - return msg -} - -// MySQLWarning is an error type which represents a single MySQL warning. -// Warnings are returned in groups only. See MySQLWarnings -type MySQLWarning struct { - Level string - Code string - Message string -} - -func (mc *mysqlConn) getWarnings() (err error) { - rows, err := mc.Query("SHOW WARNINGS", nil) - if err != nil { - return - } - - var warnings = MySQLWarnings{} - var values = make([]driver.Value, 3) - - for { - err = rows.Next(values) - switch err { - case nil: - warning := MySQLWarning{} - - if raw, ok := values[0].([]byte); ok { - warning.Level = string(raw) - } else { - warning.Level = fmt.Sprintf("%s", values[0]) - } - if raw, ok := values[1].([]byte); ok { - warning.Code = string(raw) - } else { - warning.Code = fmt.Sprintf("%s", values[1]) - } - if raw, ok := values[2].([]byte); ok { - warning.Message = string(raw) - } else { - warning.Message = fmt.Sprintf("%s", values[0]) - } - - warnings = append(warnings, warning) - - case io.EOF: - return warnings - - default: - rows.Close() - return - } - } -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/infile.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/infile.go deleted file mode 100644 index 0f975bb..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/infile.go +++ /dev/null @@ -1,181 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "fmt" - "io" - "os" - "strings" - "sync" -) - -var ( - fileRegister map[string]bool - fileRegisterLock sync.RWMutex - readerRegister map[string]func() io.Reader - readerRegisterLock sync.RWMutex -) - -// RegisterLocalFile adds the given file to the file whitelist, -// so that it can be used by "LOAD DATA LOCAL INFILE ". -// Alternatively you can allow the use of all local files with -// the DSN parameter 'allowAllFiles=true' -// -// filePath := "/home/gopher/data.csv" -// mysql.RegisterLocalFile(filePath) -// err := db.Exec("LOAD DATA LOCAL INFILE '" + filePath + "' INTO TABLE foo") -// if err != nil { -// ... -// -func RegisterLocalFile(filePath string) { - fileRegisterLock.Lock() - // lazy map init - if fileRegister == nil { - fileRegister = make(map[string]bool) - } - - fileRegister[strings.Trim(filePath, `"`)] = true - fileRegisterLock.Unlock() -} - -// DeregisterLocalFile removes the given filepath from the whitelist. -func DeregisterLocalFile(filePath string) { - fileRegisterLock.Lock() - delete(fileRegister, strings.Trim(filePath, `"`)) - fileRegisterLock.Unlock() -} - -// RegisterReaderHandler registers a handler function which is used -// to receive a io.Reader. -// The Reader can be used by "LOAD DATA LOCAL INFILE Reader::". -// If the handler returns a io.ReadCloser Close() is called when the -// request is finished. -// -// mysql.RegisterReaderHandler("data", func() io.Reader { -// var csvReader io.Reader // Some Reader that returns CSV data -// ... // Open Reader here -// return csvReader -// }) -// err := db.Exec("LOAD DATA LOCAL INFILE 'Reader::data' INTO TABLE foo") -// if err != nil { -// ... -// -func RegisterReaderHandler(name string, handler func() io.Reader) { - readerRegisterLock.Lock() - // lazy map init - if readerRegister == nil { - readerRegister = make(map[string]func() io.Reader) - } - - readerRegister[name] = handler - readerRegisterLock.Unlock() -} - -// DeregisterReaderHandler removes the ReaderHandler function with -// the given name from the registry. -func DeregisterReaderHandler(name string) { - readerRegisterLock.Lock() - delete(readerRegister, name) - readerRegisterLock.Unlock() -} - -func deferredClose(err *error, closer io.Closer) { - closeErr := closer.Close() - if *err == nil { - *err = closeErr - } -} - -func (mc *mysqlConn) handleInFileRequest(name string) (err error) { - var rdr io.Reader - var data []byte - packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP - if mc.maxWriteSize < packetSize { - packetSize = mc.maxWriteSize - } - - if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader - // The server might return an an absolute path. See issue #355. - name = name[idx+8:] - - readerRegisterLock.RLock() - handler, inMap := readerRegister[name] - readerRegisterLock.RUnlock() - - if inMap { - rdr = handler() - if rdr != nil { - if cl, ok := rdr.(io.Closer); ok { - defer deferredClose(&err, cl) - } - } else { - err = fmt.Errorf("Reader '%s' is ", name) - } - } else { - err = fmt.Errorf("Reader '%s' is not registered", name) - } - } else { // File - name = strings.Trim(name, `"`) - fileRegisterLock.RLock() - fr := fileRegister[name] - fileRegisterLock.RUnlock() - if mc.cfg.AllowAllFiles || fr { - var file *os.File - var fi os.FileInfo - - if file, err = os.Open(name); err == nil { - defer deferredClose(&err, file) - - // get file size - if fi, err = file.Stat(); err == nil { - rdr = file - if fileSize := int(fi.Size()); fileSize < packetSize { - packetSize = fileSize - } - } - } - } else { - err = fmt.Errorf("local file '%s' is not registered", name) - } - } - - // send content packets - if err == nil { - data := make([]byte, 4+packetSize) - var n int - for err == nil { - n, err = rdr.Read(data[4:]) - if n > 0 { - if ioErr := mc.writePacket(data[:4+n]); ioErr != nil { - return ioErr - } - } - } - if err == io.EOF { - err = nil - } - } - - // send empty packet (termination) - if data == nil { - data = make([]byte, 4) - } - if ioErr := mc.writePacket(data[:4]); ioErr != nil { - return ioErr - } - - // read OK packet - if err == nil { - return mc.readResultOK() - } - - mc.readPacket() - return err -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/packets.go deleted file mode 100644 index 8d91665..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/packets.go +++ /dev/null @@ -1,1243 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "bytes" - "crypto/tls" - "database/sql/driver" - "encoding/binary" - "errors" - "fmt" - "io" - "math" - "time" -) - -// Packets documentation: -// http://dev.mysql.com/doc/internals/en/client-server-protocol.html - -// Read packet to buffer 'data' -func (mc *mysqlConn) readPacket() ([]byte, error) { - var payload []byte - for { - // Read packet header - data, err := mc.buf.readNext(4) - if err != nil { - errLog.Print(err) - mc.Close() - return nil, driver.ErrBadConn - } - - // Packet Length [24 bit] - pktLen := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) - - if pktLen < 1 { - errLog.Print(ErrMalformPkt) - mc.Close() - return nil, driver.ErrBadConn - } - - // Check Packet Sync [8 bit] - if data[3] != mc.sequence { - if data[3] > mc.sequence { - return nil, ErrPktSyncMul - } - return nil, ErrPktSync - } - mc.sequence++ - - // Read packet body [pktLen bytes] - data, err = mc.buf.readNext(pktLen) - if err != nil { - errLog.Print(err) - mc.Close() - return nil, driver.ErrBadConn - } - - isLastPacket := (pktLen < maxPacketSize) - - // Zero allocations for non-splitting packets - if isLastPacket && payload == nil { - return data, nil - } - - payload = append(payload, data...) - - if isLastPacket { - return payload, nil - } - } -} - -// Write packet buffer 'data' -func (mc *mysqlConn) writePacket(data []byte) error { - pktLen := len(data) - 4 - - if pktLen > mc.maxPacketAllowed { - return ErrPktTooLarge - } - - for { - var size int - if pktLen >= maxPacketSize { - data[0] = 0xff - data[1] = 0xff - data[2] = 0xff - size = maxPacketSize - } else { - data[0] = byte(pktLen) - data[1] = byte(pktLen >> 8) - data[2] = byte(pktLen >> 16) - size = pktLen - } - data[3] = mc.sequence - - // Write packet - if mc.writeTimeout > 0 { - if err := mc.netConn.SetWriteDeadline(time.Now().Add(mc.writeTimeout)); err != nil { - return err - } - } - - n, err := mc.netConn.Write(data[:4+size]) - if err == nil && n == 4+size { - mc.sequence++ - if size != maxPacketSize { - return nil - } - pktLen -= size - data = data[size:] - continue - } - - // Handle error - if err == nil { // n != len(data) - errLog.Print(ErrMalformPkt) - } else { - errLog.Print(err) - } - return driver.ErrBadConn - } -} - -/****************************************************************************** -* Initialisation Process * -******************************************************************************/ - -// Handshake Initialization Packet -// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake -func (mc *mysqlConn) readInitPacket() ([]byte, error) { - data, err := mc.readPacket() - if err != nil { - return nil, err - } - - if data[0] == iERR { - return nil, mc.handleErrorPacket(data) - } - - // protocol version [1 byte] - if data[0] < minProtocolVersion { - return nil, fmt.Errorf( - "unsupported protocol version %d. Version %d or higher is required", - data[0], - minProtocolVersion, - ) - } - - // server version [null terminated string] - // connection id [4 bytes] - pos := 1 + bytes.IndexByte(data[1:], 0x00) + 1 + 4 - - // first part of the password cipher [8 bytes] - cipher := data[pos : pos+8] - - // (filler) always 0x00 [1 byte] - pos += 8 + 1 - - // capability flags (lower 2 bytes) [2 bytes] - mc.flags = clientFlag(binary.LittleEndian.Uint16(data[pos : pos+2])) - if mc.flags&clientProtocol41 == 0 { - return nil, ErrOldProtocol - } - if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - return nil, ErrNoTLS - } - pos += 2 - - if len(data) > pos { - // character set [1 byte] - // status flags [2 bytes] - // capability flags (upper 2 bytes) [2 bytes] - // length of auth-plugin-data [1 byte] - // reserved (all [00]) [10 bytes] - pos += 1 + 2 + 2 + 1 + 10 - - // second part of the password cipher [mininum 13 bytes], - // where len=MAX(13, length of auth-plugin-data - 8) - // - // The web documentation is ambiguous about the length. However, - // according to mysql-5.7/sql/auth/sql_authentication.cc line 538, - // the 13th byte is "\0 byte, terminating the second part of - // a scramble". So the second part of the password cipher is - // a NULL terminated string that's at least 13 bytes with the - // last byte being NULL. - // - // The official Python library uses the fixed length 12 - // which seems to work but technically could have a hidden bug. - cipher = append(cipher, data[pos:pos+12]...) - - // TODO: Verify string termination - // EOF if version (>= 5.5.7 and < 5.5.10) or (>= 5.6.0 and < 5.6.2) - // \NUL otherwise - // - //if data[len(data)-1] == 0 { - // return - //} - //return ErrMalformPkt - - // make a memory safe copy of the cipher slice - var b [20]byte - copy(b[:], cipher) - return b[:], nil - } - - // make a memory safe copy of the cipher slice - var b [8]byte - copy(b[:], cipher) - return b[:], nil -} - -// Client Authentication Packet -// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse -func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { - // Adjust client flags based on server support - clientFlags := clientProtocol41 | - clientSecureConn | - clientLongPassword | - clientTransactions | - clientLocalFiles | - clientPluginAuth | - clientMultiResults | - mc.flags&clientLongFlag - - if mc.cfg.ClientFoundRows { - clientFlags |= clientFoundRows - } - - // To enable TLS / SSL - if mc.cfg.tls != nil { - clientFlags |= clientSSL - } - - if mc.cfg.MultiStatements { - clientFlags |= clientMultiStatements - } - - // User Password - scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.Passwd)) - - pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + 1 + len(scrambleBuff) + 21 + 1 - - // To specify a db name - if n := len(mc.cfg.DBName); n > 0 { - clientFlags |= clientConnectWithDB - pktLen += n + 1 - } - - // Calculate packet length and get buffer with that size - data := mc.buf.takeSmallBuffer(pktLen + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // ClientFlags [32 bit] - data[4] = byte(clientFlags) - data[5] = byte(clientFlags >> 8) - data[6] = byte(clientFlags >> 16) - data[7] = byte(clientFlags >> 24) - - // MaxPacketSize [32 bit] (none) - data[8] = 0x00 - data[9] = 0x00 - data[10] = 0x00 - data[11] = 0x00 - - // Charset [1 byte] - var found bool - data[12], found = collations[mc.cfg.Collation] - if !found { - // Note possibility for false negatives: - // could be triggered although the collation is valid if the - // collations map does not contain entries the server supports. - return errors.New("unknown collation") - } - - // SSL Connection Request Packet - // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest - if mc.cfg.tls != nil { - // Send TLS / SSL request packet - if err := mc.writePacket(data[:(4+4+1+23)+4]); err != nil { - return err - } - - // Switch to TLS - tlsConn := tls.Client(mc.netConn, mc.cfg.tls) - if err := tlsConn.Handshake(); err != nil { - return err - } - mc.netConn = tlsConn - mc.buf.nc = tlsConn - } - - // Filler [23 bytes] (all 0x00) - pos := 13 - for ; pos < 13+23; pos++ { - data[pos] = 0 - } - - // User [null terminated string] - if len(mc.cfg.User) > 0 { - pos += copy(data[pos:], mc.cfg.User) - } - data[pos] = 0x00 - pos++ - - // ScrambleBuffer [length encoded integer] - data[pos] = byte(len(scrambleBuff)) - pos += 1 + copy(data[pos+1:], scrambleBuff) - - // Databasename [null terminated string] - if len(mc.cfg.DBName) > 0 { - pos += copy(data[pos:], mc.cfg.DBName) - data[pos] = 0x00 - pos++ - } - - // Assume native client during response - pos += copy(data[pos:], "mysql_native_password") - data[pos] = 0x00 - - // Send Auth packet - return mc.writePacket(data) -} - -// Client old authentication packet -// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeOldAuthPacket(cipher []byte) error { - // User password - scrambleBuff := scrambleOldPassword(cipher, []byte(mc.cfg.Passwd)) - - // Calculate the packet length and add a tailing 0 - pktLen := len(scrambleBuff) + 1 - data := mc.buf.takeSmallBuffer(4 + pktLen) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // Add the scrambled password [null terminated string] - copy(data[4:], scrambleBuff) - data[4+pktLen-1] = 0x00 - - return mc.writePacket(data) -} - -// Client clear text authentication packet -// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeClearAuthPacket() error { - // Calculate the packet length and add a tailing 0 - pktLen := len(mc.cfg.Passwd) + 1 - data := mc.buf.takeSmallBuffer(4 + pktLen) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // Add the clear password [null terminated string] - copy(data[4:], mc.cfg.Passwd) - data[4+pktLen-1] = 0x00 - - return mc.writePacket(data) -} - -/****************************************************************************** -* Command Packets * -******************************************************************************/ - -func (mc *mysqlConn) writeCommandPacket(command byte) error { - // Reset Packet Sequence - mc.sequence = 0 - - data := mc.buf.takeSmallBuffer(4 + 1) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // Add command byte - data[4] = command - - // Send CMD packet - return mc.writePacket(data) -} - -func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { - // Reset Packet Sequence - mc.sequence = 0 - - pktLen := 1 + len(arg) - data := mc.buf.takeBuffer(pktLen + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // Add command byte - data[4] = command - - // Add arg - copy(data[5:], arg) - - // Send CMD packet - return mc.writePacket(data) -} - -func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { - // Reset Packet Sequence - mc.sequence = 0 - - data := mc.buf.takeSmallBuffer(4 + 1 + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // Add command byte - data[4] = command - - // Add arg [32 bit] - data[5] = byte(arg) - data[6] = byte(arg >> 8) - data[7] = byte(arg >> 16) - data[8] = byte(arg >> 24) - - // Send CMD packet - return mc.writePacket(data) -} - -/****************************************************************************** -* Result Packets * -******************************************************************************/ - -// Returns error if Packet is not an 'Result OK'-Packet -func (mc *mysqlConn) readResultOK() error { - data, err := mc.readPacket() - if err == nil { - // packet indicator - switch data[0] { - - case iOK: - return mc.handleOkPacket(data) - - case iEOF: - if len(data) > 1 { - plugin := string(data[1:bytes.IndexByte(data, 0x00)]) - if plugin == "mysql_old_password" { - // using old_passwords - return ErrOldPassword - } else if plugin == "mysql_clear_password" { - // using clear text password - return ErrCleartextPassword - } else { - return ErrUnknownPlugin - } - } else { - return ErrOldPassword - } - - default: // Error otherwise - return mc.handleErrorPacket(data) - } - } - return err -} - -// Result Set Header Packet -// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::Resultset -func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { - data, err := mc.readPacket() - if err == nil { - switch data[0] { - - case iOK: - return 0, mc.handleOkPacket(data) - - case iERR: - return 0, mc.handleErrorPacket(data) - - case iLocalInFile: - return 0, mc.handleInFileRequest(string(data[1:])) - } - - // column count - num, _, n := readLengthEncodedInteger(data) - if n-len(data) == 0 { - return int(num), nil - } - - return 0, ErrMalformPkt - } - return 0, err -} - -// Error Packet -// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-ERR_Packet -func (mc *mysqlConn) handleErrorPacket(data []byte) error { - if data[0] != iERR { - return ErrMalformPkt - } - - // 0xff [1 byte] - - // Error Number [16 bit uint] - errno := binary.LittleEndian.Uint16(data[1:3]) - - pos := 3 - - // SQL State [optional: # + 5bytes string] - if data[3] == 0x23 { - //sqlstate := string(data[4 : 4+5]) - pos = 9 - } - - // Error Message [string] - return &MySQLError{ - Number: errno, - Message: string(data[pos:]), - } -} - -func readStatus(b []byte) statusFlag { - return statusFlag(b[0]) | statusFlag(b[1])<<8 -} - -// Ok Packet -// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet -func (mc *mysqlConn) handleOkPacket(data []byte) error { - var n, m int - - // 0x00 [1 byte] - - // Affected rows [Length Coded Binary] - mc.affectedRows, _, n = readLengthEncodedInteger(data[1:]) - - // Insert id [Length Coded Binary] - mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) - - // server_status [2 bytes] - mc.status = readStatus(data[1+n+m : 1+n+m+2]) - if err := mc.discardResults(); err != nil { - return err - } - - // warning count [2 bytes] - if !mc.strict { - return nil - } - - pos := 1 + n + m + 2 - if binary.LittleEndian.Uint16(data[pos:pos+2]) > 0 { - return mc.getWarnings() - } - return nil -} - -// Read Packets as Field Packets until EOF-Packet or an Error appears -// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41 -func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { - columns := make([]mysqlField, count) - - for i := 0; ; i++ { - data, err := mc.readPacket() - if err != nil { - return nil, err - } - - // EOF Packet - if data[0] == iEOF && (len(data) == 5 || len(data) == 1) { - if i == count { - return columns, nil - } - return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns)) - } - - // Catalog - pos, err := skipLengthEncodedString(data) - if err != nil { - return nil, err - } - - // Database [len coded string] - n, err := skipLengthEncodedString(data[pos:]) - if err != nil { - return nil, err - } - pos += n - - // Table [len coded string] - if mc.cfg.ColumnsWithAlias { - tableName, _, n, err := readLengthEncodedString(data[pos:]) - if err != nil { - return nil, err - } - pos += n - columns[i].tableName = string(tableName) - } else { - n, err = skipLengthEncodedString(data[pos:]) - if err != nil { - return nil, err - } - pos += n - } - - // Original table [len coded string] - n, err = skipLengthEncodedString(data[pos:]) - if err != nil { - return nil, err - } - pos += n - - // Name [len coded string] - name, _, n, err := readLengthEncodedString(data[pos:]) - if err != nil { - return nil, err - } - columns[i].name = string(name) - pos += n - - // Original name [len coded string] - n, err = skipLengthEncodedString(data[pos:]) - if err != nil { - return nil, err - } - - // Filler [uint8] - // Charset [charset, collation uint8] - // Length [uint32] - pos += n + 1 + 2 + 4 - - // Field type [uint8] - columns[i].fieldType = data[pos] - pos++ - - // Flags [uint16] - columns[i].flags = fieldFlag(binary.LittleEndian.Uint16(data[pos : pos+2])) - pos += 2 - - // Decimals [uint8] - columns[i].decimals = data[pos] - //pos++ - - // Default value [len coded binary] - //if pos < len(data) { - // defaultVal, _, err = bytesToLengthCodedBinary(data[pos:]) - //} - } -} - -// Read Packets as Field Packets until EOF-Packet or an Error appears -// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::ResultsetRow -func (rows *textRows) readRow(dest []driver.Value) error { - mc := rows.mc - - data, err := mc.readPacket() - if err != nil { - return err - } - - // EOF Packet - if data[0] == iEOF && len(data) == 5 { - // server_status [2 bytes] - rows.mc.status = readStatus(data[3:]) - if err := rows.mc.discardResults(); err != nil { - return err - } - rows.mc = nil - return io.EOF - } - if data[0] == iERR { - rows.mc = nil - return mc.handleErrorPacket(data) - } - - // RowSet Packet - var n int - var isNull bool - pos := 0 - - for i := range dest { - // Read bytes and convert to string - dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) - pos += n - if err == nil { - if !isNull { - if !mc.parseTime { - continue - } else { - switch rows.columns[i].fieldType { - case fieldTypeTimestamp, fieldTypeDateTime, - fieldTypeDate, fieldTypeNewDate: - dest[i], err = parseDateTime( - string(dest[i].([]byte)), - mc.cfg.Loc, - ) - if err == nil { - continue - } - default: - continue - } - } - - } else { - dest[i] = nil - continue - } - } - return err // err != nil - } - - return nil -} - -// Reads Packets until EOF-Packet or an Error appears. Returns count of Packets read -func (mc *mysqlConn) readUntilEOF() error { - for { - data, err := mc.readPacket() - - // No Err and no EOF Packet - if err == nil && data[0] != iEOF { - continue - } - if err == nil && data[0] == iEOF && len(data) == 5 { - mc.status = readStatus(data[3:]) - } - - return err // Err or EOF - } -} - -/****************************************************************************** -* Prepared Statements * -******************************************************************************/ - -// Prepare Result Packets -// http://dev.mysql.com/doc/internals/en/com-stmt-prepare-response.html -func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) { - data, err := stmt.mc.readPacket() - if err == nil { - // packet indicator [1 byte] - if data[0] != iOK { - return 0, stmt.mc.handleErrorPacket(data) - } - - // statement id [4 bytes] - stmt.id = binary.LittleEndian.Uint32(data[1:5]) - - // Column count [16 bit uint] - columnCount := binary.LittleEndian.Uint16(data[5:7]) - - // Param count [16 bit uint] - stmt.paramCount = int(binary.LittleEndian.Uint16(data[7:9])) - - // Reserved [8 bit] - - // Warning count [16 bit uint] - if !stmt.mc.strict { - return columnCount, nil - } - - // Check for warnings count > 0, only available in MySQL > 4.1 - if len(data) >= 12 && binary.LittleEndian.Uint16(data[10:12]) > 0 { - return columnCount, stmt.mc.getWarnings() - } - return columnCount, nil - } - return 0, err -} - -// http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html -func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { - maxLen := stmt.mc.maxPacketAllowed - 1 - pktLen := maxLen - - // After the header (bytes 0-3) follows before the data: - // 1 byte command - // 4 bytes stmtID - // 2 bytes paramID - const dataOffset = 1 + 4 + 2 - - // Can not use the write buffer since - // a) the buffer is too small - // b) it is in use - data := make([]byte, 4+1+4+2+len(arg)) - - copy(data[4+dataOffset:], arg) - - for argLen := len(arg); argLen > 0; argLen -= pktLen - dataOffset { - if dataOffset+argLen < maxLen { - pktLen = dataOffset + argLen - } - - stmt.mc.sequence = 0 - // Add command byte [1 byte] - data[4] = comStmtSendLongData - - // Add stmtID [32 bit] - data[5] = byte(stmt.id) - data[6] = byte(stmt.id >> 8) - data[7] = byte(stmt.id >> 16) - data[8] = byte(stmt.id >> 24) - - // Add paramID [16 bit] - data[9] = byte(paramID) - data[10] = byte(paramID >> 8) - - // Send CMD packet - err := stmt.mc.writePacket(data[:4+pktLen]) - if err == nil { - data = data[pktLen-dataOffset:] - continue - } - return err - - } - - // Reset Packet Sequence - stmt.mc.sequence = 0 - return nil -} - -// Execute Prepared Statement -// http://dev.mysql.com/doc/internals/en/com-stmt-execute.html -func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { - if len(args) != stmt.paramCount { - return fmt.Errorf( - "argument count mismatch (got: %d; has: %d)", - len(args), - stmt.paramCount, - ) - } - - const minPktLen = 4 + 1 + 4 + 1 + 4 - mc := stmt.mc - - // Reset packet-sequence - mc.sequence = 0 - - var data []byte - - if len(args) == 0 { - data = mc.buf.takeBuffer(minPktLen) - } else { - data = mc.buf.takeCompleteBuffer() - } - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // command [1 byte] - data[4] = comStmtExecute - - // statement_id [4 bytes] - data[5] = byte(stmt.id) - data[6] = byte(stmt.id >> 8) - data[7] = byte(stmt.id >> 16) - data[8] = byte(stmt.id >> 24) - - // flags (0: CURSOR_TYPE_NO_CURSOR) [1 byte] - data[9] = 0x00 - - // iteration_count (uint32(1)) [4 bytes] - data[10] = 0x01 - data[11] = 0x00 - data[12] = 0x00 - data[13] = 0x00 - - if len(args) > 0 { - pos := minPktLen - - var nullMask []byte - if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { - // buffer has to be extended but we don't know by how much so - // we depend on append after all data with known sizes fit. - // We stop at that because we deal with a lot of columns here - // which makes the required allocation size hard to guess. - tmp := make([]byte, pos+maskLen+typesLen) - copy(tmp[:pos], data[:pos]) - data = tmp - nullMask = data[pos : pos+maskLen] - pos += maskLen - } else { - nullMask = data[pos : pos+maskLen] - for i := 0; i < maskLen; i++ { - nullMask[i] = 0 - } - pos += maskLen - } - - // newParameterBoundFlag 1 [1 byte] - data[pos] = 0x01 - pos++ - - // type of each parameter [len(args)*2 bytes] - paramTypes := data[pos:] - pos += len(args) * 2 - - // value of each parameter [n bytes] - paramValues := data[pos:pos] - valuesCap := cap(paramValues) - - for i, arg := range args { - // build NULL-bitmap - if arg == nil { - nullMask[i/8] |= 1 << (uint(i) & 7) - paramTypes[i+i] = fieldTypeNULL - paramTypes[i+i+1] = 0x00 - continue - } - - // cache types and values - switch v := arg.(type) { - case int64: - paramTypes[i+i] = fieldTypeLongLong - paramTypes[i+i+1] = 0x00 - - if cap(paramValues)-len(paramValues)-8 >= 0 { - paramValues = paramValues[:len(paramValues)+8] - binary.LittleEndian.PutUint64( - paramValues[len(paramValues)-8:], - uint64(v), - ) - } else { - paramValues = append(paramValues, - uint64ToBytes(uint64(v))..., - ) - } - - case float64: - paramTypes[i+i] = fieldTypeDouble - paramTypes[i+i+1] = 0x00 - - if cap(paramValues)-len(paramValues)-8 >= 0 { - paramValues = paramValues[:len(paramValues)+8] - binary.LittleEndian.PutUint64( - paramValues[len(paramValues)-8:], - math.Float64bits(v), - ) - } else { - paramValues = append(paramValues, - uint64ToBytes(math.Float64bits(v))..., - ) - } - - case bool: - paramTypes[i+i] = fieldTypeTiny - paramTypes[i+i+1] = 0x00 - - if v { - paramValues = append(paramValues, 0x01) - } else { - paramValues = append(paramValues, 0x00) - } - - case []byte: - // Common case (non-nil value) first - if v != nil { - paramTypes[i+i] = fieldTypeString - paramTypes[i+i+1] = 0x00 - - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { - paramValues = appendLengthEncodedInteger(paramValues, - uint64(len(v)), - ) - paramValues = append(paramValues, v...) - } else { - if err := stmt.writeCommandLongData(i, v); err != nil { - return err - } - } - continue - } - - // Handle []byte(nil) as a NULL value - nullMask[i/8] |= 1 << (uint(i) & 7) - paramTypes[i+i] = fieldTypeNULL - paramTypes[i+i+1] = 0x00 - - case string: - paramTypes[i+i] = fieldTypeString - paramTypes[i+i+1] = 0x00 - - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { - paramValues = appendLengthEncodedInteger(paramValues, - uint64(len(v)), - ) - paramValues = append(paramValues, v...) - } else { - if err := stmt.writeCommandLongData(i, []byte(v)); err != nil { - return err - } - } - - case time.Time: - paramTypes[i+i] = fieldTypeString - paramTypes[i+i+1] = 0x00 - - var val []byte - if v.IsZero() { - val = []byte("0000-00-00") - } else { - val = []byte(v.In(mc.cfg.Loc).Format(timeFormat)) - } - - paramValues = appendLengthEncodedInteger(paramValues, - uint64(len(val)), - ) - paramValues = append(paramValues, val...) - - default: - return fmt.Errorf("can not convert type: %T", arg) - } - } - - // Check if param values exceeded the available buffer - // In that case we must build the data packet with the new values buffer - if valuesCap != cap(paramValues) { - data = append(data[:pos], paramValues...) - mc.buf.buf = data - } - - pos += len(paramValues) - data = data[:pos] - } - - return mc.writePacket(data) -} - -func (mc *mysqlConn) discardResults() error { - for mc.status&statusMoreResultsExists != 0 { - resLen, err := mc.readResultSetHeaderPacket() - if err != nil { - return err - } - if resLen > 0 { - // columns - if err := mc.readUntilEOF(); err != nil { - return err - } - // rows - if err := mc.readUntilEOF(); err != nil { - return err - } - } else { - mc.status &^= statusMoreResultsExists - } - } - return nil -} - -// http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html -func (rows *binaryRows) readRow(dest []driver.Value) error { - data, err := rows.mc.readPacket() - if err != nil { - return err - } - - // packet indicator [1 byte] - if data[0] != iOK { - // EOF Packet - if data[0] == iEOF && len(data) == 5 { - rows.mc.status = readStatus(data[3:]) - if err := rows.mc.discardResults(); err != nil { - return err - } - rows.mc = nil - return io.EOF - } - rows.mc = nil - - // Error otherwise - return rows.mc.handleErrorPacket(data) - } - - // NULL-bitmap, [(column-count + 7 + 2) / 8 bytes] - pos := 1 + (len(dest)+7+2)>>3 - nullMask := data[1:pos] - - for i := range dest { - // Field is NULL - // (byte >> bit-pos) % 2 == 1 - if ((nullMask[(i+2)>>3] >> uint((i+2)&7)) & 1) == 1 { - dest[i] = nil - continue - } - - // Convert to byte-coded string - switch rows.columns[i].fieldType { - case fieldTypeNULL: - dest[i] = nil - continue - - // Numeric Types - case fieldTypeTiny: - if rows.columns[i].flags&flagUnsigned != 0 { - dest[i] = int64(data[pos]) - } else { - dest[i] = int64(int8(data[pos])) - } - pos++ - continue - - case fieldTypeShort, fieldTypeYear: - if rows.columns[i].flags&flagUnsigned != 0 { - dest[i] = int64(binary.LittleEndian.Uint16(data[pos : pos+2])) - } else { - dest[i] = int64(int16(binary.LittleEndian.Uint16(data[pos : pos+2]))) - } - pos += 2 - continue - - case fieldTypeInt24, fieldTypeLong: - if rows.columns[i].flags&flagUnsigned != 0 { - dest[i] = int64(binary.LittleEndian.Uint32(data[pos : pos+4])) - } else { - dest[i] = int64(int32(binary.LittleEndian.Uint32(data[pos : pos+4]))) - } - pos += 4 - continue - - case fieldTypeLongLong: - if rows.columns[i].flags&flagUnsigned != 0 { - val := binary.LittleEndian.Uint64(data[pos : pos+8]) - if val > math.MaxInt64 { - dest[i] = uint64ToString(val) - } else { - dest[i] = int64(val) - } - } else { - dest[i] = int64(binary.LittleEndian.Uint64(data[pos : pos+8])) - } - pos += 8 - continue - - case fieldTypeFloat: - dest[i] = float32(math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4]))) - pos += 4 - continue - - case fieldTypeDouble: - dest[i] = math.Float64frombits(binary.LittleEndian.Uint64(data[pos : pos+8])) - pos += 8 - continue - - // Length coded Binary Strings - case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, - fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, - fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON: - var isNull bool - var n int - dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) - pos += n - if err == nil { - if !isNull { - continue - } else { - dest[i] = nil - continue - } - } - return err - - case - fieldTypeDate, fieldTypeNewDate, // Date YYYY-MM-DD - fieldTypeTime, // Time [-][H]HH:MM:SS[.fractal] - fieldTypeTimestamp, fieldTypeDateTime: // Timestamp YYYY-MM-DD HH:MM:SS[.fractal] - - num, isNull, n := readLengthEncodedInteger(data[pos:]) - pos += n - - switch { - case isNull: - dest[i] = nil - continue - case rows.columns[i].fieldType == fieldTypeTime: - // database/sql does not support an equivalent to TIME, return a string - var dstlen uint8 - switch decimals := rows.columns[i].decimals; decimals { - case 0x00, 0x1f: - dstlen = 8 - case 1, 2, 3, 4, 5, 6: - dstlen = 8 + 1 + decimals - default: - return fmt.Errorf( - "protocol error, illegal decimals value %d", - rows.columns[i].decimals, - ) - } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, true) - case rows.mc.parseTime: - dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.Loc) - default: - var dstlen uint8 - if rows.columns[i].fieldType == fieldTypeDate { - dstlen = 10 - } else { - switch decimals := rows.columns[i].decimals; decimals { - case 0x00, 0x1f: - dstlen = 19 - case 1, 2, 3, 4, 5, 6: - dstlen = 19 + 1 + decimals - default: - return fmt.Errorf( - "protocol error, illegal decimals value %d", - rows.columns[i].decimals, - ) - } - } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, false) - } - - if err == nil { - pos += int(num) - continue - } else { - return err - } - - // Please report if this happens! - default: - return fmt.Errorf("unknown field type %d", rows.columns[i].fieldType) - } - } - - return nil -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/result.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/result.go deleted file mode 100644 index c6438d0..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/result.go +++ /dev/null @@ -1,22 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -type mysqlResult struct { - affectedRows int64 - insertId int64 -} - -func (res *mysqlResult) LastInsertId() (int64, error) { - return res.insertId, nil -} - -func (res *mysqlResult) RowsAffected() (int64, error) { - return res.affectedRows, nil -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/rows.go deleted file mode 100644 index c08255e..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/rows.go +++ /dev/null @@ -1,112 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "database/sql/driver" - "io" -) - -type mysqlField struct { - tableName string - name string - flags fieldFlag - fieldType byte - decimals byte -} - -type mysqlRows struct { - mc *mysqlConn - columns []mysqlField -} - -type binaryRows struct { - mysqlRows -} - -type textRows struct { - mysqlRows -} - -type emptyRows struct{} - -func (rows *mysqlRows) Columns() []string { - columns := make([]string, len(rows.columns)) - if rows.mc != nil && rows.mc.cfg.ColumnsWithAlias { - for i := range columns { - if tableName := rows.columns[i].tableName; len(tableName) > 0 { - columns[i] = tableName + "." + rows.columns[i].name - } else { - columns[i] = rows.columns[i].name - } - } - } else { - for i := range columns { - columns[i] = rows.columns[i].name - } - } - return columns -} - -func (rows *mysqlRows) Close() error { - mc := rows.mc - if mc == nil { - return nil - } - if mc.netConn == nil { - return ErrInvalidConn - } - - // Remove unread packets from stream - err := mc.readUntilEOF() - if err == nil { - if err = mc.discardResults(); err != nil { - return err - } - } - - rows.mc = nil - return err -} - -func (rows *binaryRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - if mc.netConn == nil { - return ErrInvalidConn - } - - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (rows *textRows) Next(dest []driver.Value) error { - if mc := rows.mc; mc != nil { - if mc.netConn == nil { - return ErrInvalidConn - } - - // Fetch next row from stream - return rows.readRow(dest) - } - return io.EOF -} - -func (rows emptyRows) Columns() []string { - return nil -} - -func (rows emptyRows) Close() error { - return nil -} - -func (rows emptyRows) Next(dest []driver.Value) error { - return io.EOF -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/statement.go deleted file mode 100644 index ead9a6b..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/statement.go +++ /dev/null @@ -1,150 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "database/sql/driver" - "fmt" - "reflect" - "strconv" -) - -type mysqlStmt struct { - mc *mysqlConn - id uint32 - paramCount int - columns []mysqlField // cached from the first query -} - -func (stmt *mysqlStmt) Close() error { - if stmt.mc == nil || stmt.mc.netConn == nil { - errLog.Print(ErrInvalidConn) - return driver.ErrBadConn - } - - err := stmt.mc.writeCommandPacketUint32(comStmtClose, stmt.id) - stmt.mc = nil - return err -} - -func (stmt *mysqlStmt) NumInput() int { - return stmt.paramCount -} - -func (stmt *mysqlStmt) ColumnConverter(idx int) driver.ValueConverter { - return converter{} -} - -func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { - if stmt.mc.netConn == nil { - errLog.Print(ErrInvalidConn) - return nil, driver.ErrBadConn - } - // Send command - err := stmt.writeExecutePacket(args) - if err != nil { - return nil, err - } - - mc := stmt.mc - - mc.affectedRows = 0 - mc.insertId = 0 - - // Read Result - resLen, err := mc.readResultSetHeaderPacket() - if err == nil { - if resLen > 0 { - // Columns - err = mc.readUntilEOF() - if err != nil { - return nil, err - } - - // Rows - err = mc.readUntilEOF() - } - if err == nil { - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, nil - } - } - - return nil, err -} - -func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { - if stmt.mc.netConn == nil { - errLog.Print(ErrInvalidConn) - return nil, driver.ErrBadConn - } - // Send command - err := stmt.writeExecutePacket(args) - if err != nil { - return nil, err - } - - mc := stmt.mc - - // Read Result - resLen, err := mc.readResultSetHeaderPacket() - if err != nil { - return nil, err - } - - rows := new(binaryRows) - - if resLen > 0 { - rows.mc = mc - // Columns - // If not cached, read them and cache them - if stmt.columns == nil { - rows.columns, err = mc.readColumns(resLen) - stmt.columns = rows.columns - } else { - rows.columns = stmt.columns - err = mc.readUntilEOF() - } - } - - return rows, err -} - -type converter struct{} - -func (c converter) ConvertValue(v interface{}) (driver.Value, error) { - if driver.IsValue(v) { - return v, nil - } - - rv := reflect.ValueOf(v) - switch rv.Kind() { - case reflect.Ptr: - // indirect pointers - if rv.IsNil() { - return nil, nil - } - return c.ConvertValue(rv.Elem().Interface()) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil - case reflect.Float32, reflect.Float64: - return rv.Float(), nil - } - return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/transaction.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/transaction.go deleted file mode 100644 index 33c749b..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/transaction.go +++ /dev/null @@ -1,31 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -type mysqlTx struct { - mc *mysqlConn -} - -func (tx *mysqlTx) Commit() (err error) { - if tx.mc == nil || tx.mc.netConn == nil { - return ErrInvalidConn - } - err = tx.mc.exec("COMMIT") - tx.mc = nil - return -} - -func (tx *mysqlTx) Rollback() (err error) { - if tx.mc == nil || tx.mc.netConn == nil { - return ErrInvalidConn - } - err = tx.mc.exec("ROLLBACK") - tx.mc = nil - return -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/utils.go deleted file mode 100644 index d523b7f..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/go-sql-driver/mysql/utils.go +++ /dev/null @@ -1,740 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package mysql - -import ( - "crypto/sha1" - "crypto/tls" - "database/sql/driver" - "encoding/binary" - "fmt" - "io" - "strings" - "time" -) - -var ( - tlsConfigRegister map[string]*tls.Config // Register for custom tls.Configs -) - -// RegisterTLSConfig registers a custom tls.Config to be used with sql.Open. -// Use the key as a value in the DSN where tls=value. -// -// rootCertPool := x509.NewCertPool() -// pem, err := ioutil.ReadFile("/path/ca-cert.pem") -// if err != nil { -// log.Fatal(err) -// } -// if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { -// log.Fatal("Failed to append PEM.") -// } -// clientCert := make([]tls.Certificate, 0, 1) -// certs, err := tls.LoadX509KeyPair("/path/client-cert.pem", "/path/client-key.pem") -// if err != nil { -// log.Fatal(err) -// } -// clientCert = append(clientCert, certs) -// mysql.RegisterTLSConfig("custom", &tls.Config{ -// RootCAs: rootCertPool, -// Certificates: clientCert, -// }) -// db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") -// -func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { - return fmt.Errorf("key '%s' is reserved", key) - } - - if tlsConfigRegister == nil { - tlsConfigRegister = make(map[string]*tls.Config) - } - - tlsConfigRegister[key] = config - return nil -} - -// DeregisterTLSConfig removes the tls.Config associated with key. -func DeregisterTLSConfig(key string) { - if tlsConfigRegister != nil { - delete(tlsConfigRegister, key) - } -} - -// Returns the bool value of the input. -// The 2nd return value indicates if the input was a valid bool value -func readBool(input string) (value bool, valid bool) { - switch input { - case "1", "true", "TRUE", "True": - return true, true - case "0", "false", "FALSE", "False": - return false, true - } - - // Not a valid bool value - return -} - -/****************************************************************************** -* Authentication * -******************************************************************************/ - -// Encrypt password using 4.1+ method -func scramblePassword(scramble, password []byte) []byte { - if len(password) == 0 { - return nil - } - - // stage1Hash = SHA1(password) - crypt := sha1.New() - crypt.Write(password) - stage1 := crypt.Sum(nil) - - // scrambleHash = SHA1(scramble + SHA1(stage1Hash)) - // inner Hash - crypt.Reset() - crypt.Write(stage1) - hash := crypt.Sum(nil) - - // outer Hash - crypt.Reset() - crypt.Write(scramble) - crypt.Write(hash) - scramble = crypt.Sum(nil) - - // token = scrambleHash XOR stage1Hash - for i := range scramble { - scramble[i] ^= stage1[i] - } - return scramble -} - -// Encrypt password using pre 4.1 (old password) method -// https://github.com/atcurtis/mariadb/blob/master/mysys/my_rnd.c -type myRnd struct { - seed1, seed2 uint32 -} - -const myRndMaxVal = 0x3FFFFFFF - -// Pseudo random number generator -func newMyRnd(seed1, seed2 uint32) *myRnd { - return &myRnd{ - seed1: seed1 % myRndMaxVal, - seed2: seed2 % myRndMaxVal, - } -} - -// Tested to be equivalent to MariaDB's floating point variant -// http://play.golang.org/p/QHvhd4qved -// http://play.golang.org/p/RG0q4ElWDx -func (r *myRnd) NextByte() byte { - r.seed1 = (r.seed1*3 + r.seed2) % myRndMaxVal - r.seed2 = (r.seed1 + r.seed2 + 33) % myRndMaxVal - - return byte(uint64(r.seed1) * 31 / myRndMaxVal) -} - -// Generate binary hash from byte string using insecure pre 4.1 method -func pwHash(password []byte) (result [2]uint32) { - var add uint32 = 7 - var tmp uint32 - - result[0] = 1345345333 - result[1] = 0x12345671 - - for _, c := range password { - // skip spaces and tabs in password - if c == ' ' || c == '\t' { - continue - } - - tmp = uint32(c) - result[0] ^= (((result[0] & 63) + add) * tmp) + (result[0] << 8) - result[1] += (result[1] << 8) ^ result[0] - add += tmp - } - - // Remove sign bit (1<<31)-1) - result[0] &= 0x7FFFFFFF - result[1] &= 0x7FFFFFFF - - return -} - -// Encrypt password using insecure pre 4.1 method -func scrambleOldPassword(scramble, password []byte) []byte { - if len(password) == 0 { - return nil - } - - scramble = scramble[:8] - - hashPw := pwHash(password) - hashSc := pwHash(scramble) - - r := newMyRnd(hashPw[0]^hashSc[0], hashPw[1]^hashSc[1]) - - var out [8]byte - for i := range out { - out[i] = r.NextByte() + 64 - } - - mask := r.NextByte() - for i := range out { - out[i] ^= mask - } - - return out[:] -} - -/****************************************************************************** -* Time related utils * -******************************************************************************/ - -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - -func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { - base := "0000-00-00 00:00:00.0000000" - switch len(str) { - case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM" - if str == base[:len(str)] { - return - } - t, err = time.Parse(timeFormat[:len(str)], str) - default: - err = fmt.Errorf("invalid time string: %s", str) - return - } - - // Adjust location - if err == nil && loc != time.UTC { - y, mo, d := t.Date() - h, mi, s := t.Clock() - t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil - } - - return -} - -func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Value, error) { - switch num { - case 0: - return time.Time{}, nil - case 4: - return time.Date( - int(binary.LittleEndian.Uint16(data[:2])), // year - time.Month(data[2]), // month - int(data[3]), // day - 0, 0, 0, 0, - loc, - ), nil - case 7: - return time.Date( - int(binary.LittleEndian.Uint16(data[:2])), // year - time.Month(data[2]), // month - int(data[3]), // day - int(data[4]), // hour - int(data[5]), // minutes - int(data[6]), // seconds - 0, - loc, - ), nil - case 11: - return time.Date( - int(binary.LittleEndian.Uint16(data[:2])), // year - time.Month(data[2]), // month - int(data[3]), // day - int(data[4]), // hour - int(data[5]), // minutes - int(data[6]), // seconds - int(binary.LittleEndian.Uint32(data[7:11]))*1000, // nanoseconds - loc, - ), nil - } - return nil, fmt.Errorf("invalid DATETIME packet length %d", num) -} - -// zeroDateTime is used in formatBinaryDateTime to avoid an allocation -// if the DATE or DATETIME has the zero value. -// It must never be changed. -// The current behavior depends on database/sql copying the result. -var zeroDateTime = []byte("0000-00-00 00:00:00.000000") - -const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" -const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" - -func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value, error) { - // length expects the deterministic length of the zero value, - // negative time and 100+ hours are automatically added if needed - if len(src) == 0 { - if justTime { - return zeroDateTime[11 : 11+length], nil - } - return zeroDateTime[:length], nil - } - var dst []byte // return value - var pt, p1, p2, p3 byte // current digit pair - var zOffs byte // offset of value in zeroDateTime - if justTime { - switch length { - case - 8, // time (can be up to 10 when negative and 100+ hours) - 10, 11, 12, 13, 14, 15: // time with fractional seconds - default: - return nil, fmt.Errorf("illegal TIME length %d", length) - } - switch len(src) { - case 8, 12: - default: - return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) - } - // +2 to enable negative time and 100+ hours - dst = make([]byte, 0, length+2) - if src[0] == 1 { - dst = append(dst, '-') - } - if src[1] != 0 { - hour := uint16(src[1])*24 + uint16(src[5]) - pt = byte(hour / 100) - p1 = byte(hour - 100*uint16(pt)) - dst = append(dst, digits01[pt]) - } else { - p1 = src[5] - } - zOffs = 11 - src = src[6:] - } else { - switch length { - case 10, 19, 21, 22, 23, 24, 25, 26: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s length %d", t, length) - } - switch len(src) { - case 4, 7, 11: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) - } - dst = make([]byte, 0, length) - // start with the date - year := binary.LittleEndian.Uint16(src[:2]) - pt = byte(year / 100) - p1 = byte(year - 100*uint16(pt)) - p2, p3 = src[2], src[3] - dst = append(dst, - digits10[pt], digits01[pt], - digits10[p1], digits01[p1], '-', - digits10[p2], digits01[p2], '-', - digits10[p3], digits01[p3], - ) - if length == 10 { - return dst, nil - } - if len(src) == 4 { - return append(dst, zeroDateTime[10:length]...), nil - } - dst = append(dst, ' ') - p1 = src[4] // hour - src = src[5:] - } - // p1 is 2-digit hour, src is after hour - p2, p3 = src[0], src[1] - dst = append(dst, - digits10[p1], digits01[p1], ':', - digits10[p2], digits01[p2], ':', - digits10[p3], digits01[p3], - ) - if length <= byte(len(dst)) { - return dst, nil - } - src = src[2:] - if len(src) == 0 { - return append(dst, zeroDateTime[19:zOffs+length]...), nil - } - microsecs := binary.LittleEndian.Uint32(src[:4]) - p1 = byte(microsecs / 10000) - microsecs -= 10000 * uint32(p1) - p2 = byte(microsecs / 100) - microsecs -= 100 * uint32(p2) - p3 = byte(microsecs) - switch decimals := zOffs + length - 20; decimals { - default: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - digits10[p3], digits01[p3], - ), nil - case 1: - return append(dst, '.', - digits10[p1], - ), nil - case 2: - return append(dst, '.', - digits10[p1], digits01[p1], - ), nil - case 3: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], - ), nil - case 4: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - ), nil - case 5: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - digits10[p3], - ), nil - } -} - -/****************************************************************************** -* Convert from and to bytes * -******************************************************************************/ - -func uint64ToBytes(n uint64) []byte { - return []byte{ - byte(n), - byte(n >> 8), - byte(n >> 16), - byte(n >> 24), - byte(n >> 32), - byte(n >> 40), - byte(n >> 48), - byte(n >> 56), - } -} - -func uint64ToString(n uint64) []byte { - var a [20]byte - i := 20 - - // U+0030 = 0 - // ... - // U+0039 = 9 - - var q uint64 - for n >= 10 { - i-- - q = n / 10 - a[i] = uint8(n-q*10) + 0x30 - n = q - } - - i-- - a[i] = uint8(n) + 0x30 - - return a[i:] -} - -// treats string value as unsigned integer representation -func stringToInt(b []byte) int { - val := 0 - for i := range b { - val *= 10 - val += int(b[i] - 0x30) - } - return val -} - -// returns the string read as a bytes slice, wheter the value is NULL, -// the number of bytes read and an error, in case the string is longer than -// the input slice -func readLengthEncodedString(b []byte) ([]byte, bool, int, error) { - // Get length - num, isNull, n := readLengthEncodedInteger(b) - if num < 1 { - return b[n:n], isNull, n, nil - } - - n += int(num) - - // Check data length - if len(b) >= n { - return b[n-int(num) : n], false, n, nil - } - return nil, false, n, io.EOF -} - -// returns the number of bytes skipped and an error, in case the string is -// longer than the input slice -func skipLengthEncodedString(b []byte) (int, error) { - // Get length - num, _, n := readLengthEncodedInteger(b) - if num < 1 { - return n, nil - } - - n += int(num) - - // Check data length - if len(b) >= n { - return n, nil - } - return n, io.EOF -} - -// returns the number read, whether the value is NULL and the number of bytes read -func readLengthEncodedInteger(b []byte) (uint64, bool, int) { - // See issue #349 - if len(b) == 0 { - return 0, true, 1 - } - switch b[0] { - - // 251: NULL - case 0xfb: - return 0, true, 1 - - // 252: value of following 2 - case 0xfc: - return uint64(b[1]) | uint64(b[2])<<8, false, 3 - - // 253: value of following 3 - case 0xfd: - return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16, false, 4 - - // 254: value of following 8 - case 0xfe: - return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16 | - uint64(b[4])<<24 | uint64(b[5])<<32 | uint64(b[6])<<40 | - uint64(b[7])<<48 | uint64(b[8])<<56, - false, 9 - } - - // 0-250: value of first byte - return uint64(b[0]), false, 1 -} - -// encodes a uint64 value and appends it to the given bytes slice -func appendLengthEncodedInteger(b []byte, n uint64) []byte { - switch { - case n <= 250: - return append(b, byte(n)) - - case n <= 0xffff: - return append(b, 0xfc, byte(n), byte(n>>8)) - - case n <= 0xffffff: - return append(b, 0xfd, byte(n), byte(n>>8), byte(n>>16)) - } - return append(b, 0xfe, byte(n), byte(n>>8), byte(n>>16), byte(n>>24), - byte(n>>32), byte(n>>40), byte(n>>48), byte(n>>56)) -} - -// reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. -// If cap(buf) is not enough, reallocate new buffer. -func reserveBuffer(buf []byte, appendSize int) []byte { - newSize := len(buf) + appendSize - if cap(buf) < newSize { - // Grow buffer exponentially - newBuf := make([]byte, len(buf)*2+appendSize) - copy(newBuf, buf) - buf = newBuf - } - return buf[:newSize] -} - -// escapeBytesBackslash escapes []byte with backslashes (\) -// This escapes the contents of a string (provided as []byte) by adding backslashes before special -// characters, and turning others into specific escape sequences, such as -// turning newlines into \n and null bytes into \0. -// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L823-L932 -func escapeBytesBackslash(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for _, c := range v { - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - case '\'': - buf[pos] = '\\' - buf[pos+1] = '\'' - pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeStringBackslash is similar to escapeBytesBackslash but for string. -func escapeStringBackslash(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - switch c { - case '\x00': - buf[pos] = '\\' - buf[pos+1] = '0' - pos += 2 - case '\n': - buf[pos] = '\\' - buf[pos+1] = 'n' - pos += 2 - case '\r': - buf[pos] = '\\' - buf[pos+1] = 'r' - pos += 2 - case '\x1a': - buf[pos] = '\\' - buf[pos+1] = 'Z' - pos += 2 - case '\'': - buf[pos] = '\\' - buf[pos+1] = '\'' - pos += 2 - case '"': - buf[pos] = '\\' - buf[pos+1] = '"' - pos += 2 - case '\\': - buf[pos] = '\\' - buf[pos+1] = '\\' - pos += 2 - default: - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeBytesQuotes escapes apostrophes in []byte by doubling them up. -// This escapes the contents of a string by doubling up any apostrophes that -// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in -// effect on the server. -// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L963-L1038 -func escapeBytesQuotes(buf, v []byte) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for _, c := range v { - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} - -// escapeStringQuotes is similar to escapeBytesQuotes but for string. -func escapeStringQuotes(buf []byte, v string) []byte { - pos := len(buf) - buf = reserveBuffer(buf, len(v)*2) - - for i := 0; i < len(v); i++ { - c := v[i] - if c == '\'' { - buf[pos] = '\'' - buf[pos+1] = '\'' - pos += 2 - } else { - buf[pos] = c - pos++ - } - } - - return buf[:pos] -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/LICENSE b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/LICENSE deleted file mode 100644 index 0d31edf..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/LICENSE +++ /dev/null @@ -1,23 +0,0 @@ - Copyright (c) 2013, Jason Moiron - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/bind.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/bind.go deleted file mode 100644 index 564635c..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/bind.go +++ /dev/null @@ -1,186 +0,0 @@ -package sqlx - -import ( - "bytes" - "errors" - "reflect" - "strconv" - "strings" - - "github.com/jmoiron/sqlx/reflectx" -) - -// Bindvar types supported by Rebind, BindMap and BindStruct. -const ( - UNKNOWN = iota - QUESTION - DOLLAR - NAMED -) - -// BindType returns the bindtype for a given database given a drivername. -func BindType(driverName string) int { - switch driverName { - case "postgres", "pgx": - return DOLLAR - case "mysql": - return QUESTION - case "sqlite3": - return QUESTION - case "oci8": - return NAMED - } - return UNKNOWN -} - -// FIXME: this should be able to be tolerant of escaped ?'s in queries without -// losing much speed, and should be to avoid confusion. - -// Rebind a query from the default bindtype (QUESTION) to the target bindtype. -func Rebind(bindType int, query string) string { - switch bindType { - case QUESTION, UNKNOWN: - return query - } - - qb := []byte(query) - // Add space enough for 10 params before we have to allocate - rqb := make([]byte, 0, len(qb)+10) - j := 1 - for _, b := range qb { - if b == '?' { - switch bindType { - case DOLLAR: - rqb = append(rqb, '$') - case NAMED: - rqb = append(rqb, ':', 'a', 'r', 'g') - } - for _, b := range strconv.Itoa(j) { - rqb = append(rqb, byte(b)) - } - j++ - } else { - rqb = append(rqb, b) - } - } - return string(rqb) -} - -// Experimental implementation of Rebind which uses a bytes.Buffer. The code is -// much simpler and should be more resistant to odd unicode, but it is twice as -// slow. Kept here for benchmarking purposes and to possibly replace Rebind if -// problems arise with its somewhat naive handling of unicode. -func rebindBuff(bindType int, query string) string { - if bindType != DOLLAR { - return query - } - - b := make([]byte, 0, len(query)) - rqb := bytes.NewBuffer(b) - j := 1 - for _, r := range query { - if r == '?' { - rqb.WriteRune('$') - rqb.WriteString(strconv.Itoa(j)) - j++ - } else { - rqb.WriteRune(r) - } - } - - return rqb.String() -} - -// In expands slice values in args, returning the modified query string -// and a new arg list that can be executed by a database. The `query` should -// use the `?` bindVar. The return value uses the `?` bindVar. -func In(query string, args ...interface{}) (string, []interface{}, error) { - // argMeta stores reflect.Value and length for slices and - // the value itself for non-slice arguments - type argMeta struct { - v reflect.Value - i interface{} - length int - } - - var flatArgsCount int - var anySlices bool - - meta := make([]argMeta, len(args)) - - for i, arg := range args { - v := reflect.ValueOf(arg) - t := reflectx.Deref(v.Type()) - - if t.Kind() == reflect.Slice { - meta[i].length = v.Len() - meta[i].v = v - - anySlices = true - flatArgsCount += meta[i].length - - if meta[i].length == 0 { - return "", nil, errors.New("empty slice passed to 'in' query") - } - } else { - meta[i].i = arg - flatArgsCount++ - } - } - - // don't do any parsing if there aren't any slices; note that this means - // some errors that we might have caught below will not be returned. - if !anySlices { - return query, args, nil - } - - newArgs := make([]interface{}, 0, flatArgsCount) - - var arg, offset int - var buf bytes.Buffer - - for i := strings.IndexByte(query[offset:], '?'); i != -1; i = strings.IndexByte(query[offset:], '?') { - if arg >= len(meta) { - // if an argument wasn't passed, lets return an error; this is - // not actually how database/sql Exec/Query works, but since we are - // creating an argument list programmatically, we want to be able - // to catch these programmer errors earlier. - return "", nil, errors.New("number of bindVars exceeds arguments") - } - - argMeta := meta[arg] - arg++ - - // not a slice, continue. - // our questionmark will either be written before the next expansion - // of a slice or after the loop when writing the rest of the query - if argMeta.length == 0 { - offset = offset + i + 1 - newArgs = append(newArgs, argMeta.i) - continue - } - - // write everything up to and including our ? character - buf.WriteString(query[:offset+i+1]) - - newArgs = append(newArgs, argMeta.v.Index(0).Interface()) - - for si := 1; si < argMeta.length; si++ { - buf.WriteString(", ?") - newArgs = append(newArgs, argMeta.v.Index(si).Interface()) - } - - // slice the query and reset the offset. this avoids some bookkeeping for - // the write after the loop - query = query[offset+i+1:] - offset = 0 - } - - buf.WriteString(query) - - if arg < len(meta) { - return "", nil, errors.New("number of bindVars less than number arguments") - } - - return buf.String(), newArgs, nil -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/doc.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/doc.go deleted file mode 100644 index e2b4e60..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Package sqlx provides general purpose extensions to database/sql. -// -// It is intended to seamlessly wrap database/sql and provide convenience -// methods which are useful in the development of database driven applications. -// None of the underlying database/sql methods are changed. Instead all extended -// behavior is implemented through new methods defined on wrapper types. -// -// Additions include scanning into structs, named query support, rebinding -// queries for different drivers, convenient shorthands for common error handling -// and more. -// -package sqlx diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/named.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/named.go deleted file mode 100644 index 4df8095..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/named.go +++ /dev/null @@ -1,336 +0,0 @@ -package sqlx - -// Named Query Support -// -// * BindMap - bind query bindvars to map/struct args -// * NamedExec, NamedQuery - named query w/ struct or map -// * NamedStmt - a pre-compiled named query which is a prepared statement -// -// Internal Interfaces: -// -// * compileNamedQuery - rebind a named query, returning a query and list of names -// * bindArgs, bindMapArgs, bindAnyArgs - given a list of names, return an arglist -// -import ( - "database/sql" - "errors" - "fmt" - "reflect" - "strconv" - "unicode" - - "github.com/jmoiron/sqlx/reflectx" -) - -// NamedStmt is a prepared statement that executes named queries. Prepare it -// how you would execute a NamedQuery, but pass in a struct or map when executing. -type NamedStmt struct { - Params []string - QueryString string - Stmt *Stmt -} - -// Close closes the named statement. -func (n *NamedStmt) Close() error { - return n.Stmt.Close() -} - -// Exec executes a named statement using the struct passed. -func (n *NamedStmt) Exec(arg interface{}) (sql.Result, error) { - args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) - if err != nil { - return *new(sql.Result), err - } - return n.Stmt.Exec(args...) -} - -// Query executes a named statement using the struct argument, returning rows. -func (n *NamedStmt) Query(arg interface{}) (*sql.Rows, error) { - args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) - if err != nil { - return nil, err - } - return n.Stmt.Query(args...) -} - -// QueryRow executes a named statement against the database. Because sqlx cannot -// create a *sql.Row with an error condition pre-set for binding errors, sqlx -// returns a *sqlx.Row instead. -func (n *NamedStmt) QueryRow(arg interface{}) *Row { - args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) - if err != nil { - return &Row{err: err} - } - return n.Stmt.QueryRowx(args...) -} - -// MustExec execs a NamedStmt, panicing on error -func (n *NamedStmt) MustExec(arg interface{}) sql.Result { - res, err := n.Exec(arg) - if err != nil { - panic(err) - } - return res -} - -// Queryx using this NamedStmt -func (n *NamedStmt) Queryx(arg interface{}) (*Rows, error) { - r, err := n.Query(arg) - if err != nil { - return nil, err - } - return &Rows{Rows: r, Mapper: n.Stmt.Mapper, unsafe: isUnsafe(n)}, err -} - -// QueryRowx this NamedStmt. Because of limitations with QueryRow, this is -// an alias for QueryRow. -func (n *NamedStmt) QueryRowx(arg interface{}) *Row { - return n.QueryRow(arg) -} - -// Select using this NamedStmt -func (n *NamedStmt) Select(dest interface{}, arg interface{}) error { - rows, err := n.Queryx(arg) - if err != nil { - return err - } - // if something happens here, we want to make sure the rows are Closed - defer rows.Close() - return scanAll(rows, dest, false) -} - -// Get using this NamedStmt -func (n *NamedStmt) Get(dest interface{}, arg interface{}) error { - r := n.QueryRowx(arg) - return r.scanAny(dest, false) -} - -// Unsafe creates an unsafe version of the NamedStmt -func (n *NamedStmt) Unsafe() *NamedStmt { - r := &NamedStmt{Params: n.Params, Stmt: n.Stmt, QueryString: n.QueryString} - r.Stmt.unsafe = true - return r -} - -// A union interface of preparer and binder, required to be able to prepare -// named statements (as the bindtype must be determined). -type namedPreparer interface { - Preparer - binder -} - -func prepareNamed(p namedPreparer, query string) (*NamedStmt, error) { - bindType := BindType(p.DriverName()) - q, args, err := compileNamedQuery([]byte(query), bindType) - if err != nil { - return nil, err - } - stmt, err := Preparex(p, q) - if err != nil { - return nil, err - } - return &NamedStmt{ - QueryString: q, - Params: args, - Stmt: stmt, - }, nil -} - -func bindAnyArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) { - if maparg, ok := arg.(map[string]interface{}); ok { - return bindMapArgs(names, maparg) - } - return bindArgs(names, arg, m) -} - -// private interface to generate a list of interfaces from a given struct -// type, given a list of names to pull out of the struct. Used by public -// BindStruct interface. -func bindArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) { - arglist := make([]interface{}, 0, len(names)) - - // grab the indirected value of arg - v := reflect.ValueOf(arg) - for v = reflect.ValueOf(arg); v.Kind() == reflect.Ptr; { - v = v.Elem() - } - - fields := m.TraversalsByName(v.Type(), names) - for i, t := range fields { - if len(t) == 0 { - return arglist, fmt.Errorf("could not find name %s in %#v", names[i], arg) - } - val := reflectx.FieldByIndexesReadOnly(v, t) - arglist = append(arglist, val.Interface()) - } - - return arglist, nil -} - -// like bindArgs, but for maps. -func bindMapArgs(names []string, arg map[string]interface{}) ([]interface{}, error) { - arglist := make([]interface{}, 0, len(names)) - - for _, name := range names { - val, ok := arg[name] - if !ok { - return arglist, fmt.Errorf("could not find name %s in %#v", name, arg) - } - arglist = append(arglist, val) - } - return arglist, nil -} - -// bindStruct binds a named parameter query with fields from a struct argument. -// The rules for binding field names to parameter names follow the same -// conventions as for StructScan, including obeying the `db` struct tags. -func bindStruct(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) { - bound, names, err := compileNamedQuery([]byte(query), bindType) - if err != nil { - return "", []interface{}{}, err - } - - arglist, err := bindArgs(names, arg, m) - if err != nil { - return "", []interface{}{}, err - } - - return bound, arglist, nil -} - -// bindMap binds a named parameter query with a map of arguments. -func bindMap(bindType int, query string, args map[string]interface{}) (string, []interface{}, error) { - bound, names, err := compileNamedQuery([]byte(query), bindType) - if err != nil { - return "", []interface{}{}, err - } - - arglist, err := bindMapArgs(names, args) - return bound, arglist, err -} - -// -- Compilation of Named Queries - -// Allow digits and letters in bind params; additionally runes are -// checked against underscores, meaning that bind params can have be -// alphanumeric with underscores. Mind the difference between unicode -// digits and numbers, where '5' is a digit but '五' is not. -var allowedBindRunes = []*unicode.RangeTable{unicode.Letter, unicode.Digit} - -// FIXME: this function isn't safe for unicode named params, as a failing test -// can testify. This is not a regression but a failure of the original code -// as well. It should be modified to range over runes in a string rather than -// bytes, even though this is less convenient and slower. Hopefully the -// addition of the prepared NamedStmt (which will only do this once) will make -// up for the slightly slower ad-hoc NamedExec/NamedQuery. - -// compile a NamedQuery into an unbound query (using the '?' bindvar) and -// a list of names. -func compileNamedQuery(qs []byte, bindType int) (query string, names []string, err error) { - names = make([]string, 0, 10) - rebound := make([]byte, 0, len(qs)) - - inName := false - last := len(qs) - 1 - currentVar := 1 - name := make([]byte, 0, 10) - - for i, b := range qs { - // a ':' while we're in a name is an error - if b == ':' { - // if this is the second ':' in a '::' escape sequence, append a ':' - if inName && i > 0 && qs[i-1] == ':' { - rebound = append(rebound, ':') - inName = false - continue - } else if inName { - err = errors.New("unexpected `:` while reading named param at " + strconv.Itoa(i)) - return query, names, err - } - inName = true - name = []byte{} - // if we're in a name, and this is an allowed character, continue - } else if inName && (unicode.IsOneOf(allowedBindRunes, rune(b)) || b == '_') && i != last { - // append the byte to the name if we are in a name and not on the last byte - name = append(name, b) - // if we're in a name and it's not an allowed character, the name is done - } else if inName { - inName = false - // if this is the final byte of the string and it is part of the name, then - // make sure to add it to the name - if i == last && unicode.IsOneOf(allowedBindRunes, rune(b)) { - name = append(name, b) - } - // add the string representation to the names list - names = append(names, string(name)) - // add a proper bindvar for the bindType - switch bindType { - // oracle only supports named type bind vars even for positional - case NAMED: - rebound = append(rebound, ':') - rebound = append(rebound, name...) - case QUESTION, UNKNOWN: - rebound = append(rebound, '?') - case DOLLAR: - rebound = append(rebound, '$') - for _, b := range strconv.Itoa(currentVar) { - rebound = append(rebound, byte(b)) - } - currentVar++ - } - // add this byte to string unless it was not part of the name - if i != last { - rebound = append(rebound, b) - } else if !unicode.IsOneOf(allowedBindRunes, rune(b)) { - rebound = append(rebound, b) - } - } else { - // this is a normal byte and should just go onto the rebound query - rebound = append(rebound, b) - } - } - - return string(rebound), names, err -} - -// BindNamed binds a struct or a map to a query with named parameters. -// DEPRECATED: use sqlx.Named` instead of this, it may be removed in future. -func BindNamed(bindType int, query string, arg interface{}) (string, []interface{}, error) { - return bindNamedMapper(bindType, query, arg, mapper()) -} - -// Named takes a query using named parameters and an argument and -// returns a new query with a list of args that can be executed by -// a database. The return value uses the `?` bindvar. -func Named(query string, arg interface{}) (string, []interface{}, error) { - return bindNamedMapper(QUESTION, query, arg, mapper()) -} - -func bindNamedMapper(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) { - if maparg, ok := arg.(map[string]interface{}); ok { - return bindMap(bindType, query, maparg) - } - return bindStruct(bindType, query, arg, m) -} - -// NamedQuery binds a named query and then runs Query on the result using the -// provided Ext (sqlx.Tx, sqlx.Db). It works with both structs and with -// map[string]interface{} types. -func NamedQuery(e Ext, query string, arg interface{}) (*Rows, error) { - q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e)) - if err != nil { - return nil, err - } - return e.Queryx(q, args...) -} - -// NamedExec uses BindStruct to get a query executable by the driver and -// then runs Exec on the result. Returns an error from the binding -// or the query excution itself. -func NamedExec(e Ext, query string, arg interface{}) (sql.Result, error) { - q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e)) - if err != nil { - return nil, err - } - return e.Exec(q, args...) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go deleted file mode 100644 index 5728011..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go +++ /dev/null @@ -1,371 +0,0 @@ -// Package reflectx implements extensions to the standard reflect lib suitable -// for implementing marshaling and unmarshaling packages. The main Mapper type -// allows for Go-compatible named atribute access, including accessing embedded -// struct attributes and the ability to use functions and struct tags to -// customize field names. -// -package reflectx - -import ( - "fmt" - "reflect" - "runtime" - "strings" - "sync" -) - -// A FieldInfo is a collection of metadata about a struct field. -type FieldInfo struct { - Index []int - Path string - Field reflect.StructField - Zero reflect.Value - Name string - Options map[string]string - Embedded bool - Children []*FieldInfo - Parent *FieldInfo -} - -// A StructMap is an index of field metadata for a struct. -type StructMap struct { - Tree *FieldInfo - Index []*FieldInfo - Paths map[string]*FieldInfo - Names map[string]*FieldInfo -} - -// GetByPath returns a *FieldInfo for a given string path. -func (f StructMap) GetByPath(path string) *FieldInfo { - return f.Paths[path] -} - -// GetByTraversal returns a *FieldInfo for a given integer path. It is -// analagous to reflect.FieldByIndex. -func (f StructMap) GetByTraversal(index []int) *FieldInfo { - if len(index) == 0 { - return nil - } - - tree := f.Tree - for _, i := range index { - if i >= len(tree.Children) || tree.Children[i] == nil { - return nil - } - tree = tree.Children[i] - } - return tree -} - -// Mapper is a general purpose mapper of names to struct fields. A Mapper -// behaves like most marshallers, optionally obeying a field tag for name -// mapping and a function to provide a basic mapping of fields to names. -type Mapper struct { - cache map[reflect.Type]*StructMap - tagName string - tagMapFunc func(string) string - mapFunc func(string) string - mutex sync.Mutex -} - -// NewMapper returns a new mapper which optionally obeys the field tag given -// by tagName. If tagName is the empty string, it is ignored. -func NewMapper(tagName string) *Mapper { - return &Mapper{ - cache: make(map[reflect.Type]*StructMap), - tagName: tagName, - } -} - -// NewMapperTagFunc returns a new mapper which contains a mapper for field names -// AND a mapper for tag values. This is useful for tags like json which can -// have values like "name,omitempty". -func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper { - return &Mapper{ - cache: make(map[reflect.Type]*StructMap), - tagName: tagName, - mapFunc: mapFunc, - tagMapFunc: tagMapFunc, - } -} - -// NewMapperFunc returns a new mapper which optionally obeys a field tag and -// a struct field name mapper func given by f. Tags will take precedence, but -// for any other field, the mapped name will be f(field.Name) -func NewMapperFunc(tagName string, f func(string) string) *Mapper { - return &Mapper{ - cache: make(map[reflect.Type]*StructMap), - tagName: tagName, - mapFunc: f, - } -} - -// TypeMap returns a mapping of field strings to int slices representing -// the traversal down the struct to reach the field. -func (m *Mapper) TypeMap(t reflect.Type) *StructMap { - m.mutex.Lock() - mapping, ok := m.cache[t] - if !ok { - mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc) - m.cache[t] = mapping - } - m.mutex.Unlock() - return mapping -} - -// FieldMap returns the mapper's mapping of field names to reflect values. Panics -// if v's Kind is not Struct, or v is not Indirectable to a struct kind. -func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value { - v = reflect.Indirect(v) - mustBe(v, reflect.Struct) - - r := map[string]reflect.Value{} - tm := m.TypeMap(v.Type()) - for tagName, fi := range tm.Names { - r[tagName] = FieldByIndexes(v, fi.Index) - } - return r -} - -// FieldByName returns a field by the its mapped name as a reflect.Value. -// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind. -// Returns zero Value if the name is not found. -func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value { - v = reflect.Indirect(v) - mustBe(v, reflect.Struct) - - tm := m.TypeMap(v.Type()) - fi, ok := tm.Names[name] - if !ok { - return v - } - return FieldByIndexes(v, fi.Index) -} - -// FieldsByName returns a slice of values corresponding to the slice of names -// for the value. Panics if v's Kind is not Struct or v is not Indirectable -// to a struct Kind. Returns zero Value for each name not found. -func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value { - v = reflect.Indirect(v) - mustBe(v, reflect.Struct) - - tm := m.TypeMap(v.Type()) - vals := make([]reflect.Value, 0, len(names)) - for _, name := range names { - fi, ok := tm.Names[name] - if !ok { - vals = append(vals, *new(reflect.Value)) - } else { - vals = append(vals, FieldByIndexes(v, fi.Index)) - } - } - return vals -} - -// TraversalsByName returns a slice of int slices which represent the struct -// traversals for each mapped name. Panics if t is not a struct or Indirectable -// to a struct. Returns empty int slice for each name not found. -func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int { - t = Deref(t) - mustBe(t, reflect.Struct) - tm := m.TypeMap(t) - - r := make([][]int, 0, len(names)) - for _, name := range names { - fi, ok := tm.Names[name] - if !ok { - r = append(r, []int{}) - } else { - r = append(r, fi.Index) - } - } - return r -} - -// FieldByIndexes returns a value for a particular struct traversal. -func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value { - for _, i := range indexes { - v = reflect.Indirect(v).Field(i) - // if this is a pointer, it's possible it is nil - if v.Kind() == reflect.Ptr && v.IsNil() { - alloc := reflect.New(Deref(v.Type())) - v.Set(alloc) - } - if v.Kind() == reflect.Map && v.IsNil() { - v.Set(reflect.MakeMap(v.Type())) - } - } - return v -} - -// FieldByIndexesReadOnly returns a value for a particular struct traversal, -// but is not concerned with allocating nil pointers because the value is -// going to be used for reading and not setting. -func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value { - for _, i := range indexes { - v = reflect.Indirect(v).Field(i) - } - return v -} - -// Deref is Indirect for reflect.Types -func Deref(t reflect.Type) reflect.Type { - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - return t -} - -// -- helpers & utilities -- - -type kinder interface { - Kind() reflect.Kind -} - -// mustBe checks a value against a kind, panicing with a reflect.ValueError -// if the kind isn't that which is required. -func mustBe(v kinder, expected reflect.Kind) { - k := v.Kind() - if k != expected { - panic(&reflect.ValueError{Method: methodName(), Kind: k}) - } -} - -// methodName is returns the caller of the function calling methodName -func methodName() string { - pc, _, _, _ := runtime.Caller(2) - f := runtime.FuncForPC(pc) - if f == nil { - return "unknown method" - } - return f.Name() -} - -type typeQueue struct { - t reflect.Type - fi *FieldInfo - pp string // Parent path -} - -// A copying append that creates a new slice each time. -func apnd(is []int, i int) []int { - x := make([]int, len(is)+1) - for p, n := range is { - x[p] = n - } - x[len(x)-1] = i - return x -} - -// getMapping returns a mapping for the t type, using the tagName, mapFunc and -// tagMapFunc to determine the canonical names of fields. -func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc func(string) string) *StructMap { - m := []*FieldInfo{} - - root := &FieldInfo{} - queue := []typeQueue{} - queue = append(queue, typeQueue{Deref(t), root, ""}) - - for len(queue) != 0 { - // pop the first item off of the queue - tq := queue[0] - queue = queue[1:] - nChildren := 0 - if tq.t.Kind() == reflect.Struct { - nChildren = tq.t.NumField() - } - tq.fi.Children = make([]*FieldInfo, nChildren) - - // iterate through all of its fields - for fieldPos := 0; fieldPos < nChildren; fieldPos++ { - f := tq.t.Field(fieldPos) - - fi := FieldInfo{} - fi.Field = f - fi.Zero = reflect.New(f.Type).Elem() - fi.Options = map[string]string{} - - var tag, name string - if tagName != "" && strings.Contains(string(f.Tag), tagName+":") { - tag = f.Tag.Get(tagName) - name = tag - } else { - if mapFunc != nil { - name = mapFunc(f.Name) - } - } - - parts := strings.Split(name, ",") - if len(parts) > 1 { - name = parts[0] - for _, opt := range parts[1:] { - kv := strings.Split(opt, "=") - if len(kv) > 1 { - fi.Options[kv[0]] = kv[1] - } else { - fi.Options[kv[0]] = "" - } - } - } - - if tagMapFunc != nil { - tag = tagMapFunc(tag) - } - - fi.Name = name - - if tq.pp == "" || (tq.pp == "" && tag == "") { - fi.Path = fi.Name - } else { - fi.Path = fmt.Sprintf("%s.%s", tq.pp, fi.Name) - } - - // if the name is "-", disabled via a tag, skip it - if name == "-" { - continue - } - - // skip unexported fields - if len(f.PkgPath) != 0 { - continue - } - - // bfs search of anonymous embedded structs - if f.Anonymous { - pp := tq.pp - if tag != "" { - pp = fi.Path - } - - fi.Embedded = true - fi.Index = apnd(tq.fi.Index, fieldPos) - nChildren := 0 - ft := Deref(f.Type) - if ft.Kind() == reflect.Struct { - nChildren = ft.NumField() - } - fi.Children = make([]*FieldInfo, nChildren) - queue = append(queue, typeQueue{Deref(f.Type), &fi, pp}) - } else if fi.Zero.Kind() == reflect.Struct || (fi.Zero.Kind() == reflect.Ptr && fi.Zero.Type().Elem().Kind() == reflect.Struct) { - fi.Index = apnd(tq.fi.Index, fieldPos) - fi.Children = make([]*FieldInfo, Deref(f.Type).NumField()) - queue = append(queue, typeQueue{Deref(f.Type), &fi, fi.Path}) - } - - fi.Index = apnd(tq.fi.Index, fieldPos) - fi.Parent = tq.fi - tq.fi.Children[fieldPos] = &fi - m = append(m, &fi) - } - } - - flds := &StructMap{Index: m, Tree: root, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}} - for _, fi := range flds.Index { - flds.Paths[fi.Path] = fi - if fi.Name != "" && !fi.Embedded { - flds.Names[fi.Path] = fi - } - } - - return flds -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/sqlx.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/sqlx.go deleted file mode 100644 index b1ba4cf..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/jmoiron/sqlx/sqlx.go +++ /dev/null @@ -1,992 +0,0 @@ -package sqlx - -import ( - "database/sql" - "database/sql/driver" - "errors" - "fmt" - - "io/ioutil" - "path/filepath" - "reflect" - "strings" - - "github.com/jmoiron/sqlx/reflectx" -) - -// Although the NameMapper is convenient, in practice it should not -// be relied on except for application code. If you are writing a library -// that uses sqlx, you should be aware that the name mappings you expect -// can be overridded by your user's application. - -// NameMapper is used to map column names to struct field names. By default, -// it uses strings.ToLower to lowercase struct field names. It can be set -// to whatever you want, but it is encouraged to be set before sqlx is used -// as name-to-field mappings are cached after first use on a type. -var NameMapper = strings.ToLower -var origMapper = reflect.ValueOf(NameMapper) - -// Rather than creating on init, this is created when necessary so that -// importers have time to customize the NameMapper. -var mpr *reflectx.Mapper - -// mapper returns a valid mapper using the configured NameMapper func. -func mapper() *reflectx.Mapper { - if mpr == nil { - mpr = reflectx.NewMapperFunc("db", NameMapper) - } else if origMapper != reflect.ValueOf(NameMapper) { - // if NameMapper has changed, create a new mapper - mpr = reflectx.NewMapperFunc("db", NameMapper) - origMapper = reflect.ValueOf(NameMapper) - } - return mpr -} - -// isScannable takes the reflect.Type and the actual dest value and returns -// whether or not it's Scannable. Something is scannable if: -// * it is not a struct -// * it implements sql.Scanner -// * it has no exported fields -func isScannable(t reflect.Type) bool { - if reflect.PtrTo(t).Implements(_scannerInterface) { - return true - } - if t.Kind() != reflect.Struct { - return true - } - - // it's not important that we use the right mapper for this particular object, - // we're only concerned on how many exported fields this struct has - m := mapper() - if len(m.TypeMap(t).Index) == 0 { - return true - } - return false -} - -// ColScanner is an interface used by MapScan and SliceScan -type ColScanner interface { - Columns() ([]string, error) - Scan(dest ...interface{}) error - Err() error -} - -// Queryer is an interface used by Get and Select -type Queryer interface { - Query(query string, args ...interface{}) (*sql.Rows, error) - Queryx(query string, args ...interface{}) (*Rows, error) - QueryRowx(query string, args ...interface{}) *Row -} - -// Execer is an interface used by MustExec and LoadFile -type Execer interface { - Exec(query string, args ...interface{}) (sql.Result, error) -} - -// Binder is an interface for something which can bind queries (Tx, DB) -type binder interface { - DriverName() string - Rebind(string) string - BindNamed(string, interface{}) (string, []interface{}, error) -} - -// Ext is a union interface which can bind, query, and exec, used by -// NamedQuery and NamedExec. -type Ext interface { - binder - Queryer - Execer -} - -// Preparer is an interface used by Preparex. -type Preparer interface { - Prepare(query string) (*sql.Stmt, error) -} - -// determine if any of our extensions are unsafe -func isUnsafe(i interface{}) bool { - switch v := i.(type) { - case Row: - return v.unsafe - case *Row: - return v.unsafe - case Rows: - return v.unsafe - case *Rows: - return v.unsafe - case NamedStmt: - return v.Stmt.unsafe - case *NamedStmt: - return v.Stmt.unsafe - case Stmt: - return v.unsafe - case *Stmt: - return v.unsafe - case qStmt: - return v.unsafe - case *qStmt: - return v.unsafe - case DB: - return v.unsafe - case *DB: - return v.unsafe - case Tx: - return v.unsafe - case *Tx: - return v.unsafe - case sql.Rows, *sql.Rows: - return false - default: - return false - } -} - -func mapperFor(i interface{}) *reflectx.Mapper { - switch i.(type) { - case DB: - return i.(DB).Mapper - case *DB: - return i.(*DB).Mapper - case Tx: - return i.(Tx).Mapper - case *Tx: - return i.(*Tx).Mapper - default: - return mapper() - } -} - -var _scannerInterface = reflect.TypeOf((*sql.Scanner)(nil)).Elem() -var _valuerInterface = reflect.TypeOf((*driver.Valuer)(nil)).Elem() - -// Row is a reimplementation of sql.Row in order to gain access to the underlying -// sql.Rows.Columns() data, necessary for StructScan. -type Row struct { - err error - unsafe bool - rows *sql.Rows - Mapper *reflectx.Mapper -} - -// Scan is a fixed implementation of sql.Row.Scan, which does not discard the -// underlying error from the internal rows object if it exists. -func (r *Row) Scan(dest ...interface{}) error { - if r.err != nil { - return r.err - } - - // TODO(bradfitz): for now we need to defensively clone all - // []byte that the driver returned (not permitting - // *RawBytes in Rows.Scan), since we're about to close - // the Rows in our defer, when we return from this function. - // the contract with the driver.Next(...) interface is that it - // can return slices into read-only temporary memory that's - // only valid until the next Scan/Close. But the TODO is that - // for a lot of drivers, this copy will be unnecessary. We - // should provide an optional interface for drivers to - // implement to say, "don't worry, the []bytes that I return - // from Next will not be modified again." (for instance, if - // they were obtained from the network anyway) But for now we - // don't care. - defer r.rows.Close() - for _, dp := range dest { - if _, ok := dp.(*sql.RawBytes); ok { - return errors.New("sql: RawBytes isn't allowed on Row.Scan") - } - } - - if !r.rows.Next() { - if err := r.rows.Err(); err != nil { - return err - } - return sql.ErrNoRows - } - err := r.rows.Scan(dest...) - if err != nil { - return err - } - // Make sure the query can be processed to completion with no errors. - if err := r.rows.Close(); err != nil { - return err - } - return nil -} - -// Columns returns the underlying sql.Rows.Columns(), or the deferred error usually -// returned by Row.Scan() -func (r *Row) Columns() ([]string, error) { - if r.err != nil { - return []string{}, r.err - } - return r.rows.Columns() -} - -// Err returns the error encountered while scanning. -func (r *Row) Err() error { - return r.err -} - -// DB is a wrapper around sql.DB which keeps track of the driverName upon Open, -// used mostly to automatically bind named queries using the right bindvars. -type DB struct { - *sql.DB - driverName string - unsafe bool - Mapper *reflectx.Mapper -} - -// NewDb returns a new sqlx DB wrapper for a pre-existing *sql.DB. The -// driverName of the original database is required for named query support. -func NewDb(db *sql.DB, driverName string) *DB { - return &DB{DB: db, driverName: driverName, Mapper: mapper()} -} - -// DriverName returns the driverName passed to the Open function for this DB. -func (db *DB) DriverName() string { - return db.driverName -} - -// Open is the same as sql.Open, but returns an *sqlx.DB instead. -func Open(driverName, dataSourceName string) (*DB, error) { - db, err := sql.Open(driverName, dataSourceName) - if err != nil { - return nil, err - } - return &DB{DB: db, driverName: driverName, Mapper: mapper()}, err -} - -// MustOpen is the same as sql.Open, but returns an *sqlx.DB instead and panics on error. -func MustOpen(driverName, dataSourceName string) *DB { - db, err := Open(driverName, dataSourceName) - if err != nil { - panic(err) - } - return db -} - -// MapperFunc sets a new mapper for this db using the default sqlx struct tag -// and the provided mapper function. -func (db *DB) MapperFunc(mf func(string) string) { - db.Mapper = reflectx.NewMapperFunc("db", mf) -} - -// Rebind transforms a query from QUESTION to the DB driver's bindvar type. -func (db *DB) Rebind(query string) string { - return Rebind(BindType(db.driverName), query) -} - -// Unsafe returns a version of DB which will silently succeed to scan when -// columns in the SQL result have no fields in the destination struct. -// sqlx.Stmt and sqlx.Tx which are created from this DB will inherit its -// safety behavior. -func (db *DB) Unsafe() *DB { - return &DB{DB: db.DB, driverName: db.driverName, unsafe: true, Mapper: db.Mapper} -} - -// BindNamed binds a query using the DB driver's bindvar type. -func (db *DB) BindNamed(query string, arg interface{}) (string, []interface{}, error) { - return bindNamedMapper(BindType(db.driverName), query, arg, db.Mapper) -} - -// NamedQuery using this DB. -func (db *DB) NamedQuery(query string, arg interface{}) (*Rows, error) { - return NamedQuery(db, query, arg) -} - -// NamedExec using this DB. -func (db *DB) NamedExec(query string, arg interface{}) (sql.Result, error) { - return NamedExec(db, query, arg) -} - -// Select using this DB. -func (db *DB) Select(dest interface{}, query string, args ...interface{}) error { - return Select(db, dest, query, args...) -} - -// Get using this DB. -func (db *DB) Get(dest interface{}, query string, args ...interface{}) error { - return Get(db, dest, query, args...) -} - -// MustBegin starts a transaction, and panics on error. Returns an *sqlx.Tx instead -// of an *sql.Tx. -func (db *DB) MustBegin() *Tx { - tx, err := db.Beginx() - if err != nil { - panic(err) - } - return tx -} - -// Beginx begins a transaction and returns an *sqlx.Tx instead of an *sql.Tx. -func (db *DB) Beginx() (*Tx, error) { - tx, err := db.DB.Begin() - if err != nil { - return nil, err - } - return &Tx{Tx: tx, driverName: db.driverName, unsafe: db.unsafe, Mapper: db.Mapper}, err -} - -// Queryx queries the database and returns an *sqlx.Rows. -func (db *DB) Queryx(query string, args ...interface{}) (*Rows, error) { - r, err := db.DB.Query(query, args...) - if err != nil { - return nil, err - } - return &Rows{Rows: r, unsafe: db.unsafe, Mapper: db.Mapper}, err -} - -// QueryRowx queries the database and returns an *sqlx.Row. -func (db *DB) QueryRowx(query string, args ...interface{}) *Row { - rows, err := db.DB.Query(query, args...) - return &Row{rows: rows, err: err, unsafe: db.unsafe, Mapper: db.Mapper} -} - -// MustExec (panic) runs MustExec using this database. -func (db *DB) MustExec(query string, args ...interface{}) sql.Result { - return MustExec(db, query, args...) -} - -// Preparex returns an sqlx.Stmt instead of a sql.Stmt -func (db *DB) Preparex(query string) (*Stmt, error) { - return Preparex(db, query) -} - -// PrepareNamed returns an sqlx.NamedStmt -func (db *DB) PrepareNamed(query string) (*NamedStmt, error) { - return prepareNamed(db, query) -} - -// Tx is an sqlx wrapper around sql.Tx with extra functionality -type Tx struct { - *sql.Tx - driverName string - unsafe bool - Mapper *reflectx.Mapper -} - -// DriverName returns the driverName used by the DB which began this transaction. -func (tx *Tx) DriverName() string { - return tx.driverName -} - -// Rebind a query within a transaction's bindvar type. -func (tx *Tx) Rebind(query string) string { - return Rebind(BindType(tx.driverName), query) -} - -// Unsafe returns a version of Tx which will silently succeed to scan when -// columns in the SQL result have no fields in the destination struct. -func (tx *Tx) Unsafe() *Tx { - return &Tx{Tx: tx.Tx, driverName: tx.driverName, unsafe: true, Mapper: tx.Mapper} -} - -// BindNamed binds a query within a transaction's bindvar type. -func (tx *Tx) BindNamed(query string, arg interface{}) (string, []interface{}, error) { - return bindNamedMapper(BindType(tx.driverName), query, arg, tx.Mapper) -} - -// NamedQuery within a transaction. -func (tx *Tx) NamedQuery(query string, arg interface{}) (*Rows, error) { - return NamedQuery(tx, query, arg) -} - -// NamedExec a named query within a transaction. -func (tx *Tx) NamedExec(query string, arg interface{}) (sql.Result, error) { - return NamedExec(tx, query, arg) -} - -// Select within a transaction. -func (tx *Tx) Select(dest interface{}, query string, args ...interface{}) error { - return Select(tx, dest, query, args...) -} - -// Queryx within a transaction. -func (tx *Tx) Queryx(query string, args ...interface{}) (*Rows, error) { - r, err := tx.Tx.Query(query, args...) - if err != nil { - return nil, err - } - return &Rows{Rows: r, unsafe: tx.unsafe, Mapper: tx.Mapper}, err -} - -// QueryRowx within a transaction. -func (tx *Tx) QueryRowx(query string, args ...interface{}) *Row { - rows, err := tx.Tx.Query(query, args...) - return &Row{rows: rows, err: err, unsafe: tx.unsafe, Mapper: tx.Mapper} -} - -// Get within a transaction. -func (tx *Tx) Get(dest interface{}, query string, args ...interface{}) error { - return Get(tx, dest, query, args...) -} - -// MustExec runs MustExec within a transaction. -func (tx *Tx) MustExec(query string, args ...interface{}) sql.Result { - return MustExec(tx, query, args...) -} - -// Preparex a statement within a transaction. -func (tx *Tx) Preparex(query string) (*Stmt, error) { - return Preparex(tx, query) -} - -// Stmtx returns a version of the prepared statement which runs within a transaction. Provided -// stmt can be either *sql.Stmt or *sqlx.Stmt. -func (tx *Tx) Stmtx(stmt interface{}) *Stmt { - var s *sql.Stmt - switch v := stmt.(type) { - case Stmt: - s = v.Stmt - case *Stmt: - s = v.Stmt - case sql.Stmt: - s = &v - case *sql.Stmt: - s = v - default: - panic(fmt.Sprintf("non-statement type %v passed to Stmtx", reflect.ValueOf(stmt).Type())) - } - return &Stmt{Stmt: tx.Stmt(s), Mapper: tx.Mapper} -} - -// NamedStmt returns a version of the prepared statement which runs within a transaction. -func (tx *Tx) NamedStmt(stmt *NamedStmt) *NamedStmt { - return &NamedStmt{ - QueryString: stmt.QueryString, - Params: stmt.Params, - Stmt: tx.Stmtx(stmt.Stmt), - } -} - -// PrepareNamed returns an sqlx.NamedStmt -func (tx *Tx) PrepareNamed(query string) (*NamedStmt, error) { - return prepareNamed(tx, query) -} - -// Stmt is an sqlx wrapper around sql.Stmt with extra functionality -type Stmt struct { - *sql.Stmt - unsafe bool - Mapper *reflectx.Mapper -} - -// Unsafe returns a version of Stmt which will silently succeed to scan when -// columns in the SQL result have no fields in the destination struct. -func (s *Stmt) Unsafe() *Stmt { - return &Stmt{Stmt: s.Stmt, unsafe: true, Mapper: s.Mapper} -} - -// Select using the prepared statement. -func (s *Stmt) Select(dest interface{}, args ...interface{}) error { - return Select(&qStmt{s}, dest, "", args...) -} - -// Get using the prepared statement. -func (s *Stmt) Get(dest interface{}, args ...interface{}) error { - return Get(&qStmt{s}, dest, "", args...) -} - -// MustExec (panic) using this statement. Note that the query portion of the error -// output will be blank, as Stmt does not expose its query. -func (s *Stmt) MustExec(args ...interface{}) sql.Result { - return MustExec(&qStmt{s}, "", args...) -} - -// QueryRowx using this statement. -func (s *Stmt) QueryRowx(args ...interface{}) *Row { - qs := &qStmt{s} - return qs.QueryRowx("", args...) -} - -// Queryx using this statement. -func (s *Stmt) Queryx(args ...interface{}) (*Rows, error) { - qs := &qStmt{s} - return qs.Queryx("", args...) -} - -// qStmt is an unexposed wrapper which lets you use a Stmt as a Queryer & Execer by -// implementing those interfaces and ignoring the `query` argument. -type qStmt struct{ *Stmt } - -func (q *qStmt) Query(query string, args ...interface{}) (*sql.Rows, error) { - return q.Stmt.Query(args...) -} - -func (q *qStmt) Queryx(query string, args ...interface{}) (*Rows, error) { - r, err := q.Stmt.Query(args...) - if err != nil { - return nil, err - } - return &Rows{Rows: r, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper}, err -} - -func (q *qStmt) QueryRowx(query string, args ...interface{}) *Row { - rows, err := q.Stmt.Query(args...) - return &Row{rows: rows, err: err, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper} -} - -func (q *qStmt) Exec(query string, args ...interface{}) (sql.Result, error) { - return q.Stmt.Exec(args...) -} - -// Rows is a wrapper around sql.Rows which caches costly reflect operations -// during a looped StructScan -type Rows struct { - *sql.Rows - unsafe bool - Mapper *reflectx.Mapper - // these fields cache memory use for a rows during iteration w/ structScan - started bool - fields [][]int - values []interface{} -} - -// SliceScan using this Rows. -func (r *Rows) SliceScan() ([]interface{}, error) { - return SliceScan(r) -} - -// MapScan using this Rows. -func (r *Rows) MapScan(dest map[string]interface{}) error { - return MapScan(r, dest) -} - -// StructScan is like sql.Rows.Scan, but scans a single Row into a single Struct. -// Use this and iterate over Rows manually when the memory load of Select() might be -// prohibitive. *Rows.StructScan caches the reflect work of matching up column -// positions to fields to avoid that overhead per scan, which means it is not safe -// to run StructScan on the same Rows instance with different struct types. -func (r *Rows) StructScan(dest interface{}) error { - v := reflect.ValueOf(dest) - - if v.Kind() != reflect.Ptr { - return errors.New("must pass a pointer, not a value, to StructScan destination") - } - - v = reflect.Indirect(v) - - if !r.started { - columns, err := r.Columns() - if err != nil { - return err - } - m := r.Mapper - - r.fields = m.TraversalsByName(v.Type(), columns) - // if we are not unsafe and are missing fields, return an error - if f, err := missingFields(r.fields); err != nil && !r.unsafe { - return fmt.Errorf("missing destination name %s", columns[f]) - } - r.values = make([]interface{}, len(columns)) - r.started = true - } - - err := fieldsByTraversal(v, r.fields, r.values, true) - if err != nil { - return err - } - // scan into the struct field pointers and append to our results - err = r.Scan(r.values...) - if err != nil { - return err - } - return r.Err() -} - -// Connect to a database and verify with a ping. -func Connect(driverName, dataSourceName string) (*DB, error) { - db, err := Open(driverName, dataSourceName) - if err != nil { - return db, err - } - err = db.Ping() - return db, err -} - -// MustConnect connects to a database and panics on error. -func MustConnect(driverName, dataSourceName string) *DB { - db, err := Connect(driverName, dataSourceName) - if err != nil { - panic(err) - } - return db -} - -// Preparex prepares a statement. -func Preparex(p Preparer, query string) (*Stmt, error) { - s, err := p.Prepare(query) - if err != nil { - return nil, err - } - return &Stmt{Stmt: s, unsafe: isUnsafe(p), Mapper: mapperFor(p)}, err -} - -// Select executes a query using the provided Queryer, and StructScans each row -// into dest, which must be a slice. If the slice elements are scannable, then -// the result set must have only one column. Otherwise, StructScan is used. -// The *sql.Rows are closed automatically. -func Select(q Queryer, dest interface{}, query string, args ...interface{}) error { - rows, err := q.Queryx(query, args...) - if err != nil { - return err - } - // if something happens here, we want to make sure the rows are Closed - defer rows.Close() - return scanAll(rows, dest, false) -} - -// Get does a QueryRow using the provided Queryer, and scans the resulting row -// to dest. If dest is scannable, the result must only have one column. Otherwise, -// StructScan is used. Get will return sql.ErrNoRows like row.Scan would. -func Get(q Queryer, dest interface{}, query string, args ...interface{}) error { - r := q.QueryRowx(query, args...) - return r.scanAny(dest, false) -} - -// LoadFile exec's every statement in a file (as a single call to Exec). -// LoadFile may return a nil *sql.Result if errors are encountered locating or -// reading the file at path. LoadFile reads the entire file into memory, so it -// is not suitable for loading large data dumps, but can be useful for initializing -// schemas or loading indexes. -// -// FIXME: this does not really work with multi-statement files for mattn/go-sqlite3 -// or the go-mysql-driver/mysql drivers; pq seems to be an exception here. Detecting -// this by requiring something with DriverName() and then attempting to split the -// queries will be difficult to get right, and its current driver-specific behavior -// is deemed at least not complex in its incorrectness. -func LoadFile(e Execer, path string) (*sql.Result, error) { - realpath, err := filepath.Abs(path) - if err != nil { - return nil, err - } - contents, err := ioutil.ReadFile(realpath) - if err != nil { - return nil, err - } - res, err := e.Exec(string(contents)) - return &res, err -} - -// MustExec execs the query using e and panics if there was an error. -func MustExec(e Execer, query string, args ...interface{}) sql.Result { - res, err := e.Exec(query, args...) - if err != nil { - panic(err) - } - return res -} - -// SliceScan using this Rows. -func (r *Row) SliceScan() ([]interface{}, error) { - return SliceScan(r) -} - -// MapScan using this Rows. -func (r *Row) MapScan(dest map[string]interface{}) error { - return MapScan(r, dest) -} - -func (r *Row) scanAny(dest interface{}, structOnly bool) error { - if r.err != nil { - return r.err - } - defer r.rows.Close() - - v := reflect.ValueOf(dest) - if v.Kind() != reflect.Ptr { - return errors.New("must pass a pointer, not a value, to StructScan destination") - } - if v.IsNil() { - return errors.New("nil pointer passed to StructScan destination") - } - - base := reflectx.Deref(v.Type()) - scannable := isScannable(base) - - if structOnly && scannable { - return structOnlyError(base) - } - - columns, err := r.Columns() - if err != nil { - return err - } - - if scannable && len(columns) > 1 { - return fmt.Errorf("scannable dest type %s with >1 columns (%d) in result", base.Kind(), len(columns)) - } - - if scannable { - return r.Scan(dest) - } - - m := r.Mapper - - fields := m.TraversalsByName(v.Type(), columns) - // if we are not unsafe and are missing fields, return an error - if f, err := missingFields(fields); err != nil && !r.unsafe { - return fmt.Errorf("missing destination name %s", columns[f]) - } - values := make([]interface{}, len(columns)) - - err = fieldsByTraversal(v, fields, values, true) - if err != nil { - return err - } - // scan into the struct field pointers and append to our results - return r.Scan(values...) -} - -// StructScan a single Row into dest. -func (r *Row) StructScan(dest interface{}) error { - return r.scanAny(dest, true) -} - -// SliceScan a row, returning a []interface{} with values similar to MapScan. -// This function is primarly intended for use where the number of columns -// is not known. Because you can pass an []interface{} directly to Scan, -// it's recommended that you do that as it will not have to allocate new -// slices per row. -func SliceScan(r ColScanner) ([]interface{}, error) { - // ignore r.started, since we needn't use reflect for anything. - columns, err := r.Columns() - if err != nil { - return []interface{}{}, err - } - - values := make([]interface{}, len(columns)) - for i := range values { - values[i] = new(interface{}) - } - - err = r.Scan(values...) - - if err != nil { - return values, err - } - - for i := range columns { - values[i] = *(values[i].(*interface{})) - } - - return values, r.Err() -} - -// MapScan scans a single Row into the dest map[string]interface{}. -// Use this to get results for SQL that might not be under your control -// (for instance, if you're building an interface for an SQL server that -// executes SQL from input). Please do not use this as a primary interface! -// This will modify the map sent to it in place, so reuse the same map with -// care. Columns which occur more than once in the result will overwrite -// eachother! -func MapScan(r ColScanner, dest map[string]interface{}) error { - // ignore r.started, since we needn't use reflect for anything. - columns, err := r.Columns() - if err != nil { - return err - } - - values := make([]interface{}, len(columns)) - for i := range values { - values[i] = new(interface{}) - } - - err = r.Scan(values...) - if err != nil { - return err - } - - for i, column := range columns { - dest[column] = *(values[i].(*interface{})) - } - - return r.Err() -} - -type rowsi interface { - Close() error - Columns() ([]string, error) - Err() error - Next() bool - Scan(...interface{}) error -} - -// structOnlyError returns an error appropriate for type when a non-scannable -// struct is expected but something else is given -func structOnlyError(t reflect.Type) error { - isStruct := t.Kind() == reflect.Struct - isScanner := reflect.PtrTo(t).Implements(_scannerInterface) - if !isStruct { - return fmt.Errorf("expected %s but got %s", reflect.Struct, t.Kind()) - } - if isScanner { - return fmt.Errorf("structscan expects a struct dest but the provided struct type %s implements scanner", t.Name()) - } - return fmt.Errorf("expected a struct, but struct %s has no exported fields", t.Name()) -} - -// scanAll scans all rows into a destination, which must be a slice of any -// type. If the destination slice type is a Struct, then StructScan will be -// used on each row. If the destination is some other kind of base type, then -// each row must only have one column which can scan into that type. This -// allows you to do something like: -// -// rows, _ := db.Query("select id from people;") -// var ids []int -// scanAll(rows, &ids, false) -// -// and ids will be a list of the id results. I realize that this is a desirable -// interface to expose to users, but for now it will only be exposed via changes -// to `Get` and `Select`. The reason that this has been implemented like this is -// this is the only way to not duplicate reflect work in the new API while -// maintaining backwards compatibility. -func scanAll(rows rowsi, dest interface{}, structOnly bool) error { - var v, vp reflect.Value - - value := reflect.ValueOf(dest) - - // json.Unmarshal returns errors for these - if value.Kind() != reflect.Ptr { - return errors.New("must pass a pointer, not a value, to StructScan destination") - } - if value.IsNil() { - return errors.New("nil pointer passed to StructScan destination") - } - direct := reflect.Indirect(value) - - slice, err := baseType(value.Type(), reflect.Slice) - if err != nil { - return err - } - - isPtr := slice.Elem().Kind() == reflect.Ptr - base := reflectx.Deref(slice.Elem()) - scannable := isScannable(base) - - if structOnly && scannable { - return structOnlyError(base) - } - - columns, err := rows.Columns() - if err != nil { - return err - } - - // if it's a base type make sure it only has 1 column; if not return an error - if scannable && len(columns) > 1 { - return fmt.Errorf("non-struct dest type %s with >1 columns (%d)", base.Kind(), len(columns)) - } - - if !scannable { - var values []interface{} - var m *reflectx.Mapper - - switch rows.(type) { - case *Rows: - m = rows.(*Rows).Mapper - default: - m = mapper() - } - - fields := m.TraversalsByName(base, columns) - // if we are not unsafe and are missing fields, return an error - if f, err := missingFields(fields); err != nil && !isUnsafe(rows) { - return fmt.Errorf("missing destination name %s", columns[f]) - } - values = make([]interface{}, len(columns)) - - for rows.Next() { - // create a new struct type (which returns PtrTo) and indirect it - vp = reflect.New(base) - v = reflect.Indirect(vp) - - err = fieldsByTraversal(v, fields, values, true) - - // scan into the struct field pointers and append to our results - err = rows.Scan(values...) - if err != nil { - return err - } - - if isPtr { - direct.Set(reflect.Append(direct, vp)) - } else { - direct.Set(reflect.Append(direct, v)) - } - } - } else { - for rows.Next() { - vp = reflect.New(base) - err = rows.Scan(vp.Interface()) - // append - if isPtr { - direct.Set(reflect.Append(direct, vp)) - } else { - direct.Set(reflect.Append(direct, reflect.Indirect(vp))) - } - } - } - - return rows.Err() -} - -// FIXME: StructScan was the very first bit of API in sqlx, and now unfortunately -// it doesn't really feel like it's named properly. There is an incongruency -// between this and the way that StructScan (which might better be ScanStruct -// anyway) works on a rows object. - -// StructScan all rows from an sql.Rows or an sqlx.Rows into the dest slice. -// StructScan will scan in the entire rows result, so if you need do not want to -// allocate structs for the entire result, use Queryx and see sqlx.Rows.StructScan. -// If rows is sqlx.Rows, it will use its mapper, otherwise it will use the default. -func StructScan(rows rowsi, dest interface{}) error { - return scanAll(rows, dest, true) - -} - -// reflect helpers - -func baseType(t reflect.Type, expected reflect.Kind) (reflect.Type, error) { - t = reflectx.Deref(t) - if t.Kind() != expected { - return nil, fmt.Errorf("expected %s but got %s", expected, t.Kind()) - } - return t, nil -} - -// fieldsByName fills a values interface with fields from the passed value based -// on the traversals in int. If ptrs is true, return addresses instead of values. -// We write this instead of using FieldsByName to save allocations and map lookups -// when iterating over many rows. Empty traversals will get an interface pointer. -// Because of the necessity of requesting ptrs or values, it's considered a bit too -// specialized for inclusion in reflectx itself. -func fieldsByTraversal(v reflect.Value, traversals [][]int, values []interface{}, ptrs bool) error { - v = reflect.Indirect(v) - if v.Kind() != reflect.Struct { - return errors.New("argument not a struct") - } - - for i, traversal := range traversals { - if len(traversal) == 0 { - values[i] = new(interface{}) - continue - } - f := reflectx.FieldByIndexes(v, traversal) - if ptrs { - values[i] = f.Addr().Interface() - } else { - values[i] = f.Interface() - } - } - return nil -} - -func missingFields(transversals [][]int) (field int, err error) { - for i, t := range transversals { - if len(t) == 0 { - return i, errors.New("missing field") - } - } - return 0, nil -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/LICENSE b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/LICENSE deleted file mode 100644 index ade9307..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -All files in this repository are licensed as follows. If you contribute -to this repository, it is assumed that you license your contribution -under the same license unless you state otherwise. - -All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file. - -This software is licensed under the LGPLv3, included below. - -As a special exception to the GNU Lesser General Public License version 3 -("LGPL3"), the copyright holders of this Library give you permission to -convey to a third party a Combined Work that links statically or dynamically -to this Library without providing any Minimal Corresponding Source or -Minimal Application Code as set out in 4d or providing the installation -information set out in section 4e, provided that you comply with the other -provisions of LGPL3 and provided that you meet, for the Application the -terms and conditions of the license(s) which apply to the Application. - -Except as stated in this special exception, the provisions of LGPL3 will -continue to comply in full to this Library. If you modify this Library, you -may apply this exception to your version of this Library, but you are not -obliged to do so. If you do not wish to do so, delete this exception -statement from your version. This exception does not (and cannot) modify any -license terms which apply to the Application, with which you must still -comply. - - - GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - - This version of the GNU Lesser General Public License incorporates -the terms and conditions of version 3 of the GNU General Public -License, supplemented by the additional permissions listed below. - - 0. Additional Definitions. - - As used herein, "this License" refers to version 3 of the GNU Lesser -General Public License, and the "GNU GPL" refers to version 3 of the GNU -General Public License. - - "The Library" refers to a covered work governed by this License, -other than an Application or a Combined Work as defined below. - - An "Application" is any work that makes use of an interface provided -by the Library, but which is not otherwise based on the Library. -Defining a subclass of a class defined by the Library is deemed a mode -of using an interface provided by the Library. - - A "Combined Work" is a work produced by combining or linking an -Application with the Library. The particular version of the Library -with which the Combined Work was made is also called the "Linked -Version". - - The "Minimal Corresponding Source" for a Combined Work means the -Corresponding Source for the Combined Work, excluding any source code -for portions of the Combined Work that, considered in isolation, are -based on the Application, and not on the Linked Version. - - The "Corresponding Application Code" for a Combined Work means the -object code and/or source code for the Application, including any data -and utility programs needed for reproducing the Combined Work from the -Application, but excluding the System Libraries of the Combined Work. - - 1. Exception to Section 3 of the GNU GPL. - - You may convey a covered work under sections 3 and 4 of this License -without being bound by section 3 of the GNU GPL. - - 2. Conveying Modified Versions. - - If you modify a copy of the Library, and, in your modifications, a -facility refers to a function or data to be supplied by an Application -that uses the facility (other than as an argument passed when the -facility is invoked), then you may convey a copy of the modified -version: - - a) under this License, provided that you make a good faith effort to - ensure that, in the event an Application does not supply the - function or data, the facility still operates, and performs - whatever part of its purpose remains meaningful, or - - b) under the GNU GPL, with none of the additional permissions of - this License applicable to that copy. - - 3. Object Code Incorporating Material from Library Header Files. - - The object code form of an Application may incorporate material from -a header file that is part of the Library. You may convey such object -code under terms of your choice, provided that, if the incorporated -material is not limited to numerical parameters, data structure -layouts and accessors, or small macros, inline functions and templates -(ten or fewer lines in length), you do both of the following: - - a) Give prominent notice with each copy of the object code that the - Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the object code with a copy of the GNU GPL and this license - document. - - 4. Combined Works. - - You may convey a Combined Work under terms of your choice that, -taken together, effectively do not restrict modification of the -portions of the Library contained in the Combined Work and reverse -engineering for debugging such modifications, if you also do each of -the following: - - a) Give prominent notice with each copy of the Combined Work that - the Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the Combined Work with a copy of the GNU GPL and this license - document. - - c) For a Combined Work that displays copyright notices during - execution, include the copyright notice for the Library among - these notices, as well as a reference directing the user to the - copies of the GNU GPL and this license document. - - d) Do one of the following: - - 0) Convey the Minimal Corresponding Source under the terms of this - License, and the Corresponding Application Code in a form - suitable for, and under terms that permit, the user to - recombine or relink the Application with a modified version of - the Linked Version to produce a modified Combined Work, in the - manner specified by section 6 of the GNU GPL for conveying - Corresponding Source. - - 1) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (a) uses at run time - a copy of the Library already present on the user's computer - system, and (b) will operate properly with a modified version - of the Library that is interface-compatible with the Linked - Version. - - e) Provide Installation Information, but only if you would otherwise - be required to provide such information under section 6 of the - GNU GPL, and only to the extent that such information is - necessary to install and execute a modified version of the - Combined Work produced by recombining or relinking the - Application with a modified version of the Linked Version. (If - you use option 4d0, the Installation Information must accompany - the Minimal Corresponding Source and Corresponding Application - Code. If you use option 4d1, you must provide the Installation - Information in the manner specified by section 6 of the GNU GPL - for conveying Corresponding Source.) - - 5. Combined Libraries. - - You may place library facilities that are a work based on the -Library side by side in a single library together with other library -facilities that are not Applications and are not covered by this -License, and convey such a combined library under terms of your -choice, if you do both of the following: - - a) Accompany the combined library with a copy of the same work based - on the Library, uncombined with any other library facilities, - conveyed under the terms of this License. - - b) Give prominent notice with the combined library that part of it - is a work based on the Library, and explaining where to find the - accompanying uncombined form of the same work. - - 6. Revised Versions of the GNU Lesser General Public License. - - The Free Software Foundation may publish revised and/or new versions -of the GNU Lesser General Public License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. - - Each version is given a distinguishing version number. If the -Library as you received it specifies that a certain numbered version -of the GNU Lesser General Public License "or any later version" -applies to it, you have the option of following the terms and -conditions either of that published version or of any later version -published by the Free Software Foundation. If the Library as you -received it does not specify a version number of the GNU Lesser -General Public License, you may choose any version of the GNU Lesser -General Public License ever published by the Free Software Foundation. - - If the Library as you received it specifies that a proxy can decide -whether future versions of the GNU Lesser General Public License shall -apply, that proxy's public statement of acceptance of any version is -permanent authorization for you to choose that version for the -Library. diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/doc.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/doc.go deleted file mode 100644 index 35b119a..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/doc.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2013, 2014 Canonical Ltd. -// Licensed under the LGPLv3, see LICENCE file for details. - -/* -[godoc-link-here] - -The juju/errors provides an easy way to annotate errors without losing the -orginal error context. - -The exported `New` and `Errorf` functions are designed to replace the -`errors.New` and `fmt.Errorf` functions respectively. The same underlying -error is there, but the package also records the location at which the error -was created. - -A primary use case for this library is to add extra context any time an -error is returned from a function. - - if err := SomeFunc(); err != nil { - return err - } - -This instead becomes: - - if err := SomeFunc(); err != nil { - return errors.Trace(err) - } - -which just records the file and line number of the Trace call, or - - if err := SomeFunc(); err != nil { - return errors.Annotate(err, "more context") - } - -which also adds an annotation to the error. - -When you want to check to see if an error is of a particular type, a helper -function is normally exported by the package that returned the error, like the -`os` package does. The underlying cause of the error is available using the -`Cause` function. - - os.IsNotExist(errors.Cause(err)) - -The result of the `Error()` call on an annotated error is the annotations joined -with colons, then the result of the `Error()` method for the underlying error -that was the cause. - - err := errors.Errorf("original") - err = errors.Annotatef(err, "context") - err = errors.Annotatef(err, "more context") - err.Error() -> "more context: context: original" - -Obviously recording the file, line and functions is not very useful if you -cannot get them back out again. - - errors.ErrorStack(err) - -will return something like: - - first error - github.com/juju/errors/annotation_test.go:193: - github.com/juju/errors/annotation_test.go:194: annotation - github.com/juju/errors/annotation_test.go:195: - github.com/juju/errors/annotation_test.go:196: more context - github.com/juju/errors/annotation_test.go:197: - -The first error was generated by an external system, so there was no location -associated. The second, fourth, and last lines were generated with Trace calls, -and the other two through Annotate. - -Sometimes when responding to an error you want to return a more specific error -for the situation. - - if err := FindField(field); err != nil { - return errors.Wrap(err, errors.NotFoundf(field)) - } - -This returns an error where the complete error stack is still available, and -`errors.Cause()` will return the `NotFound` error. - -*/ -package errors diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/error.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/error.go deleted file mode 100644 index 8c51c45..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/error.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2014 Canonical Ltd. -// Licensed under the LGPLv3, see LICENCE file for details. - -package errors - -import ( - "fmt" - "reflect" - "runtime" -) - -// Err holds a description of an error along with information about -// where the error was created. -// -// It may be embedded in custom error types to add extra information that -// this errors package can understand. -type Err struct { - // message holds an annotation of the error. - message string - - // cause holds the cause of the error as returned - // by the Cause method. - cause error - - // previous holds the previous error in the error stack, if any. - previous error - - // file and line hold the source code location where the error was - // created. - file string - line int -} - -// NewErr is used to return an Err for the purpose of embedding in other -// structures. The location is not specified, and needs to be set with a call -// to SetLocation. -// -// For example: -// type FooError struct { -// errors.Err -// code int -// } -// -// func NewFooError(code int) error { -// err := &FooError{errors.NewErr("foo"), code} -// err.SetLocation(1) -// return err -// } -func NewErr(format string, args ...interface{}) Err { - return Err{ - message: fmt.Sprintf(format, args...), - } -} - -// NewErrWithCause is used to return an Err with case by other error for the purpose of embedding in other -// structures. The location is not specified, and needs to be set with a call -// to SetLocation. -// -// For example: -// type FooError struct { -// errors.Err -// code int -// } -// -// func (e *FooError) Annotate(format string, args ...interface{}) error { -// err := &FooError{errors.NewErrWithCause(e.Err, format, args...), e.code} -// err.SetLocation(1) -// return err -// }) -func NewErrWithCause(other error, format string, args ...interface{}) Err { - return Err{ - message: fmt.Sprintf(format, args...), - cause: Cause(other), - previous: other, - } -} - -// Location is the file and line of where the error was most recently -// created or annotated. -func (e *Err) Location() (filename string, line int) { - return e.file, e.line -} - -// Underlying returns the previous error in the error stack, if any. A client -// should not ever really call this method. It is used to build the error -// stack and should not be introspected by client calls. Or more -// specifically, clients should not depend on anything but the `Cause` of an -// error. -func (e *Err) Underlying() error { - return e.previous -} - -// The Cause of an error is the most recent error in the error stack that -// meets one of these criteria: the original error that was raised; the new -// error that was passed into the Wrap function; the most recently masked -// error; or nil if the error itself is considered the Cause. Normally this -// method is not invoked directly, but instead through the Cause stand alone -// function. -func (e *Err) Cause() error { - return e.cause -} - -// Message returns the message stored with the most recent location. This is -// the empty string if the most recent call was Trace, or the message stored -// with Annotate or Mask. -func (e *Err) Message() string { - return e.message -} - -// Error implements error.Error. -func (e *Err) Error() string { - // We want to walk up the stack of errors showing the annotations - // as long as the cause is the same. - err := e.previous - if !sameError(Cause(err), e.cause) && e.cause != nil { - err = e.cause - } - switch { - case err == nil: - return e.message - case e.message == "": - return err.Error() - } - return fmt.Sprintf("%s: %v", e.message, err) -} - -// SetLocation records the source location of the error at callDepth stack -// frames above the call. -func (e *Err) SetLocation(callDepth int) { - _, file, line, _ := runtime.Caller(callDepth + 1) - e.file = trimGoPath(file) - e.line = line -} - -// StackTrace returns one string for each location recorded in the stack of -// errors. The first value is the originating error, with a line for each -// other annotation or tracing of the error. -func (e *Err) StackTrace() []string { - return errorStack(e) -} - -// Ideally we'd have a way to check identity, but deep equals will do. -func sameError(e1, e2 error) bool { - return reflect.DeepEqual(e1, e2) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/errortypes.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/errortypes.go deleted file mode 100644 index 10b3b19..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/errortypes.go +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2014 Canonical Ltd. -// Licensed under the LGPLv3, see LICENCE file for details. - -package errors - -import ( - "fmt" -) - -// wrap is a helper to construct an *wrapper. -func wrap(err error, format, suffix string, args ...interface{}) Err { - newErr := Err{ - message: fmt.Sprintf(format+suffix, args...), - previous: err, - } - newErr.SetLocation(2) - return newErr -} - -// notFound represents an error when something has not been found. -type notFound struct { - Err -} - -// NotFoundf returns an error which satisfies IsNotFound(). -func NotFoundf(format string, args ...interface{}) error { - return ¬Found{wrap(nil, format, " not found", args...)} -} - -// NewNotFound returns an error which wraps err that satisfies -// IsNotFound(). -func NewNotFound(err error, msg string) error { - return ¬Found{wrap(err, msg, "")} -} - -// IsNotFound reports whether err was created with NotFoundf() or -// NewNotFound(). -func IsNotFound(err error) bool { - err = Cause(err) - _, ok := err.(*notFound) - return ok -} - -// userNotFound represents an error when an inexistent user is looked up. -type userNotFound struct { - Err -} - -// UserNotFoundf returns an error which satisfies IsUserNotFound(). -func UserNotFoundf(format string, args ...interface{}) error { - return &userNotFound{wrap(nil, format, " user not found", args...)} -} - -// NewUserNotFound returns an error which wraps err and satisfies -// IsUserNotFound(). -func NewUserNotFound(err error, msg string) error { - return &userNotFound{wrap(err, msg, "")} -} - -// IsUserNotFound reports whether err was created with UserNotFoundf() or -// NewUserNotFound(). -func IsUserNotFound(err error) bool { - err = Cause(err) - _, ok := err.(*userNotFound) - return ok -} - -// unauthorized represents an error when an operation is unauthorized. -type unauthorized struct { - Err -} - -// Unauthorizedf returns an error which satisfies IsUnauthorized(). -func Unauthorizedf(format string, args ...interface{}) error { - return &unauthorized{wrap(nil, format, "", args...)} -} - -// NewUnauthorized returns an error which wraps err and satisfies -// IsUnauthorized(). -func NewUnauthorized(err error, msg string) error { - return &unauthorized{wrap(err, msg, "")} -} - -// IsUnauthorized reports whether err was created with Unauthorizedf() or -// NewUnauthorized(). -func IsUnauthorized(err error) bool { - err = Cause(err) - _, ok := err.(*unauthorized) - return ok -} - -// notImplemented represents an error when something is not -// implemented. -type notImplemented struct { - Err -} - -// NotImplementedf returns an error which satisfies IsNotImplemented(). -func NotImplementedf(format string, args ...interface{}) error { - return ¬Implemented{wrap(nil, format, " not implemented", args...)} -} - -// NewNotImplemented returns an error which wraps err and satisfies -// IsNotImplemented(). -func NewNotImplemented(err error, msg string) error { - return ¬Implemented{wrap(err, msg, "")} -} - -// IsNotImplemented reports whether err was created with -// NotImplementedf() or NewNotImplemented(). -func IsNotImplemented(err error) bool { - err = Cause(err) - _, ok := err.(*notImplemented) - return ok -} - -// alreadyExists represents and error when something already exists. -type alreadyExists struct { - Err -} - -// AlreadyExistsf returns an error which satisfies IsAlreadyExists(). -func AlreadyExistsf(format string, args ...interface{}) error { - return &alreadyExists{wrap(nil, format, " already exists", args...)} -} - -// NewAlreadyExists returns an error which wraps err and satisfies -// IsAlreadyExists(). -func NewAlreadyExists(err error, msg string) error { - return &alreadyExists{wrap(err, msg, "")} -} - -// IsAlreadyExists reports whether the error was created with -// AlreadyExistsf() or NewAlreadyExists(). -func IsAlreadyExists(err error) bool { - err = Cause(err) - _, ok := err.(*alreadyExists) - return ok -} - -// notSupported represents an error when something is not supported. -type notSupported struct { - Err -} - -// NotSupportedf returns an error which satisfies IsNotSupported(). -func NotSupportedf(format string, args ...interface{}) error { - return ¬Supported{wrap(nil, format, " not supported", args...)} -} - -// NewNotSupported returns an error which wraps err and satisfies -// IsNotSupported(). -func NewNotSupported(err error, msg string) error { - return ¬Supported{wrap(err, msg, "")} -} - -// IsNotSupported reports whether the error was created with -// NotSupportedf() or NewNotSupported(). -func IsNotSupported(err error) bool { - err = Cause(err) - _, ok := err.(*notSupported) - return ok -} - -// notValid represents an error when something is not valid. -type notValid struct { - Err -} - -// NotValidf returns an error which satisfies IsNotValid(). -func NotValidf(format string, args ...interface{}) error { - return ¬Valid{wrap(nil, format, " not valid", args...)} -} - -// NewNotValid returns an error which wraps err and satisfies IsNotValid(). -func NewNotValid(err error, msg string) error { - return ¬Valid{wrap(err, msg, "")} -} - -// IsNotValid reports whether the error was created with NotValidf() or -// NewNotValid(). -func IsNotValid(err error) bool { - err = Cause(err) - _, ok := err.(*notValid) - return ok -} - -// notProvisioned represents an error when something is not yet provisioned. -type notProvisioned struct { - Err -} - -// NotProvisionedf returns an error which satisfies IsNotProvisioned(). -func NotProvisionedf(format string, args ...interface{}) error { - return ¬Provisioned{wrap(nil, format, " not provisioned", args...)} -} - -// NewNotProvisioned returns an error which wraps err that satisfies -// IsNotProvisioned(). -func NewNotProvisioned(err error, msg string) error { - return ¬Provisioned{wrap(err, msg, "")} -} - -// IsNotProvisioned reports whether err was created with NotProvisionedf() or -// NewNotProvisioned(). -func IsNotProvisioned(err error) bool { - err = Cause(err) - _, ok := err.(*notProvisioned) - return ok -} - -// notAssigned represents an error when something is not yet assigned to -// something else. -type notAssigned struct { - Err -} - -// NotAssignedf returns an error which satisfies IsNotAssigned(). -func NotAssignedf(format string, args ...interface{}) error { - return ¬Assigned{wrap(nil, format, " not assigned", args...)} -} - -// NewNotAssigned returns an error which wraps err that satisfies -// IsNotAssigned(). -func NewNotAssigned(err error, msg string) error { - return ¬Assigned{wrap(err, msg, "")} -} - -// IsNotAssigned reports whether err was created with NotAssignedf() or -// NewNotAssigned(). -func IsNotAssigned(err error) bool { - err = Cause(err) - _, ok := err.(*notAssigned) - return ok -} - -// badRequest represents an error when a request has bad parameters. -type badRequest struct { - Err -} - -// BadRequestf returns an error which satisfies IsBadRequest(). -func BadRequestf(format string, args ...interface{}) error { - return &badRequest{wrap(nil, format, "", args...)} -} - -// NewBadRequest returns an error which wraps err that satisfies -// IsBadRequest(). -func NewBadRequest(err error, msg string) error { - return &badRequest{wrap(err, msg, "")} -} - -// IsBadRequest reports whether err was created with BadRequestf() or -// NewBadRequest(). -func IsBadRequest(err error) bool { - err = Cause(err) - _, ok := err.(*badRequest) - return ok -} - -// methodNotAllowed represents an error when an HTTP request -// is made with an inappropriate method. -type methodNotAllowed struct { - Err -} - -// MethodNotAllowedf returns an error which satisfies IsMethodNotAllowed(). -func MethodNotAllowedf(format string, args ...interface{}) error { - return &methodNotAllowed{wrap(nil, format, "", args...)} -} - -// NewMethodNotAllowed returns an error which wraps err that satisfies -// IsMethodNotAllowed(). -func NewMethodNotAllowed(err error, msg string) error { - return &methodNotAllowed{wrap(err, msg, "")} -} - -// IsMethodNotAllowed reports whether err was created with MethodNotAllowedf() or -// NewMethodNotAllowed(). -func IsMethodNotAllowed(err error) bool { - err = Cause(err) - _, ok := err.(*methodNotAllowed) - return ok -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/functions.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/functions.go deleted file mode 100644 index 994208d..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/functions.go +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright 2014 Canonical Ltd. -// Licensed under the LGPLv3, see LICENCE file for details. - -package errors - -import ( - "fmt" - "strings" -) - -// New is a drop in replacement for the standard libary errors module that records -// the location that the error is created. -// -// For example: -// return errors.New("validation failed") -// -func New(message string) error { - err := &Err{message: message} - err.SetLocation(1) - return err -} - -// Errorf creates a new annotated error and records the location that the -// error is created. This should be a drop in replacement for fmt.Errorf. -// -// For example: -// return errors.Errorf("validation failed: %s", message) -// -func Errorf(format string, args ...interface{}) error { - err := &Err{message: fmt.Sprintf(format, args...)} - err.SetLocation(1) - return err -} - -// Trace adds the location of the Trace call to the stack. The Cause of the -// resulting error is the same as the error parameter. If the other error is -// nil, the result will be nil. -// -// For example: -// if err := SomeFunc(); err != nil { -// return errors.Trace(err) -// } -// -func Trace(other error) error { - if other == nil { - return nil - } - err := &Err{previous: other, cause: Cause(other)} - err.SetLocation(1) - return err -} - -// Annotate is used to add extra context to an existing error. The location of -// the Annotate call is recorded with the annotations. The file, line and -// function are also recorded. -// -// For example: -// if err := SomeFunc(); err != nil { -// return errors.Annotate(err, "failed to frombulate") -// } -// -func Annotate(other error, message string) error { - if other == nil { - return nil - } - err := &Err{ - previous: other, - cause: Cause(other), - message: message, - } - err.SetLocation(1) - return err -} - -// Annotatef is used to add extra context to an existing error. The location of -// the Annotate call is recorded with the annotations. The file, line and -// function are also recorded. -// -// For example: -// if err := SomeFunc(); err != nil { -// return errors.Annotatef(err, "failed to frombulate the %s", arg) -// } -// -func Annotatef(other error, format string, args ...interface{}) error { - if other == nil { - return nil - } - err := &Err{ - previous: other, - cause: Cause(other), - message: fmt.Sprintf(format, args...), - } - err.SetLocation(1) - return err -} - -// DeferredAnnotatef annotates the given error (when it is not nil) with the given -// format string and arguments (like fmt.Sprintf). If *err is nil, DeferredAnnotatef -// does nothing. This method is used in a defer statement in order to annotate any -// resulting error with the same message. -// -// For example: -// -// defer DeferredAnnotatef(&err, "failed to frombulate the %s", arg) -// -func DeferredAnnotatef(err *error, format string, args ...interface{}) { - if *err == nil { - return - } - newErr := &Err{ - message: fmt.Sprintf(format, args...), - cause: Cause(*err), - previous: *err, - } - newErr.SetLocation(1) - *err = newErr -} - -// Wrap changes the Cause of the error. The location of the Wrap call is also -// stored in the error stack. -// -// For example: -// if err := SomeFunc(); err != nil { -// newErr := &packageError{"more context", private_value} -// return errors.Wrap(err, newErr) -// } -// -func Wrap(other, newDescriptive error) error { - err := &Err{ - previous: other, - cause: newDescriptive, - } - err.SetLocation(1) - return err -} - -// Wrapf changes the Cause of the error, and adds an annotation. The location -// of the Wrap call is also stored in the error stack. -// -// For example: -// if err := SomeFunc(); err != nil { -// return errors.Wrapf(err, simpleErrorType, "invalid value %q", value) -// } -// -func Wrapf(other, newDescriptive error, format string, args ...interface{}) error { - err := &Err{ - message: fmt.Sprintf(format, args...), - previous: other, - cause: newDescriptive, - } - err.SetLocation(1) - return err -} - -// Mask masks the given error with the given format string and arguments (like -// fmt.Sprintf), returning a new error that maintains the error stack, but -// hides the underlying error type. The error string still contains the full -// annotations. If you want to hide the annotations, call Wrap. -func Maskf(other error, format string, args ...interface{}) error { - if other == nil { - return nil - } - err := &Err{ - message: fmt.Sprintf(format, args...), - previous: other, - } - err.SetLocation(1) - return err -} - -// Mask hides the underlying error type, and records the location of the masking. -func Mask(other error) error { - if other == nil { - return nil - } - err := &Err{ - previous: other, - } - err.SetLocation(1) - return err -} - -// Cause returns the cause of the given error. This will be either the -// original error, or the result of a Wrap or Mask call. -// -// Cause is the usual way to diagnose errors that may have been wrapped by -// the other errors functions. -func Cause(err error) error { - var diag error - if err, ok := err.(causer); ok { - diag = err.Cause() - } - if diag != nil { - return diag - } - return err -} - -type causer interface { - Cause() error -} - -type wrapper interface { - // Message returns the top level error message, - // not including the message from the Previous - // error. - Message() string - - // Underlying returns the Previous error, or nil - // if there is none. - Underlying() error -} - -type locationer interface { - Location() (string, int) -} - -var ( - _ wrapper = (*Err)(nil) - _ locationer = (*Err)(nil) - _ causer = (*Err)(nil) -) - -// Details returns information about the stack of errors wrapped by err, in -// the format: -// -// [{filename:99: error one} {otherfile:55: cause of error one}] -// -// This is a terse alternative to ErrorStack as it returns a single line. -func Details(err error) string { - if err == nil { - return "[]" - } - var s []byte - s = append(s, '[') - for { - s = append(s, '{') - if err, ok := err.(locationer); ok { - file, line := err.Location() - if file != "" { - s = append(s, fmt.Sprintf("%s:%d", file, line)...) - s = append(s, ": "...) - } - } - if cerr, ok := err.(wrapper); ok { - s = append(s, cerr.Message()...) - err = cerr.Underlying() - } else { - s = append(s, err.Error()...) - err = nil - } - s = append(s, '}') - if err == nil { - break - } - s = append(s, ' ') - } - s = append(s, ']') - return string(s) -} - -// ErrorStack returns a string representation of the annotated error. If the -// error passed as the parameter is not an annotated error, the result is -// simply the result of the Error() method on that error. -// -// If the error is an annotated error, a multi-line string is returned where -// each line represents one entry in the annotation stack. The full filename -// from the call stack is used in the output. -// -// first error -// github.com/juju/errors/annotation_test.go:193: -// github.com/juju/errors/annotation_test.go:194: annotation -// github.com/juju/errors/annotation_test.go:195: -// github.com/juju/errors/annotation_test.go:196: more context -// github.com/juju/errors/annotation_test.go:197: -func ErrorStack(err error) string { - return strings.Join(errorStack(err), "\n") -} - -func errorStack(err error) []string { - if err == nil { - return nil - } - - // We want the first error first - var lines []string - for { - var buff []byte - if err, ok := err.(locationer); ok { - file, line := err.Location() - // Strip off the leading GOPATH/src path elements. - file = trimGoPath(file) - if file != "" { - buff = append(buff, fmt.Sprintf("%s:%d", file, line)...) - buff = append(buff, ": "...) - } - } - if cerr, ok := err.(wrapper); ok { - message := cerr.Message() - buff = append(buff, message...) - // If there is a cause for this error, and it is different to the cause - // of the underlying error, then output the error string in the stack trace. - var cause error - if err1, ok := err.(causer); ok { - cause = err1.Cause() - } - err = cerr.Underlying() - if cause != nil && !sameError(Cause(err), cause) { - if message != "" { - buff = append(buff, ": "...) - } - buff = append(buff, cause.Error()...) - } - } else { - buff = append(buff, err.Error()...) - err = nil - } - lines = append(lines, string(buff)) - if err == nil { - break - } - } - // reverse the lines to get the original error, which was at the end of - // the list, back to the start. - var result []string - for i := len(lines); i > 0; i-- { - result = append(result, lines[i-1]) - } - return result -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/path.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/path.go deleted file mode 100644 index a7b726a..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/juju/errors/path.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2013, 2014 Canonical Ltd. -// Licensed under the LGPLv3, see LICENCE file for details. - -package errors - -import ( - "runtime" - "strings" -) - -// prefixSize is used internally to trim the user specific path from the -// front of the returned filenames from the runtime call stack. -var prefixSize int - -// goPath is the deduced path based on the location of this file as compiled. -var goPath string - -func init() { - _, file, _, ok := runtime.Caller(0) - if file == "?" { - return - } - if ok { - // We know that the end of the file should be: - // github.com/juju/errors/path.go - size := len(file) - suffix := len("github.com/juju/errors/path.go") - goPath = file[:size-suffix] - prefixSize = len(goPath) - } -} - -func trimGoPath(filename string) string { - if strings.HasPrefix(filename, goPath) { - return filename[prefixSize:] - } - return filename -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/LICENSE b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/LICENSE deleted file mode 100644 index 6600f1c..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/LICENSE +++ /dev/null @@ -1,165 +0,0 @@ -GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - - This version of the GNU Lesser General Public License incorporates -the terms and conditions of version 3 of the GNU General Public -License, supplemented by the additional permissions listed below. - - 0. Additional Definitions. - - As used herein, "this License" refers to version 3 of the GNU Lesser -General Public License, and the "GNU GPL" refers to version 3 of the GNU -General Public License. - - "The Library" refers to a covered work governed by this License, -other than an Application or a Combined Work as defined below. - - An "Application" is any work that makes use of an interface provided -by the Library, but which is not otherwise based on the Library. -Defining a subclass of a class defined by the Library is deemed a mode -of using an interface provided by the Library. - - A "Combined Work" is a work produced by combining or linking an -Application with the Library. The particular version of the Library -with which the Combined Work was made is also called the "Linked -Version". - - The "Minimal Corresponding Source" for a Combined Work means the -Corresponding Source for the Combined Work, excluding any source code -for portions of the Combined Work that, considered in isolation, are -based on the Application, and not on the Linked Version. - - The "Corresponding Application Code" for a Combined Work means the -object code and/or source code for the Application, including any data -and utility programs needed for reproducing the Combined Work from the -Application, but excluding the System Libraries of the Combined Work. - - 1. Exception to Section 3 of the GNU GPL. - - You may convey a covered work under sections 3 and 4 of this License -without being bound by section 3 of the GNU GPL. - - 2. Conveying Modified Versions. - - If you modify a copy of the Library, and, in your modifications, a -facility refers to a function or data to be supplied by an Application -that uses the facility (other than as an argument passed when the -facility is invoked), then you may convey a copy of the modified -version: - - a) under this License, provided that you make a good faith effort to - ensure that, in the event an Application does not supply the - function or data, the facility still operates, and performs - whatever part of its purpose remains meaningful, or - - b) under the GNU GPL, with none of the additional permissions of - this License applicable to that copy. - - 3. Object Code Incorporating Material from Library Header Files. - - The object code form of an Application may incorporate material from -a header file that is part of the Library. You may convey such object -code under terms of your choice, provided that, if the incorporated -material is not limited to numerical parameters, data structure -layouts and accessors, or small macros, inline functions and templates -(ten or fewer lines in length), you do both of the following: - - a) Give prominent notice with each copy of the object code that the - Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the object code with a copy of the GNU GPL and this license - document. - - 4. Combined Works. - - You may convey a Combined Work under terms of your choice that, -taken together, effectively do not restrict modification of the -portions of the Library contained in the Combined Work and reverse -engineering for debugging such modifications, if you also do each of -the following: - - a) Give prominent notice with each copy of the Combined Work that - the Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the Combined Work with a copy of the GNU GPL and this license - document. - - c) For a Combined Work that displays copyright notices during - execution, include the copyright notice for the Library among - these notices, as well as a reference directing the user to the - copies of the GNU GPL and this license document. - - d) Do one of the following: - - 0) Convey the Minimal Corresponding Source under the terms of this - License, and the Corresponding Application Code in a form - suitable for, and under terms that permit, the user to - recombine or relink the Application with a modified version of - the Linked Version to produce a modified Combined Work, in the - manner specified by section 6 of the GNU GPL for conveying - Corresponding Source. - - 1) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (a) uses at run time - a copy of the Library already present on the user's computer - system, and (b) will operate properly with a modified version - of the Library that is interface-compatible with the Linked - Version. - - e) Provide Installation Information, but only if you would otherwise - be required to provide such information under section 6 of the - GNU GPL, and only to the extent that such information is - necessary to install and execute a modified version of the - Combined Work produced by recombining or relinking the - Application with a modified version of the Linked Version. (If - you use option 4d0, the Installation Information must accompany - the Minimal Corresponding Source and Corresponding Application - Code. If you use option 4d1, you must provide the Installation - Information in the manner specified by section 6 of the GNU GPL - for conveying Corresponding Source.) - - 5. Combined Libraries. - - You may place library facilities that are a work based on the -Library side by side in a single library together with other library -facilities that are not Applications and are not covered by this -License, and convey such a combined library under terms of your -choice, if you do both of the following: - - a) Accompany the combined library with a copy of the same work based - on the Library, uncombined with any other library facilities, - conveyed under the terms of this License. - - b) Give prominent notice with the combined library that part of it - is a work based on the Library, and explaining where to find the - accompanying uncombined form of the same work. - - 6. Revised Versions of the GNU Lesser General Public License. - - The Free Software Foundation may publish revised and/or new versions -of the GNU Lesser General Public License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. - - Each version is given a distinguishing version number. If the -Library as you received it specifies that a certain numbered version -of the GNU Lesser General Public License "or any later version" -applies to it, you have the option of following the terms and -conditions either of that published version or of any later version -published by the Free Software Foundation. If the Library as you -received it does not specify a version number of the GNU Lesser -General Public License, you may choose any version of the GNU Lesser -General Public License ever published by the Free Software Foundation. - - If the Library as you received it specifies that a proxy can decide -whether future versions of the GNU Lesser General Public License shall -apply, that proxy's public statement of acceptance of any version is -permanent authorization for you to choose that version for the -Library. diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_unix.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_unix.go deleted file mode 100644 index 37f407d..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_unix.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build freebsd openbsd netbsd dragonfly darwin linux - -package log - -import ( - "log" - "os" - "syscall" -) - -func CrashLog(file string) { - f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - log.Println(err.Error()) - } else { - syscall.Dup2(int(f.Fd()), 2) - } -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_win.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_win.go deleted file mode 100644 index 7d612ee..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/crash_win.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build windows - -package log - -import ( - "log" - "os" - "syscall" -) - -var ( - kernel32 = syscall.MustLoadDLL("kernel32.dll") - procSetStdHandle = kernel32.MustFindProc("SetStdHandle") -) - -func setStdHandle(stdhandle int32, handle syscall.Handle) error { - r0, _, e1 := syscall.Syscall(procSetStdHandle.Addr(), 2, uintptr(stdhandle), uintptr(handle), 0) - if r0 == 0 { - if e1 != 0 { - return error(e1) - } - return syscall.EINVAL - } - return nil -} - -func CrashLog(file string) { - f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - log.Println(err.Error()) - } else { - err = setStdHandle(syscall.STD_ERROR_HANDLE, syscall.Handle(f.Fd())) - if err != nil { - log.Println(err.Error()) - } - } -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/log.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/log.go deleted file mode 100644 index 896b393..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/ngaut/log/log.go +++ /dev/null @@ -1,380 +0,0 @@ -//high level log wrapper, so it can output different log based on level -package log - -import ( - "fmt" - "io" - "log" - "os" - "runtime" - "sync" - "time" -) - -const ( - Ldate = log.Ldate - Llongfile = log.Llongfile - Lmicroseconds = log.Lmicroseconds - Lshortfile = log.Lshortfile - LstdFlags = log.LstdFlags - Ltime = log.Ltime -) - -type ( - LogLevel int - LogType int -) - -const ( - LOG_FATAL = LogType(0x1) - LOG_ERROR = LogType(0x2) - LOG_WARNING = LogType(0x4) - LOG_INFO = LogType(0x8) - LOG_DEBUG = LogType(0x10) -) - -const ( - LOG_LEVEL_NONE = LogLevel(0x0) - LOG_LEVEL_FATAL = LOG_LEVEL_NONE | LogLevel(LOG_FATAL) - LOG_LEVEL_ERROR = LOG_LEVEL_FATAL | LogLevel(LOG_ERROR) - LOG_LEVEL_WARN = LOG_LEVEL_ERROR | LogLevel(LOG_WARNING) - LOG_LEVEL_INFO = LOG_LEVEL_WARN | LogLevel(LOG_INFO) - LOG_LEVEL_DEBUG = LOG_LEVEL_INFO | LogLevel(LOG_DEBUG) - LOG_LEVEL_ALL = LOG_LEVEL_DEBUG -) - -const FORMAT_TIME_DAY string = "20060102" -const FORMAT_TIME_HOUR string = "2006010215" - -var _log *logger = New() - -func init() { - SetFlags(Ldate | Ltime | Lshortfile) - SetHighlighting(runtime.GOOS != "windows") -} - -func Logger() *log.Logger { - return _log._log -} - -func SetLevel(level LogLevel) { - _log.SetLevel(level) -} -func GetLogLevel() LogLevel { - return _log.level -} - -func SetOutput(out io.Writer) { - _log.SetOutput(out) -} - -func SetOutputByName(path string) error { - return _log.SetOutputByName(path) -} - -func SetFlags(flags int) { - _log._log.SetFlags(flags) -} - -func Info(v ...interface{}) { - _log.Info(v...) -} - -func Infof(format string, v ...interface{}) { - _log.Infof(format, v...) -} - -func Debug(v ...interface{}) { - _log.Debug(v...) -} - -func Debugf(format string, v ...interface{}) { - _log.Debugf(format, v...) -} - -func Warn(v ...interface{}) { - _log.Warning(v...) -} - -func Warnf(format string, v ...interface{}) { - _log.Warningf(format, v...) -} - -func Warning(v ...interface{}) { - _log.Warning(v...) -} - -func Warningf(format string, v ...interface{}) { - _log.Warningf(format, v...) -} - -func Error(v ...interface{}) { - _log.Error(v...) -} - -func Errorf(format string, v ...interface{}) { - _log.Errorf(format, v...) -} - -func Fatal(v ...interface{}) { - _log.Fatal(v...) -} - -func Fatalf(format string, v ...interface{}) { - _log.Fatalf(format, v...) -} - -func SetLevelByString(level string) { - _log.SetLevelByString(level) -} - -func SetHighlighting(highlighting bool) { - _log.SetHighlighting(highlighting) -} - -func SetRotateByDay() { - _log.SetRotateByDay() -} - -func SetRotateByHour() { - _log.SetRotateByHour() -} - -type logger struct { - _log *log.Logger - level LogLevel - highlighting bool - - dailyRolling bool - hourRolling bool - - fileName string - logSuffix string - fd *os.File - - lock sync.Mutex -} - -func (l *logger) SetHighlighting(highlighting bool) { - l.highlighting = highlighting -} - -func (l *logger) SetLevel(level LogLevel) { - l.level = level -} - -func (l *logger) SetLevelByString(level string) { - l.level = StringToLogLevel(level) -} - -func (l *logger) SetRotateByDay() { - l.dailyRolling = true - l.logSuffix = genDayTime(time.Now()) -} - -func (l *logger) SetRotateByHour() { - l.hourRolling = true - l.logSuffix = genHourTime(time.Now()) -} - -func (l *logger) rotate() error { - l.lock.Lock() - defer l.lock.Unlock() - - var suffix string - if l.dailyRolling { - suffix = genDayTime(time.Now()) - } else if l.hourRolling { - suffix = genHourTime(time.Now()) - } else { - return nil - } - - // Notice: if suffix is not equal to l.LogSuffix, then rotate - if suffix != l.logSuffix { - err := l.doRotate(suffix) - if err != nil { - return err - } - } - - return nil -} - -func (l *logger) doRotate(suffix string) error { - // Notice: Not check error, is this ok? - l.fd.Close() - - lastFileName := l.fileName + "." + l.logSuffix - err := os.Rename(l.fileName, lastFileName) - if err != nil { - return err - } - - err = l.SetOutputByName(l.fileName) - if err != nil { - return err - } - - l.logSuffix = suffix - - return nil -} - -func (l *logger) SetOutput(out io.Writer) { - l._log = log.New(out, l._log.Prefix(), l._log.Flags()) -} - -func (l *logger) SetOutputByName(path string) error { - f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666) - if err != nil { - log.Fatal(err) - } - - l.SetOutput(f) - - l.fileName = path - l.fd = f - - return err -} - -func (l *logger) log(t LogType, v ...interface{}) { - if l.level|LogLevel(t) != l.level { - return - } - - err := l.rotate() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - return - } - - v1 := make([]interface{}, len(v)+2) - logStr, logColor := LogTypeToString(t) - if l.highlighting { - v1[0] = "\033" + logColor + "m[" + logStr + "]" - copy(v1[1:], v) - v1[len(v)+1] = "\033[0m" - } else { - v1[0] = "[" + logStr + "]" - copy(v1[1:], v) - v1[len(v)+1] = "" - } - - s := fmt.Sprintln(v1...) - l._log.Output(4, s) -} - -func (l *logger) logf(t LogType, format string, v ...interface{}) { - if l.level|LogLevel(t) != l.level { - return - } - - err := l.rotate() - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - return - } - - logStr, logColor := LogTypeToString(t) - var s string - if l.highlighting { - s = "\033" + logColor + "m[" + logStr + "] " + fmt.Sprintf(format, v...) + "\033[0m" - } else { - s = "[" + logStr + "] " + fmt.Sprintf(format, v...) - } - l._log.Output(4, s) -} - -func (l *logger) Fatal(v ...interface{}) { - l.log(LOG_FATAL, v...) - os.Exit(-1) -} - -func (l *logger) Fatalf(format string, v ...interface{}) { - l.logf(LOG_FATAL, format, v...) - os.Exit(-1) -} - -func (l *logger) Error(v ...interface{}) { - l.log(LOG_ERROR, v...) -} - -func (l *logger) Errorf(format string, v ...interface{}) { - l.logf(LOG_ERROR, format, v...) -} - -func (l *logger) Warning(v ...interface{}) { - l.log(LOG_WARNING, v...) -} - -func (l *logger) Warningf(format string, v ...interface{}) { - l.logf(LOG_WARNING, format, v...) -} - -func (l *logger) Debug(v ...interface{}) { - l.log(LOG_DEBUG, v...) -} - -func (l *logger) Debugf(format string, v ...interface{}) { - l.logf(LOG_DEBUG, format, v...) -} - -func (l *logger) Info(v ...interface{}) { - l.log(LOG_INFO, v...) -} - -func (l *logger) Infof(format string, v ...interface{}) { - l.logf(LOG_INFO, format, v...) -} - -func StringToLogLevel(level string) LogLevel { - switch level { - case "fatal": - return LOG_LEVEL_FATAL - case "error": - return LOG_LEVEL_ERROR - case "warn": - return LOG_LEVEL_WARN - case "warning": - return LOG_LEVEL_WARN - case "debug": - return LOG_LEVEL_DEBUG - case "info": - return LOG_LEVEL_INFO - } - return LOG_LEVEL_ALL -} - -func LogTypeToString(t LogType) (string, string) { - switch t { - case LOG_FATAL: - return "fatal", "[0;31" - case LOG_ERROR: - return "error", "[0;31" - case LOG_WARNING: - return "warning", "[0;33" - case LOG_DEBUG: - return "debug", "[0;36" - case LOG_INFO: - return "info", "[0;37" - } - return "unknown", "[0;37" -} - -func genDayTime(t time.Time) string { - return t.Format(FORMAT_TIME_DAY) -} - -func genHourTime(t time.Time) string { - return t.Format(FORMAT_TIME_HOUR) -} - -func New() *logger { - return Newlogger(os.Stderr, "") -} - -func Newlogger(w io.Writer, prefix string) *logger { - return &logger{_log: log.New(w, prefix, LstdFlags), level: LOG_LEVEL_ALL, highlighting: true} -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/benchmark.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/benchmark.go deleted file mode 100644 index 46ea9dc..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/benchmark.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) 2012 The Go Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package check - -import ( - "fmt" - "runtime" - "time" -) - -var memStats runtime.MemStats - -// testingB is a type passed to Benchmark functions to manage benchmark -// timing and to specify the number of iterations to run. -type timer struct { - start time.Time // Time test or benchmark started - duration time.Duration - N int - bytes int64 - timerOn bool - benchTime time.Duration - // The initial states of memStats.Mallocs and memStats.TotalAlloc. - startAllocs uint64 - startBytes uint64 - // The net total of this test after being run. - netAllocs uint64 - netBytes uint64 -} - -// StartTimer starts timing a test. This function is called automatically -// before a benchmark starts, but it can also used to resume timing after -// a call to StopTimer. -func (c *C) StartTimer() { - if !c.timerOn { - c.start = time.Now() - c.timerOn = true - - runtime.ReadMemStats(&memStats) - c.startAllocs = memStats.Mallocs - c.startBytes = memStats.TotalAlloc - } -} - -// StopTimer stops timing a test. This can be used to pause the timer -// while performing complex initialization that you don't -// want to measure. -func (c *C) StopTimer() { - if c.timerOn { - c.duration += time.Now().Sub(c.start) - c.timerOn = false - runtime.ReadMemStats(&memStats) - c.netAllocs += memStats.Mallocs - c.startAllocs - c.netBytes += memStats.TotalAlloc - c.startBytes - } -} - -// ResetTimer sets the elapsed benchmark time to zero. -// It does not affect whether the timer is running. -func (c *C) ResetTimer() { - if c.timerOn { - c.start = time.Now() - runtime.ReadMemStats(&memStats) - c.startAllocs = memStats.Mallocs - c.startBytes = memStats.TotalAlloc - } - c.duration = 0 - c.netAllocs = 0 - c.netBytes = 0 -} - -// SetBytes informs the number of bytes that the benchmark processes -// on each iteration. If this is called in a benchmark it will also -// report MB/s. -func (c *C) SetBytes(n int64) { - c.bytes = n -} - -func (c *C) nsPerOp() int64 { - if c.N <= 0 { - return 0 - } - return c.duration.Nanoseconds() / int64(c.N) -} - -func (c *C) mbPerSec() float64 { - if c.bytes <= 0 || c.duration <= 0 || c.N <= 0 { - return 0 - } - return (float64(c.bytes) * float64(c.N) / 1e6) / c.duration.Seconds() -} - -func (c *C) timerString() string { - if c.N <= 0 { - return fmt.Sprintf("%3.3fs", float64(c.duration.Nanoseconds())/1e9) - } - mbs := c.mbPerSec() - mb := "" - if mbs != 0 { - mb = fmt.Sprintf("\t%7.2f MB/s", mbs) - } - nsop := c.nsPerOp() - ns := fmt.Sprintf("%10d ns/op", nsop) - if c.N > 0 && nsop < 100 { - // The format specifiers here make sure that - // the ones digits line up for all three possible formats. - if nsop < 10 { - ns = fmt.Sprintf("%13.2f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) - } else { - ns = fmt.Sprintf("%12.1f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) - } - } - memStats := "" - if c.benchMem { - allocedBytes := fmt.Sprintf("%8d B/op", int64(c.netBytes)/int64(c.N)) - allocs := fmt.Sprintf("%8d allocs/op", int64(c.netAllocs)/int64(c.N)) - memStats = fmt.Sprintf("\t%s\t%s", allocedBytes, allocs) - } - return fmt.Sprintf("%8d\t%s%s%s", c.N, ns, mb, memStats) -} - -func min(x, y int) int { - if x > y { - return y - } - return x -} - -func max(x, y int) int { - if x < y { - return y - } - return x -} - -// roundDown10 rounds a number down to the nearest power of 10. -func roundDown10(n int) int { - var tens = 0 - // tens = floor(log_10(n)) - for n > 10 { - n = n / 10 - tens++ - } - // result = 10^tens - result := 1 - for i := 0; i < tens; i++ { - result *= 10 - } - return result -} - -// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. -func roundUp(n int) int { - base := roundDown10(n) - if n < (2 * base) { - return 2 * base - } - if n < (5 * base) { - return 5 * base - } - return 10 * base -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/check.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/check.go deleted file mode 100644 index c99392a..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/check.go +++ /dev/null @@ -1,954 +0,0 @@ -// Package check is a rich testing extension for Go's testing package. -// -// For details about the project, see: -// -// http://labix.org/gocheck -// -package check - -import ( - "bytes" - "errors" - "fmt" - "io" - "math/rand" - "os" - "path" - "path/filepath" - "reflect" - "regexp" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -// ----------------------------------------------------------------------- -// Internal type which deals with suite method calling. - -const ( - fixtureKd = iota - testKd -) - -type funcKind int - -const ( - succeededSt = iota - failedSt - skippedSt - panickedSt - fixturePanickedSt - missedSt -) - -type funcStatus uint32 - -// A method value can't reach its own Method structure. -type methodType struct { - reflect.Value - Info reflect.Method -} - -func newMethod(receiver reflect.Value, i int) *methodType { - return &methodType{receiver.Method(i), receiver.Type().Method(i)} -} - -func (method *methodType) PC() uintptr { - return method.Info.Func.Pointer() -} - -func (method *methodType) suiteName() string { - t := method.Info.Type.In(0) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - return t.Name() -} - -func (method *methodType) String() string { - return method.suiteName() + "." + method.Info.Name -} - -func (method *methodType) matches(re *regexp.Regexp) bool { - return (re.MatchString(method.Info.Name) || - re.MatchString(method.suiteName()) || - re.MatchString(method.String())) -} - -type C struct { - method *methodType - kind funcKind - testName string - _status funcStatus - logb *logger - logw io.Writer - done chan *C - reason string - mustFail bool - tempDir *tempDir - benchMem bool - startTime time.Time - timer -} - -func (c *C) status() funcStatus { - return funcStatus(atomic.LoadUint32((*uint32)(&c._status))) -} - -func (c *C) setStatus(s funcStatus) { - atomic.StoreUint32((*uint32)(&c._status), uint32(s)) -} - -func (c *C) stopNow() { - runtime.Goexit() -} - -// logger is a concurrency safe byte.Buffer -type logger struct { - sync.Mutex - writer bytes.Buffer -} - -func (l *logger) Write(buf []byte) (int, error) { - l.Lock() - defer l.Unlock() - return l.writer.Write(buf) -} - -func (l *logger) WriteTo(w io.Writer) (int64, error) { - l.Lock() - defer l.Unlock() - return l.writer.WriteTo(w) -} - -func (l *logger) String() string { - l.Lock() - defer l.Unlock() - return l.writer.String() -} - -// ----------------------------------------------------------------------- -// Handling of temporary files and directories. - -type tempDir struct { - sync.Mutex - path string - counter int -} - -func (td *tempDir) newPath() string { - td.Lock() - defer td.Unlock() - if td.path == "" { - var err error - for i := 0; i != 100; i++ { - path := fmt.Sprintf("%s%ccheck-%d", os.TempDir(), os.PathSeparator, rand.Int()) - if err = os.Mkdir(path, 0700); err == nil { - td.path = path - break - } - } - if td.path == "" { - panic("Couldn't create temporary directory: " + err.Error()) - } - } - result := filepath.Join(td.path, strconv.Itoa(td.counter)) - td.counter += 1 - return result -} - -func (td *tempDir) removeAll() { - td.Lock() - defer td.Unlock() - if td.path != "" { - err := os.RemoveAll(td.path) - if err != nil { - fmt.Fprintf(os.Stderr, "WARNING: Error cleaning up temporaries: "+err.Error()) - } - } -} - -// Create a new temporary directory which is automatically removed after -// the suite finishes running. -func (c *C) MkDir() string { - path := c.tempDir.newPath() - if err := os.Mkdir(path, 0700); err != nil { - panic(fmt.Sprintf("Couldn't create temporary directory %s: %s", path, err.Error())) - } - return path -} - -// ----------------------------------------------------------------------- -// Low-level logging functions. - -func (c *C) log(args ...interface{}) { - c.writeLog([]byte(fmt.Sprint(args...) + "\n")) -} - -func (c *C) logf(format string, args ...interface{}) { - c.writeLog([]byte(fmt.Sprintf(format+"\n", args...))) -} - -func (c *C) logNewLine() { - c.writeLog([]byte{'\n'}) -} - -func (c *C) writeLog(buf []byte) { - c.logb.Write(buf) - if c.logw != nil { - c.logw.Write(buf) - } -} - -func hasStringOrError(x interface{}) (ok bool) { - _, ok = x.(fmt.Stringer) - if ok { - return - } - _, ok = x.(error) - return -} - -func (c *C) logValue(label string, value interface{}) { - if label == "" { - if hasStringOrError(value) { - c.logf("... %#v (%q)", value, value) - } else { - c.logf("... %#v", value) - } - } else if value == nil { - c.logf("... %s = nil", label) - } else { - if hasStringOrError(value) { - fv := fmt.Sprintf("%#v", value) - qv := fmt.Sprintf("%q", value) - if fv != qv { - c.logf("... %s %s = %s (%s)", label, reflect.TypeOf(value), fv, qv) - return - } - } - if s, ok := value.(string); ok && isMultiLine(s) { - c.logf(`... %s %s = "" +`, label, reflect.TypeOf(value)) - c.logMultiLine(s) - } else { - c.logf("... %s %s = %#v", label, reflect.TypeOf(value), value) - } - } -} - -func (c *C) logMultiLine(s string) { - b := make([]byte, 0, len(s)*2) - i := 0 - n := len(s) - for i < n { - j := i + 1 - for j < n && s[j-1] != '\n' { - j++ - } - b = append(b, "... "...) - b = strconv.AppendQuote(b, s[i:j]) - if j < n { - b = append(b, " +"...) - } - b = append(b, '\n') - i = j - } - c.writeLog(b) -} - -func isMultiLine(s string) bool { - for i := 0; i+1 < len(s); i++ { - if s[i] == '\n' { - return true - } - } - return false -} - -func (c *C) logString(issue string) { - c.log("... ", issue) -} - -func (c *C) logCaller(skip int) { - // This is a bit heavier than it ought to be. - skip += 1 // Our own frame. - pc, callerFile, callerLine, ok := runtime.Caller(skip) - if !ok { - return - } - var testFile string - var testLine int - testFunc := runtime.FuncForPC(c.method.PC()) - if runtime.FuncForPC(pc) != testFunc { - for { - skip += 1 - if pc, file, line, ok := runtime.Caller(skip); ok { - // Note that the test line may be different on - // distinct calls for the same test. Showing - // the "internal" line is helpful when debugging. - if runtime.FuncForPC(pc) == testFunc { - testFile, testLine = file, line - break - } - } else { - break - } - } - } - if testFile != "" && (testFile != callerFile || testLine != callerLine) { - c.logCode(testFile, testLine) - } - c.logCode(callerFile, callerLine) -} - -func (c *C) logCode(path string, line int) { - c.logf("%s:%d:", nicePath(path), line) - code, err := printLine(path, line) - if code == "" { - code = "..." // XXX Open the file and take the raw line. - if err != nil { - code += err.Error() - } - } - c.log(indent(code, " ")) -} - -var valueGo = filepath.Join("reflect", "value.go") -var asmGo = filepath.Join("runtime", "asm_") - -func (c *C) logPanic(skip int, value interface{}) { - skip++ // Our own frame. - initialSkip := skip - for ; ; skip++ { - if pc, file, line, ok := runtime.Caller(skip); ok { - if skip == initialSkip { - c.logf("... Panic: %s (PC=0x%X)\n", value, pc) - } - name := niceFuncName(pc) - path := nicePath(file) - if strings.Contains(path, "/gopkg.in/check.v") { - continue - } - if name == "Value.call" && strings.HasSuffix(path, valueGo) { - continue - } - if (name == "call16" || name == "call32") && strings.Contains(path, asmGo) { - continue - } - c.logf("%s:%d\n in %s", nicePath(file), line, name) - } else { - break - } - } -} - -func (c *C) logSoftPanic(issue string) { - c.log("... Panic: ", issue) -} - -func (c *C) logArgPanic(method *methodType, expectedType string) { - c.logf("... Panic: %s argument should be %s", - niceFuncName(method.PC()), expectedType) -} - -// ----------------------------------------------------------------------- -// Some simple formatting helpers. - -var initWD, initWDErr = os.Getwd() - -func init() { - if initWDErr == nil { - initWD = strings.Replace(initWD, "\\", "/", -1) + "/" - } -} - -func nicePath(path string) string { - if initWDErr == nil { - if strings.HasPrefix(path, initWD) { - return path[len(initWD):] - } - } - return path -} - -func niceFuncPath(pc uintptr) string { - function := runtime.FuncForPC(pc) - if function != nil { - filename, line := function.FileLine(pc) - return fmt.Sprintf("%s:%d", nicePath(filename), line) - } - return "" -} - -func niceFuncName(pc uintptr) string { - function := runtime.FuncForPC(pc) - if function != nil { - name := path.Base(function.Name()) - if i := strings.Index(name, "."); i > 0 { - name = name[i+1:] - } - if strings.HasPrefix(name, "(*") { - if i := strings.Index(name, ")"); i > 0 { - name = name[2:i] + name[i+1:] - } - } - if i := strings.LastIndex(name, ".*"); i != -1 { - name = name[:i] + "." + name[i+2:] - } - if i := strings.LastIndex(name, "·"); i != -1 { - name = name[:i] + "." + name[i+2:] - } - return name - } - return "" -} - -// ----------------------------------------------------------------------- -// Result tracker to aggregate call results. - -type Result struct { - Succeeded int - Failed int - Skipped int - Panicked int - FixturePanicked int - ExpectedFailures int - Missed int // Not even tried to run, related to a panic in the fixture. - RunError error // Houston, we've got a problem. - WorkDir string // If KeepWorkDir is true -} - -type resultTracker struct { - result Result - _lastWasProblem bool - _waiting int - _missed int - _expectChan chan *C - _doneChan chan *C - _stopChan chan bool -} - -func newResultTracker() *resultTracker { - return &resultTracker{_expectChan: make(chan *C), // Synchronous - _doneChan: make(chan *C, 32), // Asynchronous - _stopChan: make(chan bool)} // Synchronous -} - -func (tracker *resultTracker) start() { - go tracker._loopRoutine() -} - -func (tracker *resultTracker) waitAndStop() { - <-tracker._stopChan -} - -func (tracker *resultTracker) expectCall(c *C) { - tracker._expectChan <- c -} - -func (tracker *resultTracker) callDone(c *C) { - tracker._doneChan <- c -} - -func (tracker *resultTracker) _loopRoutine() { - for { - var c *C - if tracker._waiting > 0 { - // Calls still running. Can't stop. - select { - // XXX Reindent this (not now to make diff clear) - case c = <-tracker._expectChan: - tracker._waiting += 1 - case c = <-tracker._doneChan: - tracker._waiting -= 1 - switch c.status() { - case succeededSt: - if c.kind == testKd { - if c.mustFail { - tracker.result.ExpectedFailures++ - } else { - tracker.result.Succeeded++ - } - } - case failedSt: - tracker.result.Failed++ - case panickedSt: - if c.kind == fixtureKd { - tracker.result.FixturePanicked++ - } else { - tracker.result.Panicked++ - } - case fixturePanickedSt: - // Track it as missed, since the panic - // was on the fixture, not on the test. - tracker.result.Missed++ - case missedSt: - tracker.result.Missed++ - case skippedSt: - if c.kind == testKd { - tracker.result.Skipped++ - } - } - } - } else { - // No calls. Can stop, but no done calls here. - select { - case tracker._stopChan <- true: - return - case c = <-tracker._expectChan: - tracker._waiting += 1 - case c = <-tracker._doneChan: - panic("Tracker got an unexpected done call.") - } - } - } -} - -// ----------------------------------------------------------------------- -// The underlying suite runner. - -type suiteRunner struct { - suite interface{} - setUpSuite, tearDownSuite *methodType - setUpTest, tearDownTest *methodType - tests []*methodType - tracker *resultTracker - tempDir *tempDir - keepDir bool - output *outputWriter - reportedProblemLast bool - benchTime time.Duration - benchMem bool -} - -type RunConf struct { - Output io.Writer - Stream bool - Verbose bool - Filter string - Benchmark bool - BenchmarkTime time.Duration // Defaults to 1 second - BenchmarkMem bool - KeepWorkDir bool -} - -// Create a new suiteRunner able to run all methods in the given suite. -func newSuiteRunner(suite interface{}, runConf *RunConf) *suiteRunner { - var conf RunConf - if runConf != nil { - conf = *runConf - } - if conf.Output == nil { - conf.Output = os.Stdout - } - if conf.Benchmark { - conf.Verbose = true - } - - suiteType := reflect.TypeOf(suite) - suiteNumMethods := suiteType.NumMethod() - suiteValue := reflect.ValueOf(suite) - - runner := &suiteRunner{ - suite: suite, - output: newOutputWriter(conf.Output, conf.Stream, conf.Verbose), - tracker: newResultTracker(), - benchTime: conf.BenchmarkTime, - benchMem: conf.BenchmarkMem, - tempDir: &tempDir{}, - keepDir: conf.KeepWorkDir, - tests: make([]*methodType, 0, suiteNumMethods), - } - if runner.benchTime == 0 { - runner.benchTime = 1 * time.Second - } - - var filterRegexp *regexp.Regexp - if conf.Filter != "" { - if regexp, err := regexp.Compile(conf.Filter); err != nil { - msg := "Bad filter expression: " + err.Error() - runner.tracker.result.RunError = errors.New(msg) - return runner - } else { - filterRegexp = regexp - } - } - - for i := 0; i != suiteNumMethods; i++ { - method := newMethod(suiteValue, i) - switch method.Info.Name { - case "SetUpSuite": - runner.setUpSuite = method - case "TearDownSuite": - runner.tearDownSuite = method - case "SetUpTest": - runner.setUpTest = method - case "TearDownTest": - runner.tearDownTest = method - default: - prefix := "Test" - if conf.Benchmark { - prefix = "Benchmark" - } - if !strings.HasPrefix(method.Info.Name, prefix) { - continue - } - if filterRegexp == nil || method.matches(filterRegexp) { - runner.tests = append(runner.tests, method) - } - } - } - return runner -} - -// Run all methods in the given suite. -func (runner *suiteRunner) run() *Result { - if runner.tracker.result.RunError == nil && len(runner.tests) > 0 { - runner.tracker.start() - if runner.checkFixtureArgs() { - c := runner.runFixture(runner.setUpSuite, "", nil) - if c == nil || c.status() == succeededSt { - for i := 0; i != len(runner.tests); i++ { - c := runner.runTest(runner.tests[i]) - if c.status() == fixturePanickedSt { - runner.skipTests(missedSt, runner.tests[i+1:]) - break - } - } - } else if c != nil && c.status() == skippedSt { - runner.skipTests(skippedSt, runner.tests) - } else { - runner.skipTests(missedSt, runner.tests) - } - runner.runFixture(runner.tearDownSuite, "", nil) - } else { - runner.skipTests(missedSt, runner.tests) - } - runner.tracker.waitAndStop() - if runner.keepDir { - runner.tracker.result.WorkDir = runner.tempDir.path - } else { - runner.tempDir.removeAll() - } - } - return &runner.tracker.result -} - -// Create a call object with the given suite method, and fork a -// goroutine with the provided dispatcher for running it. -func (runner *suiteRunner) forkCall(method *methodType, kind funcKind, testName string, logb *logger, dispatcher func(c *C)) *C { - var logw io.Writer - if runner.output.Stream { - logw = runner.output - } - if logb == nil { - logb = new(logger) - } - c := &C{ - method: method, - kind: kind, - testName: testName, - logb: logb, - logw: logw, - tempDir: runner.tempDir, - done: make(chan *C, 1), - timer: timer{benchTime: runner.benchTime}, - startTime: time.Now(), - benchMem: runner.benchMem, - } - runner.tracker.expectCall(c) - go (func() { - runner.reportCallStarted(c) - defer runner.callDone(c) - dispatcher(c) - })() - return c -} - -// Same as forkCall(), but wait for call to finish before returning. -func (runner *suiteRunner) runFunc(method *methodType, kind funcKind, testName string, logb *logger, dispatcher func(c *C)) *C { - c := runner.forkCall(method, kind, testName, logb, dispatcher) - <-c.done - return c -} - -// Handle a finished call. If there were any panics, update the call status -// accordingly. Then, mark the call as done and report to the tracker. -func (runner *suiteRunner) callDone(c *C) { - value := recover() - if value != nil { - switch v := value.(type) { - case *fixturePanic: - if v.status == skippedSt { - c.setStatus(skippedSt) - } else { - c.logSoftPanic("Fixture has panicked (see related PANIC)") - c.setStatus(fixturePanickedSt) - } - default: - c.logPanic(1, value) - c.setStatus(panickedSt) - } - } - if c.mustFail { - switch c.status() { - case failedSt: - c.setStatus(succeededSt) - case succeededSt: - c.setStatus(failedSt) - c.logString("Error: Test succeeded, but was expected to fail") - c.logString("Reason: " + c.reason) - } - } - - runner.reportCallDone(c) - c.done <- c -} - -// Runs a fixture call synchronously. The fixture will still be run in a -// goroutine like all suite methods, but this method will not return -// while the fixture goroutine is not done, because the fixture must be -// run in a desired order. -func (runner *suiteRunner) runFixture(method *methodType, testName string, logb *logger) *C { - if method != nil { - c := runner.runFunc(method, fixtureKd, testName, logb, func(c *C) { - c.ResetTimer() - c.StartTimer() - defer c.StopTimer() - c.method.Call([]reflect.Value{reflect.ValueOf(c)}) - }) - return c - } - return nil -} - -// Run the fixture method with runFixture(), but panic with a fixturePanic{} -// in case the fixture method panics. This makes it easier to track the -// fixture panic together with other call panics within forkTest(). -func (runner *suiteRunner) runFixtureWithPanic(method *methodType, testName string, logb *logger, skipped *bool) *C { - if skipped != nil && *skipped { - return nil - } - c := runner.runFixture(method, testName, logb) - if c != nil && c.status() != succeededSt { - if skipped != nil { - *skipped = c.status() == skippedSt - } - panic(&fixturePanic{c.status(), method}) - } - return c -} - -type fixturePanic struct { - status funcStatus - method *methodType -} - -// Run the suite test method, together with the test-specific fixture, -// asynchronously. -func (runner *suiteRunner) forkTest(method *methodType) *C { - testName := method.String() - return runner.forkCall(method, testKd, testName, nil, func(c *C) { - var skipped bool - defer runner.runFixtureWithPanic(runner.tearDownTest, testName, nil, &skipped) - defer c.StopTimer() - benchN := 1 - for { - runner.runFixtureWithPanic(runner.setUpTest, testName, c.logb, &skipped) - mt := c.method.Type() - if mt.NumIn() != 1 || mt.In(0) != reflect.TypeOf(c) { - // Rather than a plain panic, provide a more helpful message when - // the argument type is incorrect. - c.setStatus(panickedSt) - c.logArgPanic(c.method, "*check.C") - return - } - if strings.HasPrefix(c.method.Info.Name, "Test") { - c.ResetTimer() - c.StartTimer() - c.method.Call([]reflect.Value{reflect.ValueOf(c)}) - return - } - if !strings.HasPrefix(c.method.Info.Name, "Benchmark") { - panic("unexpected method prefix: " + c.method.Info.Name) - } - - runtime.GC() - c.N = benchN - c.ResetTimer() - c.StartTimer() - c.method.Call([]reflect.Value{reflect.ValueOf(c)}) - c.StopTimer() - if c.status() != succeededSt || c.duration >= c.benchTime || benchN >= 1e9 { - return - } - perOpN := int(1e9) - if c.nsPerOp() != 0 { - perOpN = int(c.benchTime.Nanoseconds() / c.nsPerOp()) - } - - // Logic taken from the stock testing package: - // - Run more iterations than we think we'll need for a second (1.5x). - // - Don't grow too fast in case we had timing errors previously. - // - Be sure to run at least one more than last time. - benchN = max(min(perOpN+perOpN/2, 100*benchN), benchN+1) - benchN = roundUp(benchN) - - skipped = true // Don't run the deferred one if this panics. - runner.runFixtureWithPanic(runner.tearDownTest, testName, nil, nil) - skipped = false - } - }) -} - -// Same as forkTest(), but wait for the test to finish before returning. -func (runner *suiteRunner) runTest(method *methodType) *C { - c := runner.forkTest(method) - <-c.done - return c -} - -// Helper to mark tests as skipped or missed. A bit heavy for what -// it does, but it enables homogeneous handling of tracking, including -// nice verbose output. -func (runner *suiteRunner) skipTests(status funcStatus, methods []*methodType) { - for _, method := range methods { - runner.runFunc(method, testKd, "", nil, func(c *C) { - c.setStatus(status) - }) - } -} - -// Verify if the fixture arguments are *check.C. In case of errors, -// log the error as a panic in the fixture method call, and return false. -func (runner *suiteRunner) checkFixtureArgs() bool { - succeeded := true - argType := reflect.TypeOf(&C{}) - for _, method := range []*methodType{runner.setUpSuite, runner.tearDownSuite, runner.setUpTest, runner.tearDownTest} { - if method != nil { - mt := method.Type() - if mt.NumIn() != 1 || mt.In(0) != argType { - succeeded = false - runner.runFunc(method, fixtureKd, "", nil, func(c *C) { - c.logArgPanic(method, "*check.C") - c.setStatus(panickedSt) - }) - } - } - } - return succeeded -} - -func (runner *suiteRunner) reportCallStarted(c *C) { - runner.output.WriteCallStarted("START", c) -} - -func (runner *suiteRunner) reportCallDone(c *C) { - runner.tracker.callDone(c) - switch c.status() { - case succeededSt: - if c.mustFail { - runner.output.WriteCallSuccess("FAIL EXPECTED", c) - } else { - runner.output.WriteCallSuccess("PASS", c) - } - case skippedSt: - runner.output.WriteCallSuccess("SKIP", c) - case failedSt: - runner.output.WriteCallProblem("FAIL", c) - case panickedSt: - runner.output.WriteCallProblem("PANIC", c) - case fixturePanickedSt: - // That's a testKd call reporting that its fixture - // has panicked. The fixture call which caused the - // panic itself was tracked above. We'll report to - // aid debugging. - runner.output.WriteCallProblem("PANIC", c) - case missedSt: - runner.output.WriteCallSuccess("MISS", c) - } -} - -// ----------------------------------------------------------------------- -// Output writer manages atomic output writing according to settings. - -type outputWriter struct { - m sync.Mutex - writer io.Writer - wroteCallProblemLast bool - Stream bool - Verbose bool -} - -func newOutputWriter(writer io.Writer, stream, verbose bool) *outputWriter { - return &outputWriter{writer: writer, Stream: stream, Verbose: verbose} -} - -func (ow *outputWriter) Write(content []byte) (n int, err error) { - ow.m.Lock() - n, err = ow.writer.Write(content) - ow.m.Unlock() - return -} - -func (ow *outputWriter) WriteCallStarted(label string, c *C) { - if ow.Stream { - header := renderCallHeader(label, c, "", "\n") - ow.m.Lock() - ow.writer.Write([]byte(header)) - ow.m.Unlock() - } -} - -func (ow *outputWriter) WriteCallProblem(label string, c *C) { - var prefix string - if !ow.Stream { - prefix = "\n-----------------------------------" + - "-----------------------------------\n" - } - header := renderCallHeader(label, c, prefix, "\n\n") - ow.m.Lock() - ow.wroteCallProblemLast = true - ow.writer.Write([]byte(header)) - if !ow.Stream { - c.logb.WriteTo(ow.writer) - } - ow.m.Unlock() -} - -func (ow *outputWriter) WriteCallSuccess(label string, c *C) { - if ow.Stream || (ow.Verbose && c.kind == testKd) { - // TODO Use a buffer here. - var suffix string - if c.reason != "" { - suffix = " (" + c.reason + ")" - } - if c.status() == succeededSt { - suffix += "\t" + c.timerString() - } - suffix += "\n" - if ow.Stream { - suffix += "\n" - } - header := renderCallHeader(label, c, "", suffix) - ow.m.Lock() - // Resist temptation of using line as prefix above due to race. - if !ow.Stream && ow.wroteCallProblemLast { - header = "\n-----------------------------------" + - "-----------------------------------\n" + - header - } - ow.wroteCallProblemLast = false - ow.writer.Write([]byte(header)) - ow.m.Unlock() - } -} - -func renderCallHeader(label string, c *C, prefix, suffix string) string { - pc := c.method.PC() - return fmt.Sprintf("%s%s: %s: %s%s", prefix, label, niceFuncPath(pc), - niceFuncName(pc), suffix) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers.go deleted file mode 100644 index bac3387..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers.go +++ /dev/null @@ -1,458 +0,0 @@ -package check - -import ( - "fmt" - "reflect" - "regexp" -) - -// ----------------------------------------------------------------------- -// CommentInterface and Commentf helper, to attach extra information to checks. - -type comment struct { - format string - args []interface{} -} - -// Commentf returns an infomational value to use with Assert or Check calls. -// If the checker test fails, the provided arguments will be passed to -// fmt.Sprintf, and will be presented next to the logged failure. -// -// For example: -// -// c.Assert(v, Equals, 42, Commentf("Iteration #%d failed.", i)) -// -// Note that if the comment is constant, a better option is to -// simply use a normal comment right above or next to the line, as -// it will also get printed with any errors: -// -// c.Assert(l, Equals, 8192) // Ensure buffer size is correct (bug #123) -// -func Commentf(format string, args ...interface{}) CommentInterface { - return &comment{format, args} -} - -// CommentInterface must be implemented by types that attach extra -// information to failed checks. See the Commentf function for details. -type CommentInterface interface { - CheckCommentString() string -} - -func (c *comment) CheckCommentString() string { - return fmt.Sprintf(c.format, c.args...) -} - -// ----------------------------------------------------------------------- -// The Checker interface. - -// The Checker interface must be provided by checkers used with -// the Assert and Check verification methods. -type Checker interface { - Info() *CheckerInfo - Check(params []interface{}, names []string) (result bool, error string) -} - -// See the Checker interface. -type CheckerInfo struct { - Name string - Params []string -} - -func (info *CheckerInfo) Info() *CheckerInfo { - return info -} - -// ----------------------------------------------------------------------- -// Not checker logic inverter. - -// The Not checker inverts the logic of the provided checker. The -// resulting checker will succeed where the original one failed, and -// vice-versa. -// -// For example: -// -// c.Assert(a, Not(Equals), b) -// -func Not(checker Checker) Checker { - return ¬Checker{checker} -} - -type notChecker struct { - sub Checker -} - -func (checker *notChecker) Info() *CheckerInfo { - info := *checker.sub.Info() - info.Name = "Not(" + info.Name + ")" - return &info -} - -func (checker *notChecker) Check(params []interface{}, names []string) (result bool, error string) { - result, error = checker.sub.Check(params, names) - result = !result - return -} - -// ----------------------------------------------------------------------- -// IsNil checker. - -type isNilChecker struct { - *CheckerInfo -} - -// The IsNil checker tests whether the obtained value is nil. -// -// For example: -// -// c.Assert(err, IsNil) -// -var IsNil Checker = &isNilChecker{ - &CheckerInfo{Name: "IsNil", Params: []string{"value"}}, -} - -func (checker *isNilChecker) Check(params []interface{}, names []string) (result bool, error string) { - return isNil(params[0]), "" -} - -func isNil(obtained interface{}) (result bool) { - if obtained == nil { - result = true - } else { - switch v := reflect.ValueOf(obtained); v.Kind() { - case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: - return v.IsNil() - } - } - return -} - -// ----------------------------------------------------------------------- -// NotNil checker. Alias for Not(IsNil), since it's so common. - -type notNilChecker struct { - *CheckerInfo -} - -// The NotNil checker verifies that the obtained value is not nil. -// -// For example: -// -// c.Assert(iface, NotNil) -// -// This is an alias for Not(IsNil), made available since it's a -// fairly common check. -// -var NotNil Checker = ¬NilChecker{ - &CheckerInfo{Name: "NotNil", Params: []string{"value"}}, -} - -func (checker *notNilChecker) Check(params []interface{}, names []string) (result bool, error string) { - return !isNil(params[0]), "" -} - -// ----------------------------------------------------------------------- -// Equals checker. - -type equalsChecker struct { - *CheckerInfo -} - -// The Equals checker verifies that the obtained value is equal to -// the expected value, according to usual Go semantics for ==. -// -// For example: -// -// c.Assert(value, Equals, 42) -// -var Equals Checker = &equalsChecker{ - &CheckerInfo{Name: "Equals", Params: []string{"obtained", "expected"}}, -} - -func (checker *equalsChecker) Check(params []interface{}, names []string) (result bool, error string) { - defer func() { - if v := recover(); v != nil { - result = false - error = fmt.Sprint(v) - } - }() - return params[0] == params[1], "" -} - -// ----------------------------------------------------------------------- -// DeepEquals checker. - -type deepEqualsChecker struct { - *CheckerInfo -} - -// The DeepEquals checker verifies that the obtained value is deep-equal to -// the expected value. The check will work correctly even when facing -// slices, interfaces, and values of different types (which always fail -// the test). -// -// For example: -// -// c.Assert(value, DeepEquals, 42) -// c.Assert(array, DeepEquals, []string{"hi", "there"}) -// -var DeepEquals Checker = &deepEqualsChecker{ - &CheckerInfo{Name: "DeepEquals", Params: []string{"obtained", "expected"}}, -} - -func (checker *deepEqualsChecker) Check(params []interface{}, names []string) (result bool, error string) { - return reflect.DeepEqual(params[0], params[1]), "" -} - -// ----------------------------------------------------------------------- -// HasLen checker. - -type hasLenChecker struct { - *CheckerInfo -} - -// The HasLen checker verifies that the obtained value has the -// provided length. In many cases this is superior to using Equals -// in conjuction with the len function because in case the check -// fails the value itself will be printed, instead of its length, -// providing more details for figuring the problem. -// -// For example: -// -// c.Assert(list, HasLen, 5) -// -var HasLen Checker = &hasLenChecker{ - &CheckerInfo{Name: "HasLen", Params: []string{"obtained", "n"}}, -} - -func (checker *hasLenChecker) Check(params []interface{}, names []string) (result bool, error string) { - n, ok := params[1].(int) - if !ok { - return false, "n must be an int" - } - value := reflect.ValueOf(params[0]) - switch value.Kind() { - case reflect.Map, reflect.Array, reflect.Slice, reflect.Chan, reflect.String: - default: - return false, "obtained value type has no length" - } - return value.Len() == n, "" -} - -// ----------------------------------------------------------------------- -// ErrorMatches checker. - -type errorMatchesChecker struct { - *CheckerInfo -} - -// The ErrorMatches checker verifies that the error value -// is non nil and matches the regular expression provided. -// -// For example: -// -// c.Assert(err, ErrorMatches, "perm.*denied") -// -var ErrorMatches Checker = errorMatchesChecker{ - &CheckerInfo{Name: "ErrorMatches", Params: []string{"value", "regex"}}, -} - -func (checker errorMatchesChecker) Check(params []interface{}, names []string) (result bool, errStr string) { - if params[0] == nil { - return false, "Error value is nil" - } - err, ok := params[0].(error) - if !ok { - return false, "Value is not an error" - } - params[0] = err.Error() - names[0] = "error" - return matches(params[0], params[1]) -} - -// ----------------------------------------------------------------------- -// Matches checker. - -type matchesChecker struct { - *CheckerInfo -} - -// The Matches checker verifies that the string provided as the obtained -// value (or the string resulting from obtained.String()) matches the -// regular expression provided. -// -// For example: -// -// c.Assert(err, Matches, "perm.*denied") -// -var Matches Checker = &matchesChecker{ - &CheckerInfo{Name: "Matches", Params: []string{"value", "regex"}}, -} - -func (checker *matchesChecker) Check(params []interface{}, names []string) (result bool, error string) { - return matches(params[0], params[1]) -} - -func matches(value, regex interface{}) (result bool, error string) { - reStr, ok := regex.(string) - if !ok { - return false, "Regex must be a string" - } - valueStr, valueIsStr := value.(string) - if !valueIsStr { - if valueWithStr, valueHasStr := value.(fmt.Stringer); valueHasStr { - valueStr, valueIsStr = valueWithStr.String(), true - } - } - if valueIsStr { - matches, err := regexp.MatchString("^"+reStr+"$", valueStr) - if err != nil { - return false, "Can't compile regex: " + err.Error() - } - return matches, "" - } - return false, "Obtained value is not a string and has no .String()" -} - -// ----------------------------------------------------------------------- -// Panics checker. - -type panicsChecker struct { - *CheckerInfo -} - -// The Panics checker verifies that calling the provided zero-argument -// function will cause a panic which is deep-equal to the provided value. -// -// For example: -// -// c.Assert(func() { f(1, 2) }, Panics, &SomeErrorType{"BOOM"}). -// -// -var Panics Checker = &panicsChecker{ - &CheckerInfo{Name: "Panics", Params: []string{"function", "expected"}}, -} - -func (checker *panicsChecker) Check(params []interface{}, names []string) (result bool, error string) { - f := reflect.ValueOf(params[0]) - if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { - return false, "Function must take zero arguments" - } - defer func() { - // If the function has not panicked, then don't do the check. - if error != "" { - return - } - params[0] = recover() - names[0] = "panic" - result = reflect.DeepEqual(params[0], params[1]) - }() - f.Call(nil) - return false, "Function has not panicked" -} - -type panicMatchesChecker struct { - *CheckerInfo -} - -// The PanicMatches checker verifies that calling the provided zero-argument -// function will cause a panic with an error value matching -// the regular expression provided. -// -// For example: -// -// c.Assert(func() { f(1, 2) }, PanicMatches, `open.*: no such file or directory`). -// -// -var PanicMatches Checker = &panicMatchesChecker{ - &CheckerInfo{Name: "PanicMatches", Params: []string{"function", "expected"}}, -} - -func (checker *panicMatchesChecker) Check(params []interface{}, names []string) (result bool, errmsg string) { - f := reflect.ValueOf(params[0]) - if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { - return false, "Function must take zero arguments" - } - defer func() { - // If the function has not panicked, then don't do the check. - if errmsg != "" { - return - } - obtained := recover() - names[0] = "panic" - if e, ok := obtained.(error); ok { - params[0] = e.Error() - } else if _, ok := obtained.(string); ok { - params[0] = obtained - } else { - errmsg = "Panic value is not a string or an error" - return - } - result, errmsg = matches(params[0], params[1]) - }() - f.Call(nil) - return false, "Function has not panicked" -} - -// ----------------------------------------------------------------------- -// FitsTypeOf checker. - -type fitsTypeChecker struct { - *CheckerInfo -} - -// The FitsTypeOf checker verifies that the obtained value is -// assignable to a variable with the same type as the provided -// sample value. -// -// For example: -// -// c.Assert(value, FitsTypeOf, int64(0)) -// c.Assert(value, FitsTypeOf, os.Error(nil)) -// -var FitsTypeOf Checker = &fitsTypeChecker{ - &CheckerInfo{Name: "FitsTypeOf", Params: []string{"obtained", "sample"}}, -} - -func (checker *fitsTypeChecker) Check(params []interface{}, names []string) (result bool, error string) { - obtained := reflect.ValueOf(params[0]) - sample := reflect.ValueOf(params[1]) - if !obtained.IsValid() { - return false, "" - } - if !sample.IsValid() { - return false, "Invalid sample value" - } - return obtained.Type().AssignableTo(sample.Type()), "" -} - -// ----------------------------------------------------------------------- -// Implements checker. - -type implementsChecker struct { - *CheckerInfo -} - -// The Implements checker verifies that the obtained value -// implements the interface specified via a pointer to an interface -// variable. -// -// For example: -// -// var e os.Error -// c.Assert(err, Implements, &e) -// -var Implements Checker = &implementsChecker{ - &CheckerInfo{Name: "Implements", Params: []string{"obtained", "ifaceptr"}}, -} - -func (checker *implementsChecker) Check(params []interface{}, names []string) (result bool, error string) { - obtained := reflect.ValueOf(params[0]) - ifaceptr := reflect.ValueOf(params[1]) - if !obtained.IsValid() { - return false, "" - } - if !ifaceptr.IsValid() || ifaceptr.Kind() != reflect.Ptr || ifaceptr.Elem().Kind() != reflect.Interface { - return false, "ifaceptr should be a pointer to an interface variable" - } - return obtained.Type().Implements(ifaceptr.Elem().Type()), "" -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers2.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers2.go deleted file mode 100644 index c09bcdc..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/checkers2.go +++ /dev/null @@ -1,131 +0,0 @@ -// Extensions to the go-check unittest framework. -// -// NOTE: see https://github.com/go-check/check/pull/6 for reasons why these -// checkers live here. -package check - -import ( - "bytes" - "reflect" -) - -// ----------------------------------------------------------------------- -// IsTrue / IsFalse checker. - -type isBoolValueChecker struct { - *CheckerInfo - expected bool -} - -func (checker *isBoolValueChecker) Check( - params []interface{}, - names []string) ( - result bool, - error string) { - - obtained, ok := params[0].(bool) - if !ok { - return false, "Argument to " + checker.Name + " must be bool" - } - - return obtained == checker.expected, "" -} - -// The IsTrue checker verifies that the obtained value is true. -// -// For example: -// -// c.Assert(value, IsTrue) -// -var IsTrue Checker = &isBoolValueChecker{ - &CheckerInfo{Name: "IsTrue", Params: []string{"obtained"}}, - true, -} - -// The IsFalse checker verifies that the obtained value is false. -// -// For example: -// -// c.Assert(value, IsFalse) -// -var IsFalse Checker = &isBoolValueChecker{ - &CheckerInfo{Name: "IsFalse", Params: []string{"obtained"}}, - false, -} - -// ----------------------------------------------------------------------- -// BytesEquals checker. - -type bytesEquals struct{} - -func (b *bytesEquals) Check(params []interface{}, names []string) (bool, string) { - if len(params) != 2 { - return false, "BytesEqual takes 2 bytestring arguments" - } - b1, ok1 := params[0].([]byte) - b2, ok2 := params[1].([]byte) - - if !(ok1 && ok2) { - return false, "Arguments to BytesEqual must both be bytestrings" - } - - return bytes.Equal(b1, b2), "" -} - -func (b *bytesEquals) Info() *CheckerInfo { - return &CheckerInfo{ - Name: "BytesEquals", - Params: []string{"bytes_one", "bytes_two"}, - } -} - -// BytesEquals checker compares two bytes sequence using bytes.Equal. -// -// For example: -// -// c.Assert(b, BytesEquals, []byte("bar")) -// -// Main difference between DeepEquals and BytesEquals is that BytesEquals treats -// `nil` as empty byte sequence while DeepEquals doesn't. -// -// c.Assert(nil, BytesEquals, []byte("")) // succeeds -// c.Assert(nil, DeepEquals, []byte("")) // fails -var BytesEquals = &bytesEquals{} - -// ----------------------------------------------------------------------- -// HasKey checker. - -type hasKey struct{} - -func (h *hasKey) Check(params []interface{}, names []string) (bool, string) { - if len(params) != 2 { - return false, "HasKey takes 2 arguments: a map and a key" - } - - mapValue := reflect.ValueOf(params[0]) - if mapValue.Kind() != reflect.Map { - return false, "First argument to HasKey must be a map" - } - - keyValue := reflect.ValueOf(params[1]) - if !keyValue.Type().AssignableTo(mapValue.Type().Key()) { - return false, "Second argument must be assignable to the map key type" - } - - return mapValue.MapIndex(keyValue).IsValid(), "" -} - -func (h *hasKey) Info() *CheckerInfo { - return &CheckerInfo{ - Name: "HasKey", - Params: []string{"obtained", "key"}, - } -} - -// The HasKey checker verifies that the obtained map contains the given key. -// -// For example: -// -// c.Assert(myMap, HasKey, "foo") -// -var HasKey = &hasKey{} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/compare.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/compare.go deleted file mode 100644 index 7005cba..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/compare.go +++ /dev/null @@ -1,161 +0,0 @@ -package check - -import ( - "bytes" - "fmt" - "reflect" - "time" -) - -type compareFunc func(v1 interface{}, v2 interface{}) (bool, error) - -type valueCompare struct { - Name string - - Func compareFunc - - Operator string -} - -// v1 and v2 must have the same type -// return >0 if v1 > v2 -// return 0 if v1 = v2 -// return <0 if v1 < v2 -// now we only support int, uint, float64, string and []byte comparison -func compare(v1 interface{}, v2 interface{}) (int, error) { - value1 := reflect.ValueOf(v1) - value2 := reflect.ValueOf(v2) - - switch v1.(type) { - case int, int8, int16, int32, int64: - a1 := value1.Int() - a2 := value2.Int() - if a1 > a2 { - return 1, nil - } else if a1 == a2 { - return 0, nil - } - return -1, nil - case uint, uint8, uint16, uint32, uint64: - a1 := value1.Uint() - a2 := value2.Uint() - if a1 > a2 { - return 1, nil - } else if a1 == a2 { - return 0, nil - } - return -1, nil - case float32, float64: - a1 := value1.Float() - a2 := value2.Float() - if a1 > a2 { - return 1, nil - } else if a1 == a2 { - return 0, nil - } - return -1, nil - case string: - a1 := value1.String() - a2 := value2.String() - if a1 > a2 { - return 1, nil - } else if a1 == a2 { - return 0, nil - } - return -1, nil - case []byte: - a1 := value1.Bytes() - a2 := value2.Bytes() - return bytes.Compare(a1, a2), nil - case time.Time: - a1 := v1.(time.Time) - a2 := v2.(time.Time) - if a1.After(a2) { - return 1, nil - } else if a1.Equal(a2) { - return 0, nil - } - return -1, nil - case time.Duration: - a1 := v1.(time.Duration) - a2 := v2.(time.Duration) - if a1 > a2 { - return 1, nil - } else if a1 == a2 { - return 0, nil - } - return -1, nil - default: - return 0, fmt.Errorf("type %T is not supported now", v1) - } -} - -func less(v1 interface{}, v2 interface{}) (bool, error) { - n, err := compare(v1, v2) - if err != nil { - return false, err - } - - return n < 0, nil -} - -func lessEqual(v1 interface{}, v2 interface{}) (bool, error) { - n, err := compare(v1, v2) - if err != nil { - return false, err - } - - return n <= 0, nil -} - -func greater(v1 interface{}, v2 interface{}) (bool, error) { - n, err := compare(v1, v2) - if err != nil { - return false, err - } - - return n > 0, nil -} - -func greaterEqual(v1 interface{}, v2 interface{}) (bool, error) { - n, err := compare(v1, v2) - if err != nil { - return false, err - } - - return n >= 0, nil -} - -func (v *valueCompare) Check(params []interface{}, names []string) (bool, string) { - if len(params) != 2 { - return false, fmt.Sprintf("%s needs 2 arguments", v.Name) - } - - v1 := params[0] - v2 := params[1] - v1Type := reflect.TypeOf(v1) - v2Type := reflect.TypeOf(v2) - - if v1Type.Kind() != v2Type.Kind() { - return false, fmt.Sprintf("%s needs two same type, but %s != %s", v.Name, v1Type.Kind(), v2Type.Kind()) - } - - b, err := v.Func(v1, v2) - if err != nil { - return false, fmt.Sprintf("%s check err %v", v.Name, err) - } - - return b, "" -} - -func (v *valueCompare) Info() *CheckerInfo { - return &CheckerInfo{ - Name: v.Name, - Params: []string{"compare_one", "compare_two"}, - } -} - -var Less = &valueCompare{Name: "Less", Func: less, Operator: "<"} -var LessEqual = &valueCompare{Name: "LessEqual", Func: lessEqual, Operator: "<="} -var Greater = &valueCompare{Name: "Greater", Func: greater, Operator: ">"} -var GreaterEqual = &valueCompare{Name: "GreaterEqual", Func: greaterEqual, Operator: ">="} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/helpers.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/helpers.go deleted file mode 100644 index 58a733b..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/helpers.go +++ /dev/null @@ -1,231 +0,0 @@ -package check - -import ( - "fmt" - "strings" - "time" -) - -// TestName returns the current test name in the form "SuiteName.TestName" -func (c *C) TestName() string { - return c.testName -} - -// ----------------------------------------------------------------------- -// Basic succeeding/failing logic. - -// Failed returns whether the currently running test has already failed. -func (c *C) Failed() bool { - return c.status() == failedSt -} - -// Fail marks the currently running test as failed. -// -// Something ought to have been previously logged so the developer can tell -// what went wrong. The higher level helper functions will fail the test -// and do the logging properly. -func (c *C) Fail() { - c.setStatus(failedSt) -} - -// FailNow marks the currently running test as failed and stops running it. -// Something ought to have been previously logged so the developer can tell -// what went wrong. The higher level helper functions will fail the test -// and do the logging properly. -func (c *C) FailNow() { - c.Fail() - c.stopNow() -} - -// Succeed marks the currently running test as succeeded, undoing any -// previous failures. -func (c *C) Succeed() { - c.setStatus(succeededSt) -} - -// SucceedNow marks the currently running test as succeeded, undoing any -// previous failures, and stops running the test. -func (c *C) SucceedNow() { - c.Succeed() - c.stopNow() -} - -// ExpectFailure informs that the running test is knowingly broken for -// the provided reason. If the test does not fail, an error will be reported -// to raise attention to this fact. This method is useful to temporarily -// disable tests which cover well known problems until a better time to -// fix the problem is found, without forgetting about the fact that a -// failure still exists. -func (c *C) ExpectFailure(reason string) { - if reason == "" { - panic("Missing reason why the test is expected to fail") - } - c.mustFail = true - c.reason = reason -} - -// Skip skips the running test for the provided reason. If run from within -// SetUpTest, the individual test being set up will be skipped, and if run -// from within SetUpSuite, the whole suite is skipped. -func (c *C) Skip(reason string) { - if reason == "" { - panic("Missing reason why the test is being skipped") - } - c.reason = reason - c.setStatus(skippedSt) - c.stopNow() -} - -// ----------------------------------------------------------------------- -// Basic logging. - -// GetTestLog returns the current test error output. -func (c *C) GetTestLog() string { - return c.logb.String() -} - -// Log logs some information into the test error output. -// The provided arguments are assembled together into a string with fmt.Sprint. -func (c *C) Log(args ...interface{}) { - c.log(args...) -} - -// Log logs some information into the test error output. -// The provided arguments are assembled together into a string with fmt.Sprintf. -func (c *C) Logf(format string, args ...interface{}) { - c.logf(format, args...) -} - -// Output enables *C to be used as a logger in functions that require only -// the minimum interface of *log.Logger. -func (c *C) Output(calldepth int, s string) error { - d := time.Now().Sub(c.startTime) - msec := d / time.Millisecond - sec := d / time.Second - min := d / time.Minute - - c.Logf("[LOG] %d:%02d.%03d %s", min, sec%60, msec%1000, s) - return nil -} - -// Error logs an error into the test error output and marks the test as failed. -// The provided arguments are assembled together into a string with fmt.Sprint. -func (c *C) Error(args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) - c.logNewLine() - c.Fail() -} - -// Errorf logs an error into the test error output and marks the test as failed. -// The provided arguments are assembled together into a string with fmt.Sprintf. -func (c *C) Errorf(format string, args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprintf("Error: "+format, args...)) - c.logNewLine() - c.Fail() -} - -// Fatal logs an error into the test error output, marks the test as failed, and -// stops the test execution. The provided arguments are assembled together into -// a string with fmt.Sprint. -func (c *C) Fatal(args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) - c.logNewLine() - c.FailNow() -} - -// Fatlaf logs an error into the test error output, marks the test as failed, and -// stops the test execution. The provided arguments are assembled together into -// a string with fmt.Sprintf. -func (c *C) Fatalf(format string, args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprint("Error: ", fmt.Sprintf(format, args...))) - c.logNewLine() - c.FailNow() -} - -// ----------------------------------------------------------------------- -// Generic checks and assertions based on checkers. - -// Check verifies if the first value matches the expected value according -// to the provided checker. If they do not match, an error is logged, the -// test is marked as failed, and the test execution continues. -// -// Some checkers may not need the expected argument (e.g. IsNil). -// -// Extra arguments provided to the function are logged next to the reported -// problem when the matching fails. -func (c *C) Check(obtained interface{}, checker Checker, args ...interface{}) bool { - return c.internalCheck("Check", obtained, checker, args...) -} - -// Assert ensures that the first value matches the expected value according -// to the provided checker. If they do not match, an error is logged, the -// test is marked as failed, and the test execution stops. -// -// Some checkers may not need the expected argument (e.g. IsNil). -// -// Extra arguments provided to the function are logged next to the reported -// problem when the matching fails. -func (c *C) Assert(obtained interface{}, checker Checker, args ...interface{}) { - if !c.internalCheck("Assert", obtained, checker, args...) { - c.stopNow() - } -} - -func (c *C) internalCheck(funcName string, obtained interface{}, checker Checker, args ...interface{}) bool { - if checker == nil { - c.logCaller(2) - c.logString(fmt.Sprintf("%s(obtained, nil!?, ...):", funcName)) - c.logString("Oops.. you've provided a nil checker!") - c.logNewLine() - c.Fail() - return false - } - - // If the last argument is a bug info, extract it out. - var comment CommentInterface - if len(args) > 0 { - if c, ok := args[len(args)-1].(CommentInterface); ok { - comment = c - args = args[:len(args)-1] - } - } - - params := append([]interface{}{obtained}, args...) - info := checker.Info() - - if len(params) != len(info.Params) { - names := append([]string{info.Params[0], info.Name}, info.Params[1:]...) - c.logCaller(2) - c.logString(fmt.Sprintf("%s(%s):", funcName, strings.Join(names, ", "))) - c.logString(fmt.Sprintf("Wrong number of parameters for %s: want %d, got %d", info.Name, len(names), len(params)+1)) - c.logNewLine() - c.Fail() - return false - } - - // Copy since it may be mutated by Check. - names := append([]string{}, info.Params...) - - // Do the actual check. - result, error := checker.Check(params, names) - if !result || error != "" { - c.logCaller(2) - for i := 0; i != len(params); i++ { - c.logValue(names[i], params[i]) - } - if comment != nil { - c.logString(comment.CheckCommentString()) - } - if error != "" { - c.logString(error) - } - c.logNewLine() - c.Fail() - return false - } - return true -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/printer.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/printer.go deleted file mode 100644 index e0f7557..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/printer.go +++ /dev/null @@ -1,168 +0,0 @@ -package check - -import ( - "bytes" - "go/ast" - "go/parser" - "go/printer" - "go/token" - "os" -) - -func indent(s, with string) (r string) { - eol := true - for i := 0; i != len(s); i++ { - c := s[i] - switch { - case eol && c == '\n' || c == '\r': - case c == '\n' || c == '\r': - eol = true - case eol: - eol = false - s = s[:i] + with + s[i:] - i += len(with) - } - } - return s -} - -func printLine(filename string, line int) (string, error) { - fset := token.NewFileSet() - file, err := os.Open(filename) - if err != nil { - return "", err - } - fnode, err := parser.ParseFile(fset, filename, file, parser.ParseComments) - if err != nil { - return "", err - } - config := &printer.Config{Mode: printer.UseSpaces, Tabwidth: 4} - lp := &linePrinter{fset: fset, fnode: fnode, line: line, config: config} - ast.Walk(lp, fnode) - result := lp.output.Bytes() - // Comments leave \n at the end. - n := len(result) - for n > 0 && result[n-1] == '\n' { - n-- - } - return string(result[:n]), nil -} - -type linePrinter struct { - config *printer.Config - fset *token.FileSet - fnode *ast.File - line int - output bytes.Buffer - stmt ast.Stmt -} - -func (lp *linePrinter) emit() bool { - if lp.stmt != nil { - lp.trim(lp.stmt) - lp.printWithComments(lp.stmt) - lp.stmt = nil - return true - } - return false -} - -func (lp *linePrinter) printWithComments(n ast.Node) { - nfirst := lp.fset.Position(n.Pos()).Line - nlast := lp.fset.Position(n.End()).Line - for _, g := range lp.fnode.Comments { - cfirst := lp.fset.Position(g.Pos()).Line - clast := lp.fset.Position(g.End()).Line - if clast == nfirst-1 && lp.fset.Position(n.Pos()).Column == lp.fset.Position(g.Pos()).Column { - for _, c := range g.List { - lp.output.WriteString(c.Text) - lp.output.WriteByte('\n') - } - } - if cfirst >= nfirst && cfirst <= nlast && n.End() <= g.List[0].Slash { - // The printer will not include the comment if it starts past - // the node itself. Trick it into printing by overlapping the - // slash with the end of the statement. - g.List[0].Slash = n.End() - 1 - } - } - node := &printer.CommentedNode{n, lp.fnode.Comments} - lp.config.Fprint(&lp.output, lp.fset, node) -} - -func (lp *linePrinter) Visit(n ast.Node) (w ast.Visitor) { - if n == nil { - if lp.output.Len() == 0 { - lp.emit() - } - return nil - } - first := lp.fset.Position(n.Pos()).Line - last := lp.fset.Position(n.End()).Line - if first <= lp.line && last >= lp.line { - // Print the innermost statement containing the line. - if stmt, ok := n.(ast.Stmt); ok { - if _, ok := n.(*ast.BlockStmt); !ok { - lp.stmt = stmt - } - } - if first == lp.line && lp.emit() { - return nil - } - return lp - } - return nil -} - -func (lp *linePrinter) trim(n ast.Node) bool { - stmt, ok := n.(ast.Stmt) - if !ok { - return true - } - line := lp.fset.Position(n.Pos()).Line - if line != lp.line { - return false - } - switch stmt := stmt.(type) { - case *ast.IfStmt: - stmt.Body = lp.trimBlock(stmt.Body) - case *ast.SwitchStmt: - stmt.Body = lp.trimBlock(stmt.Body) - case *ast.TypeSwitchStmt: - stmt.Body = lp.trimBlock(stmt.Body) - case *ast.CaseClause: - stmt.Body = lp.trimList(stmt.Body) - case *ast.CommClause: - stmt.Body = lp.trimList(stmt.Body) - case *ast.BlockStmt: - stmt.List = lp.trimList(stmt.List) - } - return true -} - -func (lp *linePrinter) trimBlock(stmt *ast.BlockStmt) *ast.BlockStmt { - if !lp.trim(stmt) { - return lp.emptyBlock(stmt) - } - stmt.Rbrace = stmt.Lbrace - return stmt -} - -func (lp *linePrinter) trimList(stmts []ast.Stmt) []ast.Stmt { - for i := 0; i != len(stmts); i++ { - if !lp.trim(stmts[i]) { - stmts[i] = lp.emptyStmt(stmts[i]) - break - } - } - return stmts -} - -func (lp *linePrinter) emptyStmt(n ast.Node) *ast.ExprStmt { - return &ast.ExprStmt{&ast.Ellipsis{n.Pos(), nil}} -} - -func (lp *linePrinter) emptyBlock(n ast.Node) *ast.BlockStmt { - p := n.Pos() - return &ast.BlockStmt{p, []ast.Stmt{lp.emptyStmt(n)}, p} -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/run.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/run.go deleted file mode 100644 index da8fd79..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/pingcap/check/run.go +++ /dev/null @@ -1,175 +0,0 @@ -package check - -import ( - "bufio" - "flag" - "fmt" - "os" - "testing" - "time" -) - -// ----------------------------------------------------------------------- -// Test suite registry. - -var allSuites []interface{} - -// Suite registers the given value as a test suite to be run. Any methods -// starting with the Test prefix in the given value will be considered as -// a test method. -func Suite(suite interface{}) interface{} { - allSuites = append(allSuites, suite) - return suite -} - -// ----------------------------------------------------------------------- -// Public running interface. - -var ( - oldFilterFlag = flag.String("gocheck.f", "", "Regular expression selecting which tests and/or suites to run") - oldVerboseFlag = flag.Bool("gocheck.v", false, "Verbose mode") - oldStreamFlag = flag.Bool("gocheck.vv", false, "Super verbose mode (disables output caching)") - oldBenchFlag = flag.Bool("gocheck.b", false, "Run benchmarks") - oldBenchTime = flag.Duration("gocheck.btime", 1*time.Second, "approximate run time for each benchmark") - oldListFlag = flag.Bool("gocheck.list", false, "List the names of all tests that will be run") - oldWorkFlag = flag.Bool("gocheck.work", false, "Display and do not remove the test working directory") - - newFilterFlag = flag.String("check.f", "", "Regular expression selecting which tests and/or suites to run") - newVerboseFlag = flag.Bool("check.v", false, "Verbose mode") - newStreamFlag = flag.Bool("check.vv", false, "Super verbose mode (disables output caching)") - newBenchFlag = flag.Bool("check.b", false, "Run benchmarks") - newBenchTime = flag.Duration("check.btime", 1*time.Second, "approximate run time for each benchmark") - newBenchMem = flag.Bool("check.bmem", false, "Report memory benchmarks") - newListFlag = flag.Bool("check.list", false, "List the names of all tests that will be run") - newWorkFlag = flag.Bool("check.work", false, "Display and do not remove the test working directory") -) - -// TestingT runs all test suites registered with the Suite function, -// printing results to stdout, and reporting any failures back to -// the "testing" package. -func TestingT(testingT *testing.T) { - benchTime := *newBenchTime - if benchTime == 1*time.Second { - benchTime = *oldBenchTime - } - conf := &RunConf{ - Filter: *oldFilterFlag + *newFilterFlag, - Verbose: *oldVerboseFlag || *newVerboseFlag, - Stream: *oldStreamFlag || *newStreamFlag, - Benchmark: *oldBenchFlag || *newBenchFlag, - BenchmarkTime: benchTime, - BenchmarkMem: *newBenchMem, - KeepWorkDir: *oldWorkFlag || *newWorkFlag, - } - if *oldListFlag || *newListFlag { - w := bufio.NewWriter(os.Stdout) - for _, name := range ListAll(conf) { - fmt.Fprintln(w, name) - } - w.Flush() - return - } - result := RunAll(conf) - println(result.String()) - if !result.Passed() { - testingT.Fail() - } -} - -// RunAll runs all test suites registered with the Suite function, using the -// provided run configuration. -func RunAll(runConf *RunConf) *Result { - result := Result{} - for _, suite := range allSuites { - result.Add(Run(suite, runConf)) - } - return &result -} - -// Run runs the provided test suite using the provided run configuration. -func Run(suite interface{}, runConf *RunConf) *Result { - runner := newSuiteRunner(suite, runConf) - return runner.run() -} - -// ListAll returns the names of all the test functions registered with the -// Suite function that will be run with the provided run configuration. -func ListAll(runConf *RunConf) []string { - var names []string - for _, suite := range allSuites { - names = append(names, List(suite, runConf)...) - } - return names -} - -// List returns the names of the test functions in the given -// suite that will be run with the provided run configuration. -func List(suite interface{}, runConf *RunConf) []string { - var names []string - runner := newSuiteRunner(suite, runConf) - for _, t := range runner.tests { - names = append(names, t.String()) - } - return names -} - -// ----------------------------------------------------------------------- -// Result methods. - -func (r *Result) Add(other *Result) { - r.Succeeded += other.Succeeded - r.Skipped += other.Skipped - r.Failed += other.Failed - r.Panicked += other.Panicked - r.FixturePanicked += other.FixturePanicked - r.ExpectedFailures += other.ExpectedFailures - r.Missed += other.Missed - if r.WorkDir != "" && other.WorkDir != "" { - r.WorkDir += ":" + other.WorkDir - } else if other.WorkDir != "" { - r.WorkDir = other.WorkDir - } -} - -func (r *Result) Passed() bool { - return (r.Failed == 0 && r.Panicked == 0 && - r.FixturePanicked == 0 && r.Missed == 0 && - r.RunError == nil) -} - -func (r *Result) String() string { - if r.RunError != nil { - return "ERROR: " + r.RunError.Error() - } - - var value string - if r.Failed == 0 && r.Panicked == 0 && r.FixturePanicked == 0 && - r.Missed == 0 { - value = "OK: " - } else { - value = "OOPS: " - } - value += fmt.Sprintf("%d passed", r.Succeeded) - if r.Skipped != 0 { - value += fmt.Sprintf(", %d skipped", r.Skipped) - } - if r.ExpectedFailures != 0 { - value += fmt.Sprintf(", %d expected failures", r.ExpectedFailures) - } - if r.Failed != 0 { - value += fmt.Sprintf(", %d FAILED", r.Failed) - } - if r.Panicked != 0 { - value += fmt.Sprintf(", %d PANICKED", r.Panicked) - } - if r.FixturePanicked != 0 { - value += fmt.Sprintf(", %d FIXTURE-PANICKED", r.FixturePanicked) - } - if r.Missed != 0 { - value += fmt.Sprintf(", %d MISSED", r.Missed) - } - if r.WorkDir != "" { - value += "\nWORK=" + r.WorkDir - } - return value -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/LICENSE b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/LICENSE deleted file mode 100644 index 488357b..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (C) 2013-2016 by Maxim Bublis - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/uuid.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/uuid.go deleted file mode 100644 index 9c7fbaa..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/satori/go.uuid/uuid.go +++ /dev/null @@ -1,488 +0,0 @@ -// Copyright (C) 2013-2015 by Maxim Bublis -// -// Permission is hereby granted, free of charge, to any person obtaining -// a copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to -// permit persons to whom the Software is furnished to do so, subject to -// the following conditions: -// -// The above copyright notice and this permission notice shall be -// included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// Package uuid provides implementation of Universally Unique Identifier (UUID). -// Supported versions are 1, 3, 4 and 5 (as specified in RFC 4122) and -// version 2 (as specified in DCE 1.1). -package uuid - -import ( - "bytes" - "crypto/md5" - "crypto/rand" - "crypto/sha1" - "database/sql/driver" - "encoding/binary" - "encoding/hex" - "fmt" - "hash" - "net" - "os" - "sync" - "time" -) - -// UUID layout variants. -const ( - VariantNCS = iota - VariantRFC4122 - VariantMicrosoft - VariantFuture -) - -// UUID DCE domains. -const ( - DomainPerson = iota - DomainGroup - DomainOrg -) - -// Difference in 100-nanosecond intervals between -// UUID epoch (October 15, 1582) and Unix epoch (January 1, 1970). -const epochStart = 122192928000000000 - -// Used in string method conversion -const dash byte = '-' - -// UUID v1/v2 storage. -var ( - storageMutex sync.Mutex - storageOnce sync.Once - epochFunc = unixTimeFunc - clockSequence uint16 - lastTime uint64 - hardwareAddr [6]byte - posixUID = uint32(os.Getuid()) - posixGID = uint32(os.Getgid()) -) - -// String parse helpers. -var ( - urnPrefix = []byte("urn:uuid:") - byteGroups = []int{8, 4, 4, 4, 12} -) - -func initClockSequence() { - buf := make([]byte, 2) - safeRandom(buf) - clockSequence = binary.BigEndian.Uint16(buf) -} - -func initHardwareAddr() { - interfaces, err := net.Interfaces() - if err == nil { - for _, iface := range interfaces { - if len(iface.HardwareAddr) >= 6 { - copy(hardwareAddr[:], iface.HardwareAddr) - return - } - } - } - - // Initialize hardwareAddr randomly in case - // of real network interfaces absence - safeRandom(hardwareAddr[:]) - - // Set multicast bit as recommended in RFC 4122 - hardwareAddr[0] |= 0x01 -} - -func initStorage() { - initClockSequence() - initHardwareAddr() -} - -func safeRandom(dest []byte) { - if _, err := rand.Read(dest); err != nil { - panic(err) - } -} - -// Returns difference in 100-nanosecond intervals between -// UUID epoch (October 15, 1582) and current time. -// This is default epoch calculation function. -func unixTimeFunc() uint64 { - return epochStart + uint64(time.Now().UnixNano()/100) -} - -// UUID representation compliant with specification -// described in RFC 4122. -type UUID [16]byte - -// NullUUID can be used with the standard sql package to represent a -// UUID value that can be NULL in the database -type NullUUID struct { - UUID UUID - Valid bool -} - -// The nil UUID is special form of UUID that is specified to have all -// 128 bits set to zero. -var Nil = UUID{} - -// Predefined namespace UUIDs. -var ( - NamespaceDNS, _ = FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - NamespaceURL, _ = FromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8") - NamespaceOID, _ = FromString("6ba7b812-9dad-11d1-80b4-00c04fd430c8") - NamespaceX500, _ = FromString("6ba7b814-9dad-11d1-80b4-00c04fd430c8") -) - -// And returns result of binary AND of two UUIDs. -func And(u1 UUID, u2 UUID) UUID { - u := UUID{} - for i := 0; i < 16; i++ { - u[i] = u1[i] & u2[i] - } - return u -} - -// Or returns result of binary OR of two UUIDs. -func Or(u1 UUID, u2 UUID) UUID { - u := UUID{} - for i := 0; i < 16; i++ { - u[i] = u1[i] | u2[i] - } - return u -} - -// Equal returns true if u1 and u2 equals, otherwise returns false. -func Equal(u1 UUID, u2 UUID) bool { - return bytes.Equal(u1[:], u2[:]) -} - -// Version returns algorithm version used to generate UUID. -func (u UUID) Version() uint { - return uint(u[6] >> 4) -} - -// Variant returns UUID layout variant. -func (u UUID) Variant() uint { - switch { - case (u[8] & 0x80) == 0x00: - return VariantNCS - case (u[8]&0xc0)|0x80 == 0x80: - return VariantRFC4122 - case (u[8]&0xe0)|0xc0 == 0xc0: - return VariantMicrosoft - } - return VariantFuture -} - -// Bytes returns bytes slice representation of UUID. -func (u UUID) Bytes() []byte { - return u[:] -} - -// Returns canonical string representation of UUID: -// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. -func (u UUID) String() string { - buf := make([]byte, 36) - - hex.Encode(buf[0:8], u[0:4]) - buf[8] = dash - hex.Encode(buf[9:13], u[4:6]) - buf[13] = dash - hex.Encode(buf[14:18], u[6:8]) - buf[18] = dash - hex.Encode(buf[19:23], u[8:10]) - buf[23] = dash - hex.Encode(buf[24:], u[10:]) - - return string(buf) -} - -// SetVersion sets version bits. -func (u *UUID) SetVersion(v byte) { - u[6] = (u[6] & 0x0f) | (v << 4) -} - -// SetVariant sets variant bits as described in RFC 4122. -func (u *UUID) SetVariant() { - u[8] = (u[8] & 0xbf) | 0x80 -} - -// MarshalText implements the encoding.TextMarshaler interface. -// The encoding is the same as returned by String. -func (u UUID) MarshalText() (text []byte, err error) { - text = []byte(u.String()) - return -} - -// UnmarshalText implements the encoding.TextUnmarshaler interface. -// Following formats are supported: -// "6ba7b810-9dad-11d1-80b4-00c04fd430c8", -// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}", -// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" -func (u *UUID) UnmarshalText(text []byte) (err error) { - if len(text) < 32 { - err = fmt.Errorf("uuid: UUID string too short: %s", text) - return - } - - t := text[:] - braced := false - - if bytes.Equal(t[:9], urnPrefix) { - t = t[9:] - } else if t[0] == '{' { - braced = true - t = t[1:] - } - - b := u[:] - - for i, byteGroup := range byteGroups { - if i > 0 && t[0] == '-' { - t = t[1:] - } else if i > 0 && t[0] != '-' { - err = fmt.Errorf("uuid: invalid string format") - return - } - - if i == 2 { - if !bytes.Contains([]byte("012345"), []byte{t[0]}) { - err = fmt.Errorf("uuid: invalid version number: %s", t[0]) - return - } - } - - if len(t) < byteGroup { - err = fmt.Errorf("uuid: UUID string too short: %s", text) - return - } - - if i == 4 && len(t) > byteGroup && - ((braced && t[byteGroup] != '}') || len(t[byteGroup:]) > 1 || !braced) { - err = fmt.Errorf("uuid: UUID string too long: %s", t) - return - } - - _, err = hex.Decode(b[:byteGroup/2], t[:byteGroup]) - - if err != nil { - return - } - - t = t[byteGroup:] - b = b[byteGroup/2:] - } - - return -} - -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (u UUID) MarshalBinary() (data []byte, err error) { - data = u.Bytes() - return -} - -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. -// It will return error if the slice isn't 16 bytes long. -func (u *UUID) UnmarshalBinary(data []byte) (err error) { - if len(data) != 16 { - err = fmt.Errorf("uuid: UUID must be exactly 16 bytes long, got %d bytes", len(data)) - return - } - copy(u[:], data) - - return -} - -// Value implements the driver.Valuer interface. -func (u UUID) Value() (driver.Value, error) { - return u.String(), nil -} - -// Scan implements the sql.Scanner interface. -// A 16-byte slice is handled by UnmarshalBinary, while -// a longer byte slice or a string is handled by UnmarshalText. -func (u *UUID) Scan(src interface{}) error { - switch src := src.(type) { - case []byte: - if len(src) == 16 { - return u.UnmarshalBinary(src) - } - return u.UnmarshalText(src) - - case string: - return u.UnmarshalText([]byte(src)) - } - - return fmt.Errorf("uuid: cannot convert %T to UUID", src) -} - -// Value implements the driver.Valuer interface. -func (u NullUUID) Value() (driver.Value, error) { - if !u.Valid { - return nil, nil - } - // Delegate to UUID Value function - return u.UUID.Value() -} - -// Scan implements the sql.Scanner interface. -func (u *NullUUID) Scan(src interface{}) error { - if src == nil { - u.UUID, u.Valid = Nil, false - return nil - } - - // Delegate to UUID Scan function - u.Valid = true - return u.UUID.Scan(src) -} - -// FromBytes returns UUID converted from raw byte slice input. -// It will return error if the slice isn't 16 bytes long. -func FromBytes(input []byte) (u UUID, err error) { - err = u.UnmarshalBinary(input) - return -} - -// FromBytesOrNil returns UUID converted from raw byte slice input. -// Same behavior as FromBytes, but returns a Nil UUID on error. -func FromBytesOrNil(input []byte) UUID { - uuid, err := FromBytes(input) - if err != nil { - return Nil - } - return uuid -} - -// FromString returns UUID parsed from string input. -// Input is expected in a form accepted by UnmarshalText. -func FromString(input string) (u UUID, err error) { - err = u.UnmarshalText([]byte(input)) - return -} - -// FromStringOrNil returns UUID parsed from string input. -// Same behavior as FromString, but returns a Nil UUID on error. -func FromStringOrNil(input string) UUID { - uuid, err := FromString(input) - if err != nil { - return Nil - } - return uuid -} - -// Returns UUID v1/v2 storage state. -// Returns epoch timestamp, clock sequence, and hardware address. -func getStorage() (uint64, uint16, []byte) { - storageOnce.Do(initStorage) - - storageMutex.Lock() - defer storageMutex.Unlock() - - timeNow := epochFunc() - // Clock changed backwards since last UUID generation. - // Should increase clock sequence. - if timeNow <= lastTime { - clockSequence++ - } - lastTime = timeNow - - return timeNow, clockSequence, hardwareAddr[:] -} - -// NewV1 returns UUID based on current timestamp and MAC address. -func NewV1() UUID { - u := UUID{} - - timeNow, clockSeq, hardwareAddr := getStorage() - - binary.BigEndian.PutUint32(u[0:], uint32(timeNow)) - binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) - binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) - binary.BigEndian.PutUint16(u[8:], clockSeq) - - copy(u[10:], hardwareAddr) - - u.SetVersion(1) - u.SetVariant() - - return u -} - -// NewV2 returns DCE Security UUID based on POSIX UID/GID. -func NewV2(domain byte) UUID { - u := UUID{} - - timeNow, clockSeq, hardwareAddr := getStorage() - - switch domain { - case DomainPerson: - binary.BigEndian.PutUint32(u[0:], posixUID) - case DomainGroup: - binary.BigEndian.PutUint32(u[0:], posixGID) - } - - binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) - binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) - binary.BigEndian.PutUint16(u[8:], clockSeq) - u[9] = domain - - copy(u[10:], hardwareAddr) - - u.SetVersion(2) - u.SetVariant() - - return u -} - -// NewV3 returns UUID based on MD5 hash of namespace UUID and name. -func NewV3(ns UUID, name string) UUID { - u := newFromHash(md5.New(), ns, name) - u.SetVersion(3) - u.SetVariant() - - return u -} - -// NewV4 returns random generated UUID. -func NewV4() UUID { - u := UUID{} - safeRandom(u[:]) - u.SetVersion(4) - u.SetVariant() - - return u -} - -// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. -func NewV5(ns UUID, name string) UUID { - u := newFromHash(sha1.New(), ns, name) - u.SetVersion(5) - u.SetVariant() - - return u -} - -// Returns UUID based on hashing of namespace UUID and name. -func newFromHash(h hash.Hash, ns UUID, name string) UUID { - u := UUID{} - h.Write(ns[:]) - h.Write([]byte(name)) - copy(u[:], h.Sum(nil)) - - return u -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/LICENSE b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/LICENSE deleted file mode 100644 index 80511a0..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 siddontang - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/hack/hack.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/hack/hack.go deleted file mode 100644 index 74ee83c..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/hack/hack.go +++ /dev/null @@ -1,27 +0,0 @@ -package hack - -import ( - "reflect" - "unsafe" -) - -// no copy to change slice to string -// use your own risk -func String(b []byte) (s string) { - pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) - pstring.Data = pbytes.Data - pstring.Len = pbytes.Len - return -} - -// no copy to change string to slice -// use your own risk -func Slice(s string) (b []byte) { - pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) - pbytes.Data = pstring.Data - pbytes.Len = pstring.Len - pbytes.Cap = pstring.Len - return -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/ioutil.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/ioutil.go deleted file mode 100644 index c99c987..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/ioutil.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2012, Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package ioutil2 - -import ( - "io" - "io/ioutil" - "os" - "path" -) - -// Write file to temp and atomically move when everything else succeeds. -func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { - dir, name := path.Split(filename) - f, err := ioutil.TempFile(dir, name) - if err != nil { - return err - } - n, err := f.Write(data) - f.Close() - if err == nil && n < len(data) { - err = io.ErrShortWrite - } else { - err = os.Chmod(f.Name(), perm) - } - if err != nil { - os.Remove(f.Name()) - return err - } - return os.Rename(f.Name(), filename) -} - -// Check file exists or not -func FileExists(name string) bool { - _, err := os.Stat(name) - return !os.IsNotExist(err) -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/sectionwriter.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/sectionwriter.go deleted file mode 100644 index c02ab0d..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/ioutil2/sectionwriter.go +++ /dev/null @@ -1,69 +0,0 @@ -package ioutil2 - -import ( - "errors" - "io" -) - -var ErrExceedLimit = errors.New("write exceed limit") - -func NewSectionWriter(w io.WriterAt, off int64, n int64) *SectionWriter { - return &SectionWriter{w, off, off, off + n} -} - -type SectionWriter struct { - w io.WriterAt - base int64 - off int64 - limit int64 -} - -func (s *SectionWriter) Write(p []byte) (n int, err error) { - if s.off >= s.limit { - return 0, ErrExceedLimit - } - - if max := s.limit - s.off; int64(len(p)) > max { - return 0, ErrExceedLimit - } - - n, err = s.w.WriteAt(p, s.off) - s.off += int64(n) - return -} - -var errWhence = errors.New("Seek: invalid whence") -var errOffset = errors.New("Seek: invalid offset") - -func (s *SectionWriter) Seek(offset int64, whence int) (int64, error) { - switch whence { - default: - return 0, errWhence - case 0: - offset += s.base - case 1: - offset += s.off - case 2: - offset += s.limit - } - if offset < s.base { - return 0, errOffset - } - s.off = offset - return offset - s.base, nil -} - -func (s *SectionWriter) WriteAt(p []byte, off int64) (n int, err error) { - if off < 0 || off >= s.limit-s.base { - return 0, errOffset - } - off += s.base - if max := s.limit - off; int64(len(p)) > max { - return 0, ErrExceedLimit - } - - return s.w.WriteAt(p, off) -} - -// Size returns the size of the section in bytes. -func (s *SectionWriter) Size() int64 { return s.limit - s.base } diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/atomic.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/atomic.go deleted file mode 100644 index 424a974..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/atomic.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2013, Google Inc. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sync2 - -import ( - "sync" - "sync/atomic" - "time" -) - -type AtomicInt32 int32 - -func (i *AtomicInt32) Add(n int32) int32 { - return atomic.AddInt32((*int32)(i), n) -} - -func (i *AtomicInt32) Set(n int32) { - atomic.StoreInt32((*int32)(i), n) -} - -func (i *AtomicInt32) Get() int32 { - return atomic.LoadInt32((*int32)(i)) -} - -func (i *AtomicInt32) CompareAndSwap(oldval, newval int32) (swapped bool) { - return atomic.CompareAndSwapInt32((*int32)(i), oldval, newval) -} - -type AtomicUint32 uint32 - -func (i *AtomicUint32) Add(n uint32) uint32 { - return atomic.AddUint32((*uint32)(i), n) -} - -func (i *AtomicUint32) Set(n uint32) { - atomic.StoreUint32((*uint32)(i), n) -} - -func (i *AtomicUint32) Get() uint32 { - return atomic.LoadUint32((*uint32)(i)) -} - -func (i *AtomicUint32) CompareAndSwap(oldval, newval uint32) (swapped bool) { - return atomic.CompareAndSwapUint32((*uint32)(i), oldval, newval) -} - -type AtomicInt64 int64 - -func (i *AtomicInt64) Add(n int64) int64 { - return atomic.AddInt64((*int64)(i), n) -} - -func (i *AtomicInt64) Set(n int64) { - atomic.StoreInt64((*int64)(i), n) -} - -func (i *AtomicInt64) Get() int64 { - return atomic.LoadInt64((*int64)(i)) -} - -func (i *AtomicInt64) CompareAndSwap(oldval, newval int64) (swapped bool) { - return atomic.CompareAndSwapInt64((*int64)(i), oldval, newval) -} - -type AtomicUint64 uint64 - -func (i *AtomicUint64) Add(n uint64) uint64 { - return atomic.AddUint64((*uint64)(i), n) -} - -func (i *AtomicUint64) Set(n uint64) { - atomic.StoreUint64((*uint64)(i), n) -} - -func (i *AtomicUint64) Get() uint64 { - return atomic.LoadUint64((*uint64)(i)) -} - -func (i *AtomicUint64) CompareAndSwap(oldval, newval uint64) (swapped bool) { - return atomic.CompareAndSwapUint64((*uint64)(i), oldval, newval) -} - -type AtomicDuration int64 - -func (d *AtomicDuration) Add(duration time.Duration) time.Duration { - return time.Duration(atomic.AddInt64((*int64)(d), int64(duration))) -} - -func (d *AtomicDuration) Set(duration time.Duration) { - atomic.StoreInt64((*int64)(d), int64(duration)) -} - -func (d *AtomicDuration) Get() time.Duration { - return time.Duration(atomic.LoadInt64((*int64)(d))) -} - -func (d *AtomicDuration) CompareAndSwap(oldval, newval time.Duration) (swapped bool) { - return atomic.CompareAndSwapInt64((*int64)(d), int64(oldval), int64(newval)) -} - -// AtomicString gives you atomic-style APIs for string, but -// it's only a convenience wrapper that uses a mutex. So, it's -// not as efficient as the rest of the atomic types. -type AtomicString struct { - mu sync.Mutex - str string -} - -func (s *AtomicString) Set(str string) { - s.mu.Lock() - s.str = str - s.mu.Unlock() -} - -func (s *AtomicString) Get() string { - s.mu.Lock() - str := s.str - s.mu.Unlock() - return str -} - -func (s *AtomicString) CompareAndSwap(oldval, newval string) (swqpped bool) { - s.mu.Lock() - defer s.mu.Unlock() - if s.str == oldval { - s.str = newval - return true - } - return false -} - -type AtomicBool int32 - -func (b *AtomicBool) Set(v bool) { - if v { - atomic.StoreInt32((*int32)(b), 1) - } else { - atomic.StoreInt32((*int32)(b), 0) - } -} - -func (b *AtomicBool) Get() bool { - return atomic.LoadInt32((*int32)(b)) == 1 -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/semaphore.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/semaphore.go deleted file mode 100644 index d310da7..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/github.com/siddontang/go/sync2/semaphore.go +++ /dev/null @@ -1,65 +0,0 @@ -package sync2 - -import ( - "sync" - "sync/atomic" - "time" -) - -func NewSemaphore(initialCount int) *Semaphore { - res := &Semaphore{ - counter: int64(initialCount), - } - res.cond.L = &res.lock - return res -} - -type Semaphore struct { - lock sync.Mutex - cond sync.Cond - counter int64 -} - -func (s *Semaphore) Release() { - s.lock.Lock() - s.counter += 1 - if s.counter >= 0 { - s.cond.Signal() - } - s.lock.Unlock() -} - -func (s *Semaphore) Acquire() { - s.lock.Lock() - for s.counter < 1 { - s.cond.Wait() - } - s.counter -= 1 - s.lock.Unlock() -} - -func (s *Semaphore) AcquireTimeout(timeout time.Duration) bool { - done := make(chan bool, 1) - // Gate used to communicate between the threads and decide what the result - // is. If the main thread decides, we have timed out, otherwise we succeed. - decided := new(int32) - go func() { - s.Acquire() - if atomic.SwapInt32(decided, 1) == 0 { - done <- true - } else { - // If we already decided the result, and this thread did not win - s.Release() - } - }() - select { - case <-done: - return true - case <-time.NewTimer(timeout).C: - if atomic.SwapInt32(decided, 1) == 1 { - // The other thread already decided the result - return true - } - return false - } -} diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/LICENSE b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/LICENSE deleted file mode 100644 index 6a66aea..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/PATENTS b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/PATENTS deleted file mode 100644 index 7330990..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/PATENTS +++ /dev/null @@ -1,22 +0,0 @@ -Additional IP Rights Grant (Patents) - -"This implementation" means the copyrightable works distributed by -Google as part of the Go project. - -Google hereby grants to You a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this section) -patent license to make, have made, use, offer to sell, sell, import, -transfer and otherwise run, modify and propagate the contents of this -implementation of Go, where such license applies only to those patent -claims, both currently owned or controlled by Google and acquired in -the future, licensable by Google that are necessarily infringed by this -implementation of Go. This grant does not include claims that would be -infringed only as a consequence of further modification of this -implementation. If you or your agent or exclusive licensee institute or -order or agree to the institution of patent litigation against any -entity (including a cross-claim or counterclaim in a lawsuit) alleging -that this implementation of Go or any code incorporated within this -implementation of Go constitutes direct or contributory patent -infringement, or inducement of patent infringement, then any patent -rights granted to you under this License for this implementation of Go -shall terminate as of the date such litigation is filed. diff --git a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/context/context.go b/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/context/context.go deleted file mode 100644 index 77b64d0..0000000 --- a/vendor/github.com/siddontang/go-mysql/_vendor/vendor/golang.org/x/net/context/context.go +++ /dev/null @@ -1,447 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package context defines the Context type, which carries deadlines, -// cancelation signals, and other request-scoped values across API boundaries -// and between processes. -// -// Incoming requests to a server should create a Context, and outgoing calls to -// servers should accept a Context. The chain of function calls between must -// propagate the Context, optionally replacing it with a modified copy created -// using WithDeadline, WithTimeout, WithCancel, or WithValue. -// -// Programs that use Contexts should follow these rules to keep interfaces -// consistent across packages and enable static analysis tools to check context -// propagation: -// -// Do not store Contexts inside a struct type; instead, pass a Context -// explicitly to each function that needs it. The Context should be the first -// parameter, typically named ctx: -// -// func DoSomething(ctx context.Context, arg Arg) error { -// // ... use ctx ... -// } -// -// Do not pass a nil Context, even if a function permits it. Pass context.TODO -// if you are unsure about which Context to use. -// -// Use context Values only for request-scoped data that transits processes and -// APIs, not for passing optional parameters to functions. -// -// The same Context may be passed to functions running in different goroutines; -// Contexts are safe for simultaneous use by multiple goroutines. -// -// See http://blog.golang.org/context for example code for a server that uses -// Contexts. -package context // import "golang.org/x/net/context" - -import ( - "errors" - "fmt" - "sync" - "time" -) - -// A Context carries a deadline, a cancelation signal, and other values across -// API boundaries. -// -// Context's methods may be called by multiple goroutines simultaneously. -type Context interface { - // Deadline returns the time when work done on behalf of this context - // should be canceled. Deadline returns ok==false when no deadline is - // set. Successive calls to Deadline return the same results. - Deadline() (deadline time.Time, ok bool) - - // Done returns a channel that's closed when work done on behalf of this - // context should be canceled. Done may return nil if this context can - // never be canceled. Successive calls to Done return the same value. - // - // WithCancel arranges for Done to be closed when cancel is called; - // WithDeadline arranges for Done to be closed when the deadline - // expires; WithTimeout arranges for Done to be closed when the timeout - // elapses. - // - // Done is provided for use in select statements: - // - // // Stream generates values with DoSomething and sends them to out - // // until DoSomething returns an error or ctx.Done is closed. - // func Stream(ctx context.Context, out <-chan Value) error { - // for { - // v, err := DoSomething(ctx) - // if err != nil { - // return err - // } - // select { - // case <-ctx.Done(): - // return ctx.Err() - // case out <- v: - // } - // } - // } - // - // See http://blog.golang.org/pipelines for more examples of how to use - // a Done channel for cancelation. - Done() <-chan struct{} - - // Err returns a non-nil error value after Done is closed. Err returns - // Canceled if the context was canceled or DeadlineExceeded if the - // context's deadline passed. No other values for Err are defined. - // After Done is closed, successive calls to Err return the same value. - Err() error - - // Value returns the value associated with this context for key, or nil - // if no value is associated with key. Successive calls to Value with - // the same key returns the same result. - // - // Use context values only for request-scoped data that transits - // processes and API boundaries, not for passing optional parameters to - // functions. - // - // A key identifies a specific value in a Context. Functions that wish - // to store values in Context typically allocate a key in a global - // variable then use that key as the argument to context.WithValue and - // Context.Value. A key can be any type that supports equality; - // packages should define keys as an unexported type to avoid - // collisions. - // - // Packages that define a Context key should provide type-safe accessors - // for the values stores using that key: - // - // // Package user defines a User type that's stored in Contexts. - // package user - // - // import "golang.org/x/net/context" - // - // // User is the type of value stored in the Contexts. - // type User struct {...} - // - // // key is an unexported type for keys defined in this package. - // // This prevents collisions with keys defined in other packages. - // type key int - // - // // userKey is the key for user.User values in Contexts. It is - // // unexported; clients use user.NewContext and user.FromContext - // // instead of using this key directly. - // var userKey key = 0 - // - // // NewContext returns a new Context that carries value u. - // func NewContext(ctx context.Context, u *User) context.Context { - // return context.WithValue(ctx, userKey, u) - // } - // - // // FromContext returns the User value stored in ctx, if any. - // func FromContext(ctx context.Context) (*User, bool) { - // u, ok := ctx.Value(userKey).(*User) - // return u, ok - // } - Value(key interface{}) interface{} -} - -// Canceled is the error returned by Context.Err when the context is canceled. -var Canceled = errors.New("context canceled") - -// DeadlineExceeded is the error returned by Context.Err when the context's -// deadline passes. -var DeadlineExceeded = errors.New("context deadline exceeded") - -// An emptyCtx is never canceled, has no values, and has no deadline. It is not -// struct{}, since vars of this type must have distinct addresses. -type emptyCtx int - -func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { - return -} - -func (*emptyCtx) Done() <-chan struct{} { - return nil -} - -func (*emptyCtx) Err() error { - return nil -} - -func (*emptyCtx) Value(key interface{}) interface{} { - return nil -} - -func (e *emptyCtx) String() string { - switch e { - case background: - return "context.Background" - case todo: - return "context.TODO" - } - return "unknown empty Context" -} - -var ( - background = new(emptyCtx) - todo = new(emptyCtx) -) - -// Background returns a non-nil, empty Context. It is never canceled, has no -// values, and has no deadline. It is typically used by the main function, -// initialization, and tests, and as the top-level Context for incoming -// requests. -func Background() Context { - return background -} - -// TODO returns a non-nil, empty Context. Code should use context.TODO when -// it's unclear which Context to use or it is not yet available (because the -// surrounding function has not yet been extended to accept a Context -// parameter). TODO is recognized by static analysis tools that determine -// whether Contexts are propagated correctly in a program. -func TODO() Context { - return todo -} - -// A CancelFunc tells an operation to abandon its work. -// A CancelFunc does not wait for the work to stop. -// After the first call, subsequent calls to a CancelFunc do nothing. -type CancelFunc func() - -// WithCancel returns a copy of parent with a new Done channel. The returned -// context's Done channel is closed when the returned cancel function is called -// or when the parent context's Done channel is closed, whichever happens first. -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { - c := newCancelCtx(parent) - propagateCancel(parent, &c) - return &c, func() { c.cancel(true, Canceled) } -} - -// newCancelCtx returns an initialized cancelCtx. -func newCancelCtx(parent Context) cancelCtx { - return cancelCtx{ - Context: parent, - done: make(chan struct{}), - } -} - -// propagateCancel arranges for child to be canceled when parent is. -func propagateCancel(parent Context, child canceler) { - if parent.Done() == nil { - return // parent is never canceled - } - if p, ok := parentCancelCtx(parent); ok { - p.mu.Lock() - if p.err != nil { - // parent has already been canceled - child.cancel(false, p.err) - } else { - if p.children == nil { - p.children = make(map[canceler]bool) - } - p.children[child] = true - } - p.mu.Unlock() - } else { - go func() { - select { - case <-parent.Done(): - child.cancel(false, parent.Err()) - case <-child.Done(): - } - }() - } -} - -// parentCancelCtx follows a chain of parent references until it finds a -// *cancelCtx. This function understands how each of the concrete types in this -// package represents its parent. -func parentCancelCtx(parent Context) (*cancelCtx, bool) { - for { - switch c := parent.(type) { - case *cancelCtx: - return c, true - case *timerCtx: - return &c.cancelCtx, true - case *valueCtx: - parent = c.Context - default: - return nil, false - } - } -} - -// removeChild removes a context from its parent. -func removeChild(parent Context, child canceler) { - p, ok := parentCancelCtx(parent) - if !ok { - return - } - p.mu.Lock() - if p.children != nil { - delete(p.children, child) - } - p.mu.Unlock() -} - -// A canceler is a context type that can be canceled directly. The -// implementations are *cancelCtx and *timerCtx. -type canceler interface { - cancel(removeFromParent bool, err error) - Done() <-chan struct{} -} - -// A cancelCtx can be canceled. When canceled, it also cancels any children -// that implement canceler. -type cancelCtx struct { - Context - - done chan struct{} // closed by the first cancel call. - - mu sync.Mutex - children map[canceler]bool // set to nil by the first cancel call - err error // set to non-nil by the first cancel call -} - -func (c *cancelCtx) Done() <-chan struct{} { - return c.done -} - -func (c *cancelCtx) Err() error { - c.mu.Lock() - defer c.mu.Unlock() - return c.err -} - -func (c *cancelCtx) String() string { - return fmt.Sprintf("%v.WithCancel", c.Context) -} - -// cancel closes c.done, cancels each of c's children, and, if -// removeFromParent is true, removes c from its parent's children. -func (c *cancelCtx) cancel(removeFromParent bool, err error) { - if err == nil { - panic("context: internal error: missing cancel error") - } - c.mu.Lock() - if c.err != nil { - c.mu.Unlock() - return // already canceled - } - c.err = err - close(c.done) - for child := range c.children { - // NOTE: acquiring the child's lock while holding parent's lock. - child.cancel(false, err) - } - c.children = nil - c.mu.Unlock() - - if removeFromParent { - removeChild(c.Context, c) - } -} - -// WithDeadline returns a copy of the parent context with the deadline adjusted -// to be no later than d. If the parent's deadline is already earlier than d, -// WithDeadline(parent, d) is semantically equivalent to parent. The returned -// context's Done channel is closed when the deadline expires, when the returned -// cancel function is called, or when the parent context's Done channel is -// closed, whichever happens first. -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { - if cur, ok := parent.Deadline(); ok && cur.Before(deadline) { - // The current deadline is already sooner than the new one. - return WithCancel(parent) - } - c := &timerCtx{ - cancelCtx: newCancelCtx(parent), - deadline: deadline, - } - propagateCancel(parent, c) - d := deadline.Sub(time.Now()) - if d <= 0 { - c.cancel(true, DeadlineExceeded) // deadline has already passed - return c, func() { c.cancel(true, Canceled) } - } - c.mu.Lock() - defer c.mu.Unlock() - if c.err == nil { - c.timer = time.AfterFunc(d, func() { - c.cancel(true, DeadlineExceeded) - }) - } - return c, func() { c.cancel(true, Canceled) } -} - -// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to -// implement Done and Err. It implements cancel by stopping its timer then -// delegating to cancelCtx.cancel. -type timerCtx struct { - cancelCtx - timer *time.Timer // Under cancelCtx.mu. - - deadline time.Time -} - -func (c *timerCtx) Deadline() (deadline time.Time, ok bool) { - return c.deadline, true -} - -func (c *timerCtx) String() string { - return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now())) -} - -func (c *timerCtx) cancel(removeFromParent bool, err error) { - c.cancelCtx.cancel(false, err) - if removeFromParent { - // Remove this timerCtx from its parent cancelCtx's children. - removeChild(c.cancelCtx.Context, c) - } - c.mu.Lock() - if c.timer != nil { - c.timer.Stop() - c.timer = nil - } - c.mu.Unlock() -} - -// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete: -// -// func slowOperationWithTimeout(ctx context.Context) (Result, error) { -// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) -// defer cancel() // releases resources if slowOperation completes before timeout elapses -// return slowOperation(ctx) -// } -func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { - return WithDeadline(parent, time.Now().Add(timeout)) -} - -// WithValue returns a copy of parent in which the value associated with key is -// val. -// -// Use context Values only for request-scoped data that transits processes and -// APIs, not for passing optional parameters to functions. -func WithValue(parent Context, key interface{}, val interface{}) Context { - return &valueCtx{parent, key, val} -} - -// A valueCtx carries a key-value pair. It implements Value for that key and -// delegates all other calls to the embedded Context. -type valueCtx struct { - Context - key, val interface{} -} - -func (c *valueCtx) String() string { - return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val) -} - -func (c *valueCtx) Value(key interface{}) interface{} { - if c.key == key { - return c.val - } - return c.Context.Value(key) -} diff --git a/vendor/github.com/siddontang/go-mysql/canal/canal.go b/vendor/github.com/siddontang/go-mysql/canal/canal.go index 0b225ff..64d9aec 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/canal.go +++ b/vendor/github.com/siddontang/go-mysql/canal/canal.go @@ -1,26 +1,25 @@ package canal import ( + "context" "fmt" "io/ioutil" "os" - "path" + "regexp" "strconv" "strings" "sync" + "time" "github.com/juju/errors" - "github.com/ngaut/log" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/client" "github.com/siddontang/go-mysql/dump" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "github.com/siddontang/go-mysql/schema" - "github.com/siddontang/go/sync2" ) -var errCanalClosed = errors.New("canal was closed") - // Canal can sync your MySQL data into everywhere, like Elasticsearch, Redis, etc... // MySQL must open row format for binlog type Canal struct { @@ -30,48 +29,49 @@ type Canal struct { master *masterInfo dumper *dump.Dumper + dumped bool dumpDoneCh chan struct{} syncer *replication.BinlogSyncer - rsLock sync.Mutex - rsHandlers []RowsEventHandler + eventHandler EventHandler connLock sync.Mutex conn *client.Conn - wg sync.WaitGroup + tableLock sync.RWMutex + tables map[string]*schema.Table + errorTablesGetTime map[string]time.Time - tableLock sync.Mutex - tables map[string]*schema.Table + tableMatchCache map[string]bool + includeTableRegex []*regexp.Regexp + excludeTableRegex []*regexp.Regexp - quit chan struct{} - closed sync2.AtomicBool + ctx context.Context + cancel context.CancelFunc } +// canal will retry fetching unknown table's meta after UnknownTableRetryPeriod +var UnknownTableRetryPeriod = time.Second * time.Duration(10) +var ErrExcludedTable = errors.New("excluded table meta") + func NewCanal(cfg *Config) (*Canal, error) { c := new(Canal) c.cfg = cfg - c.closed.Set(false) - c.quit = make(chan struct{}) - os.MkdirAll(cfg.DataDir, 0755) + c.ctx, c.cancel = context.WithCancel(context.Background()) c.dumpDoneCh = make(chan struct{}) - c.rsHandlers = make([]RowsEventHandler, 0, 4) + c.eventHandler = &DummyEventHandler{} + c.tables = make(map[string]*schema.Table) + if c.cfg.DiscardNoMetaRowEvent { + c.errorTablesGetTime = make(map[string]time.Time) + } + c.master = &masterInfo{} var err error - if c.master, err = loadMasterInfo(c.masterInfoPath()); err != nil { - return nil, errors.Trace(err) - } else if len(c.master.Addr) != 0 && c.master.Addr != c.cfg.Addr { - log.Infof("MySQL addr %s in old master.info, but new %s, reset", c.master.Addr, c.cfg.Addr) - // may use another MySQL, reset - c.master = &masterInfo{} - } - c.master.Addr = c.cfg.Addr - - if err := c.prepareDumper(); err != nil { + if err = c.prepareDumper(); err != nil { return nil, errors.Trace(err) } @@ -83,6 +83,33 @@ func NewCanal(cfg *Config) (*Canal, error) { return nil, errors.Trace(err) } + // init table filter + if n := len(c.cfg.IncludeTableRegex); n > 0 { + c.includeTableRegex = make([]*regexp.Regexp, n) + for i, val := range c.cfg.IncludeTableRegex { + reg, err := regexp.Compile(val) + if err != nil { + return nil, errors.Trace(err) + } + c.includeTableRegex[i] = reg + } + } + + if n := len(c.cfg.ExcludeTableRegex); n > 0 { + c.excludeTableRegex = make([]*regexp.Regexp, n) + for i, val := range c.cfg.ExcludeTableRegex { + reg, err := regexp.Compile(val) + if err != nil { + return nil, errors.Trace(err) + } + c.excludeTableRegex[i] = reg + } + } + + if c.includeTableRegex != nil || c.excludeTableRegex != nil { + c.tableMatchCache = make(map[string]bool) + } + return c, nil } @@ -114,6 +141,15 @@ func (c *Canal) prepareDumper() error { c.dumper.AddTables(tableDB, tables...) } + charset := c.cfg.Charset + c.dumper.SetCharset(charset) + + c.dumper.SetWhere(c.cfg.Dump.Where) + c.dumper.SkipMasterData(c.cfg.Dump.SkipMasterData) + c.dumper.SetMaxAllowedPacket(c.cfg.Dump.MaxAllowedPacketMB) + // Use hex blob for mysqldump + c.dumper.SetHexBlob(true) + for _, ignoreTable := range c.cfg.Dump.IgnoreTables { if seps := strings.Split(ignoreTable, ","); len(seps) == 2 { c.dumper.AddIgnoreTables(seps[0], seps[1]) @@ -129,92 +165,208 @@ func (c *Canal) prepareDumper() error { return nil } -func (c *Canal) Start() error { - c.wg.Add(1) - go c.run() +// Run will first try to dump all data from MySQL master `mysqldump`, +// then sync from the binlog position in the dump data. +// It will run forever until meeting an error or Canal closed. +func (c *Canal) Run() error { + return c.run() +} - return nil +// RunFrom will sync from the binlog position directly, ignore mysqldump. +func (c *Canal) RunFrom(pos mysql.Position) error { + c.master.Update(pos) + + return c.Run() +} + +func (c *Canal) StartFromGTID(set mysql.GTIDSet) error { + c.master.UpdateGTIDSet(set) + + return c.Run() +} + +// Dump all data from MySQL master `mysqldump`, ignore sync binlog. +func (c *Canal) Dump() error { + if c.dumped { + return errors.New("the method Dump can't be called twice") + } + c.dumped = true + defer close(c.dumpDoneCh) + return c.dump() } func (c *Canal) run() error { - defer c.wg.Done() + defer func() { + c.cancel() + }() - if err := c.tryDump(); err != nil { - log.Errorf("canal dump mysql err: %v", err) - return errors.Trace(err) + c.master.UpdateTimestamp(uint32(time.Now().Unix())) + + if !c.dumped { + c.dumped = true + + err := c.tryDump() + close(c.dumpDoneCh) + + if err != nil { + log.Errorf("canal dump mysql err: %v", err) + return errors.Trace(err) + } } - close(c.dumpDoneCh) - - if err := c.startSyncBinlog(); err != nil { - if !c.isClosed() { - log.Errorf("canal start sync binlog err: %v", err) - } + if err := c.runSyncBinlog(); err != nil { + log.Errorf("canal start sync binlog err: %v", err) return errors.Trace(err) } return nil } -func (c *Canal) isClosed() bool { - return c.closed.Get() -} - func (c *Canal) Close() { - log.Infof("close canal") + log.Infof("closing canal") c.m.Lock() defer c.m.Unlock() - if c.isClosed() { - return - } - - c.closed.Set(true) - - close(c.quit) - + c.cancel() c.connLock.Lock() c.conn.Close() c.conn = nil c.connLock.Unlock() + c.syncer.Close() - if c.syncer != nil { - c.syncer.Close() - c.syncer = nil - } - - c.master.Close() - - c.wg.Wait() + c.eventHandler.OnPosSynced(c.master.Position(), true) } func (c *Canal) WaitDumpDone() <-chan struct{} { return c.dumpDoneCh } +func (c *Canal) Ctx() context.Context { + return c.ctx +} + +func (c *Canal) checkTableMatch(key string) bool { + // no filter, return true + if c.tableMatchCache == nil { + return true + } + + c.tableLock.RLock() + rst, ok := c.tableMatchCache[key] + c.tableLock.RUnlock() + if ok { + // cache hit + return rst + } + matchFlag := false + // check include + if c.includeTableRegex != nil { + for _, reg := range c.includeTableRegex { + if reg.MatchString(key) { + matchFlag = true + break + } + } + } + // check exclude + if matchFlag && c.excludeTableRegex != nil { + for _, reg := range c.excludeTableRegex { + if reg.MatchString(key) { + matchFlag = false + break + } + } + } + c.tableLock.Lock() + c.tableMatchCache[key] = matchFlag + c.tableLock.Unlock() + return matchFlag +} + func (c *Canal) GetTable(db string, table string) (*schema.Table, error) { key := fmt.Sprintf("%s.%s", db, table) - c.tableLock.Lock() + // if table is excluded, return error and skip parsing event or dump + if !c.checkTableMatch(key) { + return nil, ErrExcludedTable + } + c.tableLock.RLock() t, ok := c.tables[key] - c.tableLock.Unlock() + c.tableLock.RUnlock() if ok { return t, nil } + if c.cfg.DiscardNoMetaRowEvent { + c.tableLock.RLock() + lastTime, ok := c.errorTablesGetTime[key] + c.tableLock.RUnlock() + if ok && time.Now().Sub(lastTime) < UnknownTableRetryPeriod { + return nil, schema.ErrMissingTableMeta + } + } + t, err := schema.NewTable(c, db, table) if err != nil { - return nil, errors.Trace(err) + // check table not exists + if ok, err1 := schema.IsTableExist(c, db, table); err1 == nil && !ok { + return nil, schema.ErrTableNotExist + } + // work around : RDS HAHeartBeat + // ref : https://github.com/alibaba/canal/blob/master/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java#L385 + // issue : https://github.com/alibaba/canal/issues/222 + // This is a common error in RDS that canal can't get HAHealthCheckSchema's meta, so we mock a table meta. + // If canal just skip and log error, as RDS HA heartbeat interval is very short, so too many HAHeartBeat errors will be logged. + if key == schema.HAHealthCheckSchema { + // mock ha_health_check meta + ta := &schema.Table{ + Schema: db, + Name: table, + Columns: make([]schema.TableColumn, 0, 2), + Indexes: make([]*schema.Index, 0), + } + ta.AddColumn("id", "bigint(20)", "", "") + ta.AddColumn("type", "char(1)", "", "") + c.tableLock.Lock() + c.tables[key] = ta + c.tableLock.Unlock() + return ta, nil + } + // if DiscardNoMetaRowEvent is true, we just log this error + if c.cfg.DiscardNoMetaRowEvent { + c.tableLock.Lock() + c.errorTablesGetTime[key] = time.Now() + c.tableLock.Unlock() + // log error and return ErrMissingTableMeta + log.Errorf("canal get table meta err: %v", errors.Trace(err)) + return nil, schema.ErrMissingTableMeta + } + return nil, err } c.tableLock.Lock() c.tables[key] = t + if c.cfg.DiscardNoMetaRowEvent { + // if get table info success, delete this key from errorTablesGetTime + delete(c.errorTablesGetTime, key) + } c.tableLock.Unlock() return t, nil } +// ClearTableCache clear table cache +func (c *Canal) ClearTableCache(db []byte, table []byte) { + key := fmt.Sprintf("%s.%s", db, table) + c.tableLock.Lock() + delete(c.tables, key) + if c.cfg.DiscardNoMetaRowEvent { + delete(c.errorTablesGetTime, key) + } + c.tableLock.Unlock() +} + // Check MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB func (c *Canal) CheckBinlogRowImage(image string) error { // need to check MySQL binlog row image? full, minimal or noblob? @@ -246,23 +398,34 @@ func (c *Canal) checkBinlogRowFormat() error { } func (c *Canal) prepareSyncer() error { - seps := strings.Split(c.cfg.Addr, ":") - if len(seps) != 2 { - return errors.Errorf("invalid mysql addr format %s, must host:port", c.cfg.Addr) - } - - port, err := strconv.ParseUint(seps[1], 10, 16) - if err != nil { - return errors.Trace(err) - } - cfg := replication.BinlogSyncerConfig{ - ServerID: c.cfg.ServerID, - Flavor: c.cfg.Flavor, - Host: seps[0], - Port: uint16(port), - User: c.cfg.User, - Password: c.cfg.Password, + ServerID: c.cfg.ServerID, + Flavor: c.cfg.Flavor, + User: c.cfg.User, + Password: c.cfg.Password, + Charset: c.cfg.Charset, + HeartbeatPeriod: c.cfg.HeartbeatPeriod, + ReadTimeout: c.cfg.ReadTimeout, + UseDecimal: c.cfg.UseDecimal, + ParseTime: c.cfg.ParseTime, + SemiSyncEnabled: c.cfg.SemiSyncEnabled, + } + + if strings.Contains(c.cfg.Addr, "/") { + cfg.Host = c.cfg.Addr + } else { + seps := strings.Split(c.cfg.Addr, ":") + if len(seps) != 2 { + return errors.Errorf("invalid mysql addr format %s, must host:port", c.cfg.Addr) + } + + port, err := strconv.ParseUint(seps[1], 10, 16) + if err != nil { + return errors.Trace(err) + } + + cfg.Host = seps[0] + cfg.Port = uint16(port) } c.syncer = replication.NewBinlogSyncer(cfg) @@ -270,10 +433,6 @@ func (c *Canal) prepareSyncer() error { return nil } -func (c *Canal) masterInfoPath() string { - return path.Join(c.cfg.DataDir, "master.info") -} - // Execute a SQL func (c *Canal) Execute(cmd string, args ...interface{}) (rr *mysql.Result, err error) { c.connLock.Lock() @@ -303,5 +462,13 @@ func (c *Canal) Execute(cmd string, args ...interface{}) (rr *mysql.Result, err } func (c *Canal) SyncedPosition() mysql.Position { - return c.master.Pos() + return c.master.Position() +} + +func (c *Canal) SyncedTimestamp() uint32 { + return c.master.timestamp +} + +func (c *Canal) SyncedGTIDSet() mysql.GTIDSet { + return c.master.GTIDSet() } diff --git a/vendor/github.com/siddontang/go-mysql/canal/canal_test.go b/vendor/github.com/siddontang/go-mysql/canal/canal_test.go old mode 100644 new mode 100755 index b83c79e..bd16bd2 --- a/vendor/github.com/siddontang/go-mysql/canal/canal_test.go +++ b/vendor/github.com/siddontang/go-mysql/canal/canal_test.go @@ -1,13 +1,15 @@ package canal import ( + "bytes" "flag" "fmt" - "os" "testing" + "time" - "github.com/ngaut/log" + "github.com/juju/errors" . "github.com/pingcap/check" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" ) @@ -27,19 +29,28 @@ func (s *canalTestSuite) SetUpSuite(c *C) { cfg := NewDefaultConfig() cfg.Addr = fmt.Sprintf("%s:3306", *testHost) cfg.User = "root" + cfg.HeartbeatPeriod = 200 * time.Millisecond + cfg.ReadTimeout = 300 * time.Millisecond cfg.Dump.ExecutionPath = "mysqldump" cfg.Dump.TableDB = "test" cfg.Dump.Tables = []string{"canal_test"} + cfg.Dump.Where = "id>0" - os.RemoveAll(cfg.DataDir) + // include & exclude config + cfg.IncludeTableRegex = make([]string, 1) + cfg.IncludeTableRegex[0] = ".*\\.canal_test" + cfg.ExcludeTableRegex = make([]string, 2) + cfg.ExcludeTableRegex[0] = "mysql\\..*" + cfg.ExcludeTableRegex[1] = ".*\\..*_inner" var err error s.c, err = NewCanal(cfg) c.Assert(err, IsNil) - + s.execute(c, "DROP TABLE IF EXISTS test.canal_test") sql := ` CREATE TABLE IF NOT EXISTS test.canal_test ( - id int AUTO_INCREMENT, + id int AUTO_INCREMENT, + content blob DEFAULT NULL, name varchar(100), PRIMARY KEY(id) )ENGINE=innodb; @@ -48,16 +59,22 @@ func (s *canalTestSuite) SetUpSuite(c *C) { s.execute(c, sql) s.execute(c, "DELETE FROM test.canal_test") - s.execute(c, "INSERT INTO test.canal_test (name) VALUES (?), (?), (?)", "a", "b", "c") + s.execute(c, "INSERT INTO test.canal_test (content, name) VALUES (?, ?), (?, ?), (?, ?)", "1", "a", `\0\ndsfasdf`, "b", "", "c") s.execute(c, "SET GLOBAL binlog_format = 'ROW'") - s.c.RegRowsEventHandler(&testRowsEventHandler{}) - err = s.c.Start() - c.Assert(err, IsNil) + s.c.SetEventHandler(&testEventHandler{c: c}) + go func() { + err = s.c.Run() + c.Assert(err, IsNil) + }() } func (s *canalTestSuite) TearDownSuite(c *C) { + // To test the heartbeat and read timeout,so need to sleep 1 seconds without data transmission + c.Logf("Start testing the heartbeat and read timeout") + time.Sleep(time.Second) + if s.c != nil { s.c.Close() s.c = nil @@ -70,16 +87,19 @@ func (s *canalTestSuite) execute(c *C, query string, args ...interface{}) *mysql return r } -type testRowsEventHandler struct { +type testEventHandler struct { + DummyEventHandler + + c *C } -func (h *testRowsEventHandler) Do(e *RowsEvent) error { - log.Infof("%s %v\n", e.Action, e.Rows) +func (h *testEventHandler) OnRow(e *RowsEvent) error { + log.Infof("OnRow %s %v\n", e.Action, e.Rows) return nil } -func (h *testRowsEventHandler) String() string { - return "testRowsEventHandler" +func (h *testEventHandler) String() string { + return "testEventHandler" } func (s *canalTestSuite) TestCanal(c *C) { @@ -88,7 +108,126 @@ func (s *canalTestSuite) TestCanal(c *C) { for i := 1; i < 10; i++ { s.execute(c, "INSERT INTO test.canal_test (name) VALUES (?)", fmt.Sprintf("%d", i)) } + s.execute(c, "ALTER TABLE test.canal_test ADD `age` INT(5) NOT NULL AFTER `name`") + s.execute(c, "INSERT INTO test.canal_test (name,age) VALUES (?,?)", "d", "18") - err := s.c.CatchMasterPos(100) + err := s.c.CatchMasterPos(10 * time.Second) c.Assert(err, IsNil) } + +func (s *canalTestSuite) TestCanalFilter(c *C) { + // included + sch, err := s.c.GetTable("test", "canal_test") + c.Assert(err, IsNil) + c.Assert(sch, NotNil) + _, err = s.c.GetTable("not_exist_db", "canal_test") + c.Assert(errors.Trace(err), Not(Equals), ErrExcludedTable) + // excluded + sch, err = s.c.GetTable("test", "canal_test_inner") + c.Assert(errors.Cause(err), Equals, ErrExcludedTable) + c.Assert(sch, IsNil) + sch, err = s.c.GetTable("mysql", "canal_test") + c.Assert(errors.Cause(err), Equals, ErrExcludedTable) + c.Assert(sch, IsNil) + sch, err = s.c.GetTable("not_exist_db", "not_canal_test") + c.Assert(errors.Cause(err), Equals, ErrExcludedTable) + c.Assert(sch, IsNil) +} + +func TestCreateTableExp(t *testing.T) { + cases := []string{ + "CREATE TABLE `mydb.mytable` (`id` int(10)) ENGINE=InnoDB", + "CREATE TABLE `mytable` (`id` int(10)) ENGINE=InnoDB", + "CREATE TABLE IF NOT EXISTS `mytable` (`id` int(10)) ENGINE=InnoDB", + "CREATE TABLE IF NOT EXISTS mytable (`id` int(10)) ENGINE=InnoDB", + } + table := []byte("mytable") + db := []byte("mydb") + for _, s := range cases { + m := expCreateTable.FindSubmatch([]byte(s)) + mLen := len(m) + if m == nil || !bytes.Equal(m[mLen-1], table) || (len(m[mLen-2]) > 0 && !bytes.Equal(m[mLen-2], db)) { + t.Fatalf("TestCreateTableExp: case %s failed\n", s) + } + } +} + +func TestAlterTableExp(t *testing.T) { + cases := []string{ + "ALTER TABLE `mydb`.`mytable` ADD `field2` DATE NULL AFTER `field1`;", + "ALTER TABLE `mytable` ADD `field2` DATE NULL AFTER `field1`;", + "ALTER TABLE mydb.mytable ADD `field2` DATE NULL AFTER `field1`;", + "ALTER TABLE mytable ADD `field2` DATE NULL AFTER `field1`;", + "ALTER TABLE mydb.mytable ADD field2 DATE NULL AFTER `field1`;", + } + + table := []byte("mytable") + db := []byte("mydb") + for _, s := range cases { + m := expAlterTable.FindSubmatch([]byte(s)) + mLen := len(m) + if m == nil || !bytes.Equal(m[mLen-1], table) || (len(m[mLen-2]) > 0 && !bytes.Equal(m[mLen-2], db)) { + t.Fatalf("TestAlterTableExp: case %s failed\n", s) + } + } +} + +func TestRenameTableExp(t *testing.T) { + cases := []string{ + "rename table `mydb`.`mytable` to `mydb`.`mytable1`", + "rename table `mytable` to `mytable1`", + "rename table mydb.mytable to mydb.mytable1", + "rename table mytable to mytable1", + + "rename table `mydb`.`mytable` to `mydb`.`mytable2`, `mydb`.`mytable3` to `mydb`.`mytable1`", + "rename table `mytable` to `mytable2`, `mytable3` to `mytable1`", + "rename table mydb.mytable to mydb.mytable2, mydb.mytable3 to mydb.mytable1", + "rename table mytable to mytable2, mytable3 to mytable1", + } + table := []byte("mytable") + db := []byte("mydb") + for _, s := range cases { + m := expRenameTable.FindSubmatch([]byte(s)) + mLen := len(m) + if m == nil || !bytes.Equal(m[mLen-1], table) || (len(m[mLen-2]) > 0 && !bytes.Equal(m[mLen-2], db)) { + t.Fatalf("TestRenameTableExp: case %s failed\n", s) + } + } +} + +func TestDropTableExp(t *testing.T) { + cases := []string{ + "drop table test1", + "DROP TABLE test1", + "DROP TABLE test1", + "DROP table IF EXISTS test.test1", + "drop table `test1`", + "DROP TABLE `test1`", + "DROP table IF EXISTS `test`.`test1`", + "DROP TABLE `test1` /* generated by server */", + "DROP table if exists test1", + "DROP table if exists `test1`", + "DROP table if exists test.test1", + "DROP table if exists `test`.test1", + "DROP table if exists `test`.`test1`", + "DROP table if exists test.`test1`", + "DROP table if exists test.`test1`", + } + + table := []byte("test1") + for _, s := range cases { + m := expDropTable.FindSubmatch([]byte(s)) + mLen := len(m) + if m == nil { + t.Fatalf("TestDropTableExp: case %s failed\n", s) + return + } + if mLen < 4 { + t.Fatalf("TestDropTableExp: case %s failed\n", s) + return + } + if !bytes.Equal(m[mLen-1], table) { + t.Fatalf("TestDropTableExp: case %s failed\n", s) + } + } +} diff --git a/vendor/github.com/siddontang/go-mysql/canal/config.go b/vendor/github.com/siddontang/go-mysql/canal/config.go index f9bb262..d10513c 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/config.go +++ b/vendor/github.com/siddontang/go-mysql/canal/config.go @@ -7,6 +7,7 @@ import ( "github.com/BurntSushi/toml" "github.com/juju/errors" + "github.com/siddontang/go-mysql/mysql" ) type DumpConfig struct { @@ -23,8 +24,18 @@ type DumpConfig struct { // Ignore table format is db.table IgnoreTables []string `toml:"ignore_tables"` + // Dump only selected records. Quotes are mandatory + Where string `toml:"where"` + // If true, discard error msg, else, output to stderr DiscardErr bool `toml:"discard_err"` + + // Set true to skip --master-data if we have no privilege to do + // 'FLUSH TABLES WITH READ LOCK' + SkipMasterData bool `toml:"skip_master_data"` + + // Set to change the default max_allowed_packet size + MaxAllowedPacketMB int `toml:"max_allowed_packet_mb"` } type Config struct { @@ -32,11 +43,30 @@ type Config struct { User string `toml:"user"` Password string `toml:"password"` - ServerID uint32 `toml:"server_id"` - Flavor string `toml:"flavor"` - DataDir string `toml:"data_dir"` + Charset string `toml:"charset"` + ServerID uint32 `toml:"server_id"` + Flavor string `toml:"flavor"` + HeartbeatPeriod time.Duration `toml:"heartbeat_period"` + ReadTimeout time.Duration `toml:"read_timeout"` + + // IncludeTableRegex or ExcludeTableRegex should contain database name + // Only a table which matches IncludeTableRegex and dismatches ExcludeTableRegex will be processed + // eg, IncludeTableRegex : [".*\\.canal"], ExcludeTableRegex : ["mysql\\..*"] + // this will include all database's 'canal' table, except database 'mysql' + // Default IncludeTableRegex and ExcludeTableRegex are empty, this will include all tables + IncludeTableRegex []string `toml:"include_table_regex"` + ExcludeTableRegex []string `toml:"exclude_table_regex"` + + // discard row event without table meta + DiscardNoMetaRowEvent bool `toml:"discard_no_meta_row_event"` Dump DumpConfig `toml:"dump"` + + UseDecimal bool `toml:"use_decimal"` + ParseTime bool `toml:"parse_time"` + + // SemiSyncEnabled enables semi-sync or not. + SemiSyncEnabled bool `toml:"semi_sync_enabled"` } func NewConfigWithFile(name string) (*Config, error) { @@ -66,14 +96,14 @@ func NewDefaultConfig() *Config { c.User = "root" c.Password = "" - rand.Seed(time.Now().Unix()) - c.ServerID = uint32(rand.Intn(1000)) + 1001 + c.Charset = mysql.DEFAULT_CHARSET + c.ServerID = uint32(rand.New(rand.NewSource(time.Now().Unix())).Intn(1000)) + 1001 c.Flavor = "mysql" - c.DataDir = "./var" c.Dump.ExecutionPath = "mysqldump" c.Dump.DiscardErr = true + c.Dump.SkipMasterData = false return c } diff --git a/vendor/github.com/siddontang/go-mysql/canal/dump.go b/vendor/github.com/siddontang/go-mysql/canal/dump.go index 4adec2a..8dcac2b 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/dump.go +++ b/vendor/github.com/siddontang/go-mysql/canal/dump.go @@ -1,12 +1,16 @@ package canal import ( + "encoding/hex" + "fmt" "strconv" + "strings" "time" "github.com/juju/errors" - "github.com/ngaut/log" - "github.com/siddontang/go-mysql/dump" + "github.com/shopspring/decimal" + "github.com/siddontang/go-log/log" + "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/schema" ) @@ -14,6 +18,7 @@ type dumpParseHandler struct { c *Canal name string pos uint64 + gset mysql.GTIDSet } func (h *dumpParseHandler) BinLog(name string, pos uint64) error { @@ -23,12 +28,18 @@ func (h *dumpParseHandler) BinLog(name string, pos uint64) error { } func (h *dumpParseHandler) Data(db string, table string, values []string) error { - if h.c.isClosed() { - return errCanalClosed + if err := h.c.ctx.Err(); err != nil { + return err } tableInfo, err := h.c.GetTable(db, table) if err != nil { + e := errors.Cause(err) + if e == ErrExcludedTable || + e == schema.ErrTableNotExist || + e == schema.ErrMissingTableMeta { + return nil + } log.Errorf("get %s.%s information err: %v", db, table, err) return errors.Trace(err) } @@ -38,32 +49,51 @@ func (h *dumpParseHandler) Data(db string, table string, values []string) error for i, v := range values { if v == "NULL" { vs[i] = nil + } else if v == "_binary ''" { + vs[i] = []byte{} } else if v[0] != '\'' { if tableInfo.Columns[i].Type == schema.TYPE_NUMBER { n, err := strconv.ParseInt(v, 10, 64) if err != nil { - log.Errorf("parse row %v at %d error %v, skip", values, i, err) - return dump.ErrSkip + return fmt.Errorf("parse row %v at %d error %v, int expected", values, i, err) } vs[i] = n } else if tableInfo.Columns[i].Type == schema.TYPE_FLOAT { f, err := strconv.ParseFloat(v, 64) if err != nil { - log.Errorf("parse row %v at %d error %v, skip", values, i, err) - return dump.ErrSkip + return fmt.Errorf("parse row %v at %d error %v, float expected", values, i, err) } vs[i] = f + } else if tableInfo.Columns[i].Type == schema.TYPE_DECIMAL { + if h.c.cfg.UseDecimal { + d, err := decimal.NewFromString(v) + if err != nil { + return fmt.Errorf("parse row %v at %d error %v, decimal expected", values, i, err) + } + vs[i] = d + } else { + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return fmt.Errorf("parse row %v at %d error %v, float expected", values, i, err) + } + vs[i] = f + } + } else if strings.HasPrefix(v, "0x") { + buf, err := hex.DecodeString(v[2:]) + if err != nil { + return fmt.Errorf("parse row %v at %d error %v, hex literal expected", values, i, err) + } + vs[i] = string(buf) } else { - log.Errorf("parse row %v error, invalid type at %d, skip", values, i) - return dump.ErrSkip + return fmt.Errorf("parse row %v error, invalid type at %d", values, i) } } else { vs[i] = v[1 : len(v)-1] } } - events := newRowsEvent(tableInfo, InsertAction, [][]interface{}{vs}) - return h.c.travelRowsEventHandler(events) + events := newRowsEvent(tableInfo, InsertAction, [][]interface{}{vs}, nil) + return h.c.eventHandler.OnRow(events) } func (c *Canal) AddDumpDatabases(dbs ...string) { @@ -90,10 +120,64 @@ func (c *Canal) AddDumpIgnoreTables(db string, tables ...string) { c.dumper.AddIgnoreTables(db, tables...) } +func (c *Canal) dump() error { + if c.dumper == nil { + return errors.New("mysqldump does not exist") + } + + c.master.UpdateTimestamp(uint32(time.Now().Unix())) + + h := &dumpParseHandler{c: c} + // If users call StartFromGTID with empty position to start dumping with gtid, + // we record the current gtid position before dump starts. + // + // See tryDump() to see when dump is skipped. + if c.master.GTIDSet() != nil { + gset, err := c.GetMasterGTIDSet() + if err != nil { + return errors.Trace(err) + } + h.gset = gset + } + + if c.cfg.Dump.SkipMasterData { + pos, err := c.GetMasterPos() + if err != nil { + return errors.Trace(err) + } + log.Infof("skip master data, get current binlog position %v", pos) + h.name = pos.Name + h.pos = uint64(pos.Pos) + } + + start := time.Now() + log.Info("try dump MySQL and parse") + if err := c.dumper.DumpAndParse(h); err != nil { + return errors.Trace(err) + } + + pos := mysql.Position{Name: h.name, Pos: uint32(h.pos)} + c.master.Update(pos) + if err := c.eventHandler.OnPosSynced(pos, true); err != nil { + return errors.Trace(err) + } + var startPos fmt.Stringer = pos + if h.gset != nil { + c.master.UpdateGTIDSet(h.gset) + startPos = h.gset + } + log.Infof("dump MySQL and parse OK, use %0.2f seconds, start binlog replication at %s", + time.Now().Sub(start).Seconds(), startPos) + return nil +} + func (c *Canal) tryDump() error { - if len(c.master.Name) > 0 && c.master.Position > 0 { + pos := c.master.Position() + gset := c.master.GTIDSet() + if (len(pos.Name) > 0 && pos.Pos > 0) || + (gset != nil && gset.String() != "") { // we will sync with binlog name and position - log.Infof("skip dump, use last binlog replication pos (%s, %d)", c.master.Name, c.master.Position) + log.Infof("skip dump, use last binlog replication pos %s or GTID set %s", pos, gset) return nil } @@ -102,19 +186,5 @@ func (c *Canal) tryDump() error { return nil } - h := &dumpParseHandler{c: c} - - start := time.Now() - log.Info("try dump MySQL and parse") - if err := c.dumper.DumpAndParse(h); err != nil { - return errors.Trace(err) - } - - log.Infof("dump MySQL and parse OK, use %0.2f seconds, start binlog replication at (%s, %d)", - time.Now().Sub(start).Seconds(), h.name, h.pos) - - c.master.Update(h.name, uint32(h.pos)) - c.master.Save(true) - - return nil + return c.dump() } diff --git a/vendor/github.com/siddontang/go-mysql/canal/handler.go b/vendor/github.com/siddontang/go-mysql/canal/handler.go index e361181..4e47cb9 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/handler.go +++ b/vendor/github.com/siddontang/go-mysql/canal/handler.go @@ -1,41 +1,41 @@ package canal import ( - "github.com/juju/errors" - "github.com/ngaut/log" "github.com/siddontang/go-mysql/mysql" + "github.com/siddontang/go-mysql/replication" ) -var ( - ErrHandleInterrupted = errors.New("do handler error, interrupted") -) - -type RowsEventHandler interface { - // Handle RowsEvent, if return ErrHandleInterrupted, canal will - // stop the sync - Do(e *RowsEvent) error +type EventHandler interface { + OnRotate(roateEvent *replication.RotateEvent) error + // OnTableChanged is called when the table is created, altered, renamed or dropped. + // You need to clear the associated data like cache with the table. + // It will be called before OnDDL. + OnTableChanged(schema string, table string) error + OnDDL(nextPos mysql.Position, queryEvent *replication.QueryEvent) error + OnRow(e *RowsEvent) error + OnXID(nextPos mysql.Position) error + OnGTID(gtid mysql.GTIDSet) error + // OnPosSynced Use your own way to sync position. When force is true, sync position immediately. + OnPosSynced(pos mysql.Position, force bool) error String() string } -func (c *Canal) RegRowsEventHandler(h RowsEventHandler) { - c.rsLock.Lock() - c.rsHandlers = append(c.rsHandlers, h) - c.rsLock.Unlock() +type DummyEventHandler struct { } -func (c *Canal) travelRowsEventHandler(e *RowsEvent) error { - c.rsLock.Lock() - defer c.rsLock.Unlock() - - var err error - for _, h := range c.rsHandlers { - if err = h.Do(e); err != nil && !mysql.ErrorEqual(err, ErrHandleInterrupted) { - log.Errorf("handle %v err: %v", h, err) - } else if mysql.ErrorEqual(err, ErrHandleInterrupted) { - log.Errorf("handle %v err, interrupted", h) - return ErrHandleInterrupted - } - - } +func (h *DummyEventHandler) OnRotate(*replication.RotateEvent) error { return nil } +func (h *DummyEventHandler) OnTableChanged(schema string, table string) error { return nil } +func (h *DummyEventHandler) OnDDL(nextPos mysql.Position, queryEvent *replication.QueryEvent) error { return nil } +func (h *DummyEventHandler) OnRow(*RowsEvent) error { return nil } +func (h *DummyEventHandler) OnXID(mysql.Position) error { return nil } +func (h *DummyEventHandler) OnGTID(mysql.GTIDSet) error { return nil } +func (h *DummyEventHandler) OnPosSynced(mysql.Position, bool) error { return nil } +func (h *DummyEventHandler) String() string { return "DummyEventHandler" } + +// `SetEventHandler` registers the sync handler, you must register your +// own handler before starting Canal. +func (c *Canal) SetEventHandler(h EventHandler) { + c.eventHandler = h +} diff --git a/vendor/github.com/siddontang/go-mysql/canal/master.go b/vendor/github.com/siddontang/go-mysql/canal/master.go index 6897d05..10a230b 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/master.go +++ b/vendor/github.com/siddontang/go-mysql/canal/master.go @@ -1,89 +1,66 @@ package canal import ( - "bytes" - "os" "sync" - "time" - "github.com/BurntSushi/toml" - "github.com/juju/errors" - "github.com/ngaut/log" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" - "github.com/siddontang/go/ioutil2" ) type masterInfo struct { - Addr string `toml:"addr"` - Name string `toml:"bin_name"` - Position uint32 `toml:"bin_pos"` + sync.RWMutex - name string + pos mysql.Position - l sync.Mutex + gset mysql.GTIDSet - lastSaveTime time.Time + timestamp uint32 } -func loadMasterInfo(name string) (*masterInfo, error) { - var m masterInfo +func (m *masterInfo) Update(pos mysql.Position) { + log.Debugf("update master position %s", pos) - m.name = name - - f, err := os.Open(name) - if err != nil && !os.IsNotExist(errors.Cause(err)) { - return nil, errors.Trace(err) - } else if os.IsNotExist(errors.Cause(err)) { - return &m, nil - } - defer f.Close() - - _, err = toml.DecodeReader(f, &m) - - return &m, err + m.Lock() + m.pos = pos + m.Unlock() } -func (m *masterInfo) Save(force bool) error { - m.l.Lock() - defer m.l.Unlock() +func (m *masterInfo) UpdateTimestamp(ts uint32) { + log.Debugf("update master timestamp %s", ts) - n := time.Now() - if !force && n.Sub(m.lastSaveTime) < time.Second { + m.Lock() + m.timestamp = ts + m.Unlock() +} + +func (m *masterInfo) UpdateGTIDSet(gset mysql.GTIDSet) { + log.Debugf("update master gtid set %s", gset) + + m.Lock() + m.gset = gset + m.Unlock() +} + +func (m *masterInfo) Position() mysql.Position { + m.RLock() + defer m.RUnlock() + + return m.pos +} + +func (m *masterInfo) Timestamp() uint32 { + m.RLock() + defer m.RUnlock() + + return m.timestamp +} + +func (m *masterInfo) GTIDSet() mysql.GTIDSet { + m.RLock() + defer m.RUnlock() + + if m.gset == nil { return nil } - - var buf bytes.Buffer - e := toml.NewEncoder(&buf) - - e.Encode(m) - - var err error - if err = ioutil2.WriteFileAtomic(m.name, buf.Bytes(), 0644); err != nil { - log.Errorf("canal save master info to file %s err %v", m.name, err) - } - - m.lastSaveTime = n - - return errors.Trace(err) -} - -func (m *masterInfo) Update(name string, pos uint32) { - m.l.Lock() - m.Name = name - m.Position = pos - m.l.Unlock() -} - -func (m *masterInfo) Pos() mysql.Position { - var pos mysql.Position - m.l.Lock() - pos.Name = m.Name - pos.Pos = m.Position - m.l.Unlock() - - return pos -} - -func (m *masterInfo) Close() { - m.Save(true) + return m.gset.Clone() } diff --git a/vendor/github.com/siddontang/go-mysql/canal/rows.go b/vendor/github.com/siddontang/go-mysql/canal/rows.go index 5c5e467..e246ee5 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/rows.go +++ b/vendor/github.com/siddontang/go-mysql/canal/rows.go @@ -3,16 +3,18 @@ package canal import ( "fmt" - "github.com/juju/errors" + "github.com/siddontang/go-mysql/replication" "github.com/siddontang/go-mysql/schema" ) +// The action name for sync. const ( UpdateAction = "update" InsertAction = "insert" DeleteAction = "delete" ) +// RowsEvent is the event for row replication. type RowsEvent struct { Table *schema.Table Action string @@ -22,35 +24,49 @@ type RowsEvent struct { // Two rows for one event, format is [before update row, after update row] // for update v0, only one row for a event, and we don't support this version. Rows [][]interface{} + // Header can be used to inspect the event + Header *replication.EventHeader } -func newRowsEvent(table *schema.Table, action string, rows [][]interface{}) *RowsEvent { +func newRowsEvent(table *schema.Table, action string, rows [][]interface{}, header *replication.EventHeader) *RowsEvent { e := new(RowsEvent) e.Table = table e.Action = action e.Rows = rows + e.Header = header + + e.handleUnsigned() return e } -// Get primary keys in one row for a table, a table may use multi fields as the PK -func GetPKValues(table *schema.Table, row []interface{}) ([]interface{}, error) { - indexes := table.PKColumns - if len(indexes) == 0 { - return nil, errors.Errorf("table %s has no PK", table) - } else if len(table.Columns) != len(row) { - return nil, errors.Errorf("table %s has %d columns, but row data %v len is %d", table, - len(table.Columns), row, len(row)) +func (r *RowsEvent) handleUnsigned() { + // Handle Unsigned Columns here, for binlog replication, we can't know the integer is unsigned or not, + // so we use int type but this may cause overflow outside sometimes, so we must convert to the really . + // unsigned type + if len(r.Table.UnsignedColumns) == 0 { + return } - values := make([]interface{}, 0, len(indexes)) - - for _, index := range indexes { - values = append(values, row[index]) + for i := 0; i < len(r.Rows); i++ { + for _, index := range r.Table.UnsignedColumns { + switch t := r.Rows[i][index].(type) { + case int8: + r.Rows[i][index] = uint8(t) + case int16: + r.Rows[i][index] = uint16(t) + case int32: + r.Rows[i][index] = uint32(t) + case int64: + r.Rows[i][index] = uint64(t) + case int: + r.Rows[i][index] = uint(t) + default: + // nothing to do + } + } } - - return values, nil } // String implements fmt.Stringer interface. diff --git a/vendor/github.com/siddontang/go-mysql/canal/sync.go b/vendor/github.com/siddontang/go-mysql/canal/sync.go index e76eea4..4146a2d 100644 --- a/vendor/github.com/siddontang/go-mysql/canal/sync.go +++ b/vendor/github.com/siddontang/go-mysql/canal/sync.go @@ -1,49 +1,68 @@ package canal import ( + "fmt" + "regexp" "time" - "golang.org/x/net/context" - "github.com/juju/errors" - "github.com/ngaut/log" + "github.com/satori/go.uuid" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" + "github.com/siddontang/go-mysql/schema" ) -func (c *Canal) startSyncBinlog() error { - pos := mysql.Position{c.master.Name, c.master.Position} +var ( + expCreateTable = regexp.MustCompile("(?i)^CREATE\\sTABLE(\\sIF\\sNOT\\sEXISTS)?\\s`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s.*") + expAlterTable = regexp.MustCompile("(?i)^ALTER\\sTABLE\\s.*?`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s.*") + expRenameTable = regexp.MustCompile("(?i)^RENAME\\sTABLE\\s.*?`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s{1,}TO\\s.*?") + expDropTable = regexp.MustCompile("(?i)^DROP\\sTABLE(\\sIF\\sEXISTS){0,1}\\s`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}(?:$|\\s)") + expTruncateTable = regexp.MustCompile("(?i)^TRUNCATE\\s+(?:TABLE\\s+)?(?:`?([^`\\s]+)`?\\.`?)?([^`\\s]+)`?") +) - log.Infof("start sync binlog at %v", pos) +func (c *Canal) startSyncer() (*replication.BinlogStreamer, error) { + gset := c.master.GTIDSet() + if gset == nil { + pos := c.master.Position() + s, err := c.syncer.StartSync(pos) + if err != nil { + return nil, errors.Errorf("start sync replication at binlog %v error %v", pos, err) + } + log.Infof("start sync binlog at binlog file %v", pos) + return s, nil + } else { + s, err := c.syncer.StartSyncGTID(gset) + if err != nil { + return nil, errors.Errorf("start sync replication at GTID set %v error %v", gset, err) + } + log.Infof("start sync binlog at GTID set %v", gset) + return s, nil + } +} - s, err := c.syncer.StartSync(pos) +func (c *Canal) runSyncBinlog() error { + s, err := c.startSyncer() if err != nil { - return errors.Errorf("start sync replication at %v error %v", pos, err) + return err } - timeout := time.Second - forceSavePos := false + savePos := false + force := false for { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - ev, err := s.GetEvent(ctx) - cancel() - - if err == context.DeadlineExceeded { - timeout = 2 * timeout - continue - } + ev, err := s.GetEvent(c.ctx) if err != nil { return errors.Trace(err) } + savePos = false + force = false + pos := c.master.Position() - timeout = time.Second - + curPos := pos.Pos //next binlog pos pos.Pos = ev.Header.LogPos - forceSavePos = false - // We only save position with RotateEvent and XIDEvent. // For RowsEvent, we can't save the position until meeting XIDEvent // which tells the whole transaction is over. @@ -52,24 +71,105 @@ func (c *Canal) startSyncBinlog() error { case *replication.RotateEvent: pos.Name = string(e.NextLogName) pos.Pos = uint32(e.Position) - // r.ev <- pos - forceSavePos = true - log.Infof("rotate binlog to %v", pos) + log.Infof("rotate binlog to %s", pos) + savePos = true + force = true + if err = c.eventHandler.OnRotate(e); err != nil { + return errors.Trace(err) + } case *replication.RowsEvent: // we only focus row based event - if err = c.handleRowsEvent(ev); err != nil { - log.Errorf("handle rows event error %v", err) - return errors.Trace(err) + err = c.handleRowsEvent(ev) + if err != nil { + e := errors.Cause(err) + // if error is not ErrExcludedTable or ErrTableNotExist or ErrMissingTableMeta, stop canal + if e != ErrExcludedTable && + e != schema.ErrTableNotExist && + e != schema.ErrMissingTableMeta { + log.Errorf("handle rows event at (%s, %d) error %v", pos.Name, curPos, err) + return errors.Trace(err) + } } continue case *replication.XIDEvent: + if e.GSet != nil { + c.master.UpdateGTIDSet(e.GSet) + } + savePos = true // try to save the position later + if err := c.eventHandler.OnXID(pos); err != nil { + return errors.Trace(err) + } + case *replication.MariadbGTIDEvent: + // try to save the GTID later + gtid, err := mysql.ParseMariadbGTIDSet(e.GTID.String()) + if err != nil { + return errors.Trace(err) + } + if err := c.eventHandler.OnGTID(gtid); err != nil { + return errors.Trace(err) + } + case *replication.GTIDEvent: + u, _ := uuid.FromBytes(e.SID) + gtid, err := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d", u.String(), e.GNO)) + if err != nil { + return errors.Trace(err) + } + if err := c.eventHandler.OnGTID(gtid); err != nil { + return errors.Trace(err) + } + case *replication.QueryEvent: + if e.GSet != nil { + c.master.UpdateGTIDSet(e.GSet) + } + var ( + mb [][]byte + db []byte + table []byte + ) + regexps := []regexp.Regexp{*expCreateTable, *expAlterTable, *expRenameTable, *expDropTable, *expTruncateTable} + for _, reg := range regexps { + mb = reg.FindSubmatch(e.Query) + if len(mb) != 0 { + break + } + } + mbLen := len(mb) + if mbLen == 0 { + continue + } + + // the first last is table name, the second last is database name(if exists) + if len(mb[mbLen-2]) == 0 { + db = e.Schema + } else { + db = mb[mbLen-2] + } + table = mb[mbLen-1] + + savePos = true + force = true + c.ClearTableCache(db, table) + log.Infof("table structure changed, clear table cache: %s.%s\n", db, table) + if err = c.eventHandler.OnTableChanged(string(db), string(table)); err != nil && errors.Cause(err) != schema.ErrTableNotExist { + return errors.Trace(err) + } + + // Now we only handle Table Changed DDL, maybe we will support more later. + if err = c.eventHandler.OnDDL(pos, e); err != nil { + return errors.Trace(err) + } default: continue } - c.master.Update(pos.Name, pos.Pos) - c.master.Save(forceSavePos) + if savePos { + c.master.Update(pos) + c.master.UpdateTimestamp(ev.Header.Timestamp) + if err := c.eventHandler.OnPosSynced(pos, force); err != nil { + return errors.Trace(err) + } + } } return nil @@ -84,7 +184,7 @@ func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error { t, err := c.GetTable(schema, table) if err != nil { - return errors.Trace(err) + return err } var action string switch e.Header.EventType { @@ -97,25 +197,31 @@ func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error { default: return errors.Errorf("%s not supported now", e.Header.EventType) } - events := newRowsEvent(t, action, ev.Rows) - return c.travelRowsEventHandler(events) + events := newRowsEvent(t, action, ev.Rows, e.Header) + return c.eventHandler.OnRow(events) } -func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error { - if timeout <= 0 { - timeout = 60 - } +func (c *Canal) FlushBinlog() error { + _, err := c.Execute("FLUSH BINARY LOGS") + return errors.Trace(err) +} - timer := time.NewTimer(time.Duration(timeout) * time.Second) +func (c *Canal) WaitUntilPos(pos mysql.Position, timeout time.Duration) error { + timer := time.NewTimer(timeout) for { select { case <-timer.C: - return errors.Errorf("wait position %v err", pos) + return errors.Errorf("wait position %v too long > %s", pos, timeout) default: - curpos := c.master.Pos() - if curpos.Compare(pos) >= 0 { + err := c.FlushBinlog() + if err != nil { + return errors.Trace(err) + } + curPos := c.master.Position() + if curPos.Compare(pos) >= 0 { return nil } else { + log.Debugf("master pos is %v, wait catching %v", curPos, pos) time.Sleep(100 * time.Millisecond) } } @@ -124,14 +230,46 @@ func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error { return nil } -func (c *Canal) CatchMasterPos(timeout int) error { +func (c *Canal) GetMasterPos() (mysql.Position, error) { rr, err := c.Execute("SHOW MASTER STATUS") if err != nil { - return errors.Trace(err) + return mysql.Position{}, errors.Trace(err) } name, _ := rr.GetString(0, 0) pos, _ := rr.GetInt(0, 1) - return c.WaitUntilPos(mysql.Position{name, uint32(pos)}, timeout) + return mysql.Position{Name: name, Pos: uint32(pos)}, nil +} + +func (c *Canal) GetMasterGTIDSet() (mysql.GTIDSet, error) { + query := "" + switch c.cfg.Flavor { + case mysql.MariaDBFlavor: + query = "SELECT @@GLOBAL.gtid_current_pos" + default: + query = "SELECT @@GLOBAL.GTID_EXECUTED" + } + rr, err := c.Execute(query) + if err != nil { + return nil, errors.Trace(err) + } + gx, err := rr.GetString(0, 0) + if err != nil { + return nil, errors.Trace(err) + } + gset, err := mysql.ParseGTIDSet(c.cfg.Flavor, gx) + if err != nil { + return nil, errors.Trace(err) + } + return gset, nil +} + +func (c *Canal) CatchMasterPos(timeout time.Duration) error { + pos, err := c.GetMasterPos() + if err != nil { + return errors.Trace(err) + } + + return c.WaitUntilPos(pos, timeout) } diff --git a/vendor/github.com/siddontang/go-mysql/client/auth.go b/vendor/github.com/siddontang/go-mysql/client/auth.go index 85b688c..5ba9c9f 100644 --- a/vendor/github.com/siddontang/go-mysql/client/auth.go +++ b/vendor/github.com/siddontang/go-mysql/client/auth.go @@ -4,12 +4,29 @@ import ( "bytes" "crypto/tls" "encoding/binary" + "fmt" "github.com/juju/errors" . "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/packet" ) +const defaultAuthPluginName = AUTH_NATIVE_PASSWORD + +// defines the supported auth plugins +var supportedAuthPlugins = []string{AUTH_NATIVE_PASSWORD, AUTH_SHA256_PASSWORD, AUTH_CACHING_SHA2_PASSWORD} + +// helper function to determine what auth methods are allowed by this client +func authPluginAllowed(pluginName string) bool { + for _, p := range supportedAuthPlugins { + if pluginName == p { + return true + } + } + return false +} + +// See: http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake func (c *Conn) readInitialHandshake() error { data, err := c.ReadPacket() if err != nil { @@ -24,39 +41,44 @@ func (c *Conn) readInitialHandshake() error { return errors.Errorf("invalid protocol version %d, must >= 10", data[0]) } - //skip mysql version - //mysql version end with 0x00 + // skip mysql version + // mysql version end with 0x00 pos := 1 + bytes.IndexByte(data[1:], 0x00) + 1 - //connection id length is 4 + // connection id length is 4 c.connectionID = uint32(binary.LittleEndian.Uint32(data[pos : pos+4])) pos += 4 c.salt = []byte{} c.salt = append(c.salt, data[pos:pos+8]...) - //skip filter + // skip filter pos += 8 + 1 - //capability lower 2 bytes + // capability lower 2 bytes c.capability = uint32(binary.LittleEndian.Uint16(data[pos : pos+2])) - + // check protocol + if c.capability&CLIENT_PROTOCOL_41 == 0 { + return errors.New("the MySQL server can not support protocol 41 and above required by the client") + } + if c.capability&CLIENT_SSL == 0 && c.tlsConfig != nil { + return errors.New("the MySQL Server does not support TLS required by the client") + } pos += 2 if len(data) > pos { - //skip server charset + // skip server charset //c.charset = data[pos] pos += 1 c.status = binary.LittleEndian.Uint16(data[pos : pos+2]) pos += 2 - + // capability flags (upper 2 bytes) c.capability = uint32(binary.LittleEndian.Uint16(data[pos:pos+2]))<<16 | c.capability - pos += 2 - //skip auth data len or [00] - //skip reserved (all [00]) + // skip auth data len or [00] + // skip reserved (all [00]) pos += 10 + 1 // The documentation is ambiguous about the length. @@ -64,78 +86,131 @@ func (c *Conn) readInitialHandshake() error { // mysql-proxy also use 12 // which is not documented but seems to work. c.salt = append(c.salt, data[pos:pos+12]...) + pos += 13 + // auth plugin + if end := bytes.IndexByte(data[pos:], 0x00); end != -1 { + c.authPluginName = string(data[pos : pos+end]) + } else { + c.authPluginName = string(data[pos:]) + } + } + + // if server gives no default auth plugin name, use a client default + if c.authPluginName == "" { + c.authPluginName = defaultAuthPluginName } return nil } +// generate auth response data according to auth plugin +// +// NOTE: the returned boolean value indicates whether to add a \NUL to the end of data. +// it is quite tricky because MySQl server expects different formats of responses in different auth situations. +// here the \NUL needs to be added when sending back the empty password or cleartext password in 'sha256_password' +// authentication. +func (c *Conn) genAuthResponse(authData []byte) ([]byte, bool, error) { + // password hashing + switch c.authPluginName { + case AUTH_NATIVE_PASSWORD: + return CalcPassword(authData[:20], []byte(c.password)), false, nil + case AUTH_CACHING_SHA2_PASSWORD: + return CalcCachingSha2Password(authData, c.password), false, nil + case AUTH_SHA256_PASSWORD: + if len(c.password) == 0 { + return nil, true, nil + } + if c.tlsConfig != nil || c.proto == "unix" { + // write cleartext auth packet + // see: https://dev.mysql.com/doc/refman/8.0/en/sha256-pluggable-authentication.html + return []byte(c.password), true, nil + } else { + // request public key from server + // see: https://dev.mysql.com/doc/internals/en/public-key-retrieval.html + return []byte{1}, false, nil + } + default: + // not reachable + return nil, false, fmt.Errorf("auth plugin '%s' is not supported", c.authPluginName) + } +} + +// See: http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse func (c *Conn) writeAuthHandshake() error { + if !authPluginAllowed(c.authPluginName) { + return fmt.Errorf("unknow auth plugin name '%s'", c.authPluginName) + } // Adjust client capability flags based on server support capability := CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | - CLIENT_LONG_PASSWORD | CLIENT_TRANSACTIONS | CLIENT_LONG_FLAG + CLIENT_LONG_PASSWORD | CLIENT_TRANSACTIONS | CLIENT_PLUGIN_AUTH | c.capability&CLIENT_LONG_FLAG // To enable TLS / SSL - if c.TLSConfig != nil { - capability |= CLIENT_PLUGIN_AUTH + if c.tlsConfig != nil { capability |= CLIENT_SSL } - capability &= c.capability + auth, addNull, err := c.genAuthResponse(c.salt) + if err != nil { + return err + } + + // encode length of the auth plugin data + // here we use the Length-Encoded-Integer(LEI) as the data length may not fit into one byte + // see: https://dev.mysql.com/doc/internals/en/integer.html#length-encoded-integer + var authRespLEIBuf [9]byte + authRespLEI := AppendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(auth))) + if len(authRespLEI) > 1 { + // if the length can not be written in 1 byte, it must be written as a + // length encoded integer + capability |= CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA + } //packet length - //capbility 4 + //capability 4 //max-packet size 4 //charset 1 //reserved all[0] 23 - length := 4 + 4 + 1 + 23 - //username - length += len(c.user) + 1 - - //we only support secure connection - auth := CalcPassword(c.salt, []byte(c.password)) - - length += 1 + len(auth) - + //auth + //mysql_native_password + null-terminated + length := 4 + 4 + 1 + 23 + len(c.user) + 1 + len(authRespLEI) + len(auth) + 21 + 1 + if addNull { + length++ + } + // db name if len(c.db) > 0 { capability |= CLIENT_CONNECT_WITH_DB - length += len(c.db) + 1 } - // mysql_native_password + null-terminated - length += 21 + 1 - - c.capability = capability - data := make([]byte, length+4) - //capability [32 bit] + // capability [32 bit] data[4] = byte(capability) data[5] = byte(capability >> 8) data[6] = byte(capability >> 16) data[7] = byte(capability >> 24) - //MaxPacketSize [32 bit] (none) - //data[8] = 0x00 - //data[9] = 0x00 - //data[10] = 0x00 - //data[11] = 0x00 + // MaxPacketSize [32 bit] (none) + data[8] = 0x00 + data[9] = 0x00 + data[10] = 0x00 + data[11] = 0x00 - //Charset [1 byte] - //use default collation id 33 here, is utf-8 + // Charset [1 byte] + // use default collation id 33 here, is utf-8 data[12] = byte(DEFAULT_COLLATION_ID) // SSL Connection Request Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest - if c.TLSConfig != nil { + if c.tlsConfig != nil { // Send TLS / SSL request packet if err := c.WritePacket(data[:(4+4+1+23)+4]); err != nil { return err } // Switch to TLS - tlsConn := tls.Client(c.Conn.Conn, c.TLSConfig) + tlsConn := tls.Client(c.Conn.Conn, c.tlsConfig) if err := tlsConn.Handshake(); err != nil { return err } @@ -145,10 +220,13 @@ func (c *Conn) writeAuthHandshake() error { c.Sequence = currentSequence } - //Filler [23 bytes] (all 0x00) - pos := 13 + 23 + // Filler [23 bytes] (all 0x00) + pos := 13 + for ; pos < 13+23; pos++ { + data[pos] = 0 + } - //User [null terminated string] + // User [null terminated string] if len(c.user) > 0 { pos += copy(data[pos:], c.user) } @@ -156,8 +234,12 @@ func (c *Conn) writeAuthHandshake() error { pos++ // auth [length encoded integer] - data[pos] = byte(len(auth)) - pos += 1 + copy(data[pos+1:], auth) + pos += copy(data[pos:], authRespLEI) + pos += copy(data[pos:], auth) + if addNull { + data[pos] = 0x00 + pos++ + } // db [null terminated string] if len(c.db) > 0 { @@ -167,7 +249,7 @@ func (c *Conn) writeAuthHandshake() error { } // Assume native client during response - pos += copy(data[pos:], "mysql_native_password") + pos += copy(data[pos:], c.authPluginName) data[pos] = 0x00 return c.WritePacket(data) diff --git a/vendor/github.com/siddontang/go-mysql/client/client_test.go b/vendor/github.com/siddontang/go-mysql/client/client_test.go index 85dd8f2..04bfdb2 100644 --- a/vendor/github.com/siddontang/go-mysql/client/client_test.go +++ b/vendor/github.com/siddontang/go-mysql/client/client_test.go @@ -1,41 +1,56 @@ package client import ( - "crypto/tls" "flag" "fmt" "strings" "testing" + "github.com/juju/errors" . "github.com/pingcap/check" + "github.com/siddontang/go-mysql/test_util/test_keys" "github.com/siddontang/go-mysql/mysql" ) var testHost = flag.String("host", "127.0.0.1", "MySQL server host") -var testPort = flag.Int("port", 3306, "MySQL server port") +// We cover the whole range of MySQL server versions using docker-compose to bind them to different ports for testing. +// MySQL is constantly updating auth plugin to make it secure: +// starting from MySQL 8.0.4, a new auth plugin is introduced, causing plain password auth to fail with error: +// ERROR 1251 (08004): Client does not support authentication protocol requested by server; consider upgrading MySQL client +// Hint: use docker-compose to start corresponding MySQL docker containers and add the their ports here +var testPort = flag.String("port", "3306", "MySQL server port") // choose one or more form 5561,5641,3306,5722,8003,8012,8013, e.g. '3306,5722,8003' var testUser = flag.String("user", "root", "MySQL user") var testPassword = flag.String("pass", "", "MySQL password") var testDB = flag.String("db", "test", "MySQL test database") func Test(t *testing.T) { + segs := strings.Split(*testPort, ",") + for _, seg := range segs { + Suite(&clientTestSuite{port: seg}) + } TestingT(t) } type clientTestSuite struct { - c *Conn + c *Conn + port string } -var _ = Suite(&clientTestSuite{}) - func (s *clientTestSuite) SetUpSuite(c *C) { var err error - addr := fmt.Sprintf("%s:%d", *testHost, *testPort) - s.c, err = Connect(addr, *testUser, *testPassword, *testDB) + addr := fmt.Sprintf("%s:%s", *testHost, s.port) + s.c, err = Connect(addr, *testUser, *testPassword, "") if err != nil { c.Fatal(err) } + _, err = s.c.Execute("CREATE DATABASE IF NOT EXISTS " + *testDB) + c.Assert(err, IsNil) + + _, err = s.c.Execute("USE " + *testDB) + c.Assert(err, IsNil) + s.testConn_CreateTable(c) s.testStmt_CreateTable(c) } @@ -78,12 +93,15 @@ func (s *clientTestSuite) TestConn_Ping(c *C) { c.Assert(err, IsNil) } -func (s *clientTestSuite) TestConn_TLS(c *C) { +// NOTE for MySQL 5.5 and 5.6, server side has to config SSL to pass the TLS test, otherwise, it will throw error that +// MySQL server does not support TLS required by the client. However, for MySQL 5.7 and above, auto generated certificates +// are used by default so that manual config is no longer necessary. +func (s *clientTestSuite) TestConn_TLS_Verify(c *C) { // Verify that the provided tls.Config is used when attempting to connect to mysql. // An empty tls.Config will result in a connection error. - addr := fmt.Sprintf("%s:%d", *testHost, *testPort) + addr := fmt.Sprintf("%s:%s", *testHost, s.port) _, err := Connect(addr, *testUser, *testPassword, *testDB, func(c *Conn) { - c.TLSConfig = &tls.Config{} + c.UseSSL(false) }) if err == nil { c.Fatal("expected error") @@ -91,7 +109,34 @@ func (s *clientTestSuite) TestConn_TLS(c *C) { expected := "either ServerName or InsecureSkipVerify must be specified in the tls.Config" if !strings.Contains(err.Error(), expected) { - c.Fatal("expected '%s' to contain '%s'", err.Error(), expected) + c.Fatalf("expected '%s' to contain '%s'", err.Error(), expected) + } +} + +func (s *clientTestSuite) TestConn_TLS_Skip_Verify(c *C) { + // An empty tls.Config will result in a connection error but we can configure to skip it. + addr := fmt.Sprintf("%s:%s", *testHost, s.port) + _, err := Connect(addr, *testUser, *testPassword, *testDB, func(c *Conn) { + c.UseSSL(true) + }) + c.Assert(err, Equals, nil) +} + +func (s *clientTestSuite) TestConn_TLS_Certificate(c *C) { + // This test uses the TLS suite in 'go-mysql/docker/resources'. The certificates are not valid for any names. + // And if server uses auto-generated certificates, it will be an error like: + // "x509: certificate is valid for MySQL_Server_8.0.12_Auto_Generated_Server_Certificate, not not-a-valid-name" + tlsConfig := NewClientTLSConfig(test_keys.CaPem, test_keys.CertPem, test_keys.KeyPem, false, "not-a-valid-name") + addr := fmt.Sprintf("%s:%s", *testHost, s.port) + _, err := Connect(addr, *testUser, *testPassword, *testDB, func(c *Conn) { + c.SetTLSConfig(tlsConfig) + }) + if err == nil { + c.Fatal("expected error") + } + if !strings.Contains(errors.Details(err), "certificate is not valid for any names") && + !strings.Contains(errors.Details(err), "certificate is valid for") { + c.Fatalf("expected errors for server name verification, but got unknown error: %s", errors.Details(err)) } } @@ -349,4 +394,4 @@ func (s *clientTestSuite) TestStmt_Trans(c *C) { str, _ = r.GetString(0, 0) c.Assert(str, Equals, `abc`) -} +} \ No newline at end of file diff --git a/vendor/github.com/siddontang/go-mysql/client/conn.go b/vendor/github.com/siddontang/go-mysql/client/conn.go index 54ee3f0..b015b43 100644 --- a/vendor/github.com/siddontang/go-mysql/client/conn.go +++ b/vendor/github.com/siddontang/go-mysql/client/conn.go @@ -18,7 +18,8 @@ type Conn struct { user string password string db string - TLSConfig *tls.Config + tlsConfig *tls.Config + proto string capability uint32 @@ -26,7 +27,8 @@ type Conn struct { charset string - salt []byte + salt []byte + authPluginName string connectionID uint32 } @@ -56,6 +58,7 @@ func Connect(addr string, user string, password string, dbName string, options . c.user = user c.password = password c.db = dbName + c.proto = proto //use default charset here, utf-8 c.charset = DEFAULT_CHARSET @@ -85,7 +88,7 @@ func (c *Conn) handshake() error { return errors.Trace(err) } - if _, err := c.readOK(); err != nil { + if err := c.handleAuthResult(); err != nil { c.Close() return errors.Trace(err) } @@ -109,6 +112,18 @@ func (c *Conn) Ping() error { return nil } +// use default SSL +// pass to options when connect +func (c *Conn) UseSSL(insecureSkipVerify bool) { + c.tlsConfig = &tls.Config{InsecureSkipVerify: insecureSkipVerify} +} + +// use user-specified TLS config +// pass to options when connect +func (c *Conn) SetTLSConfig(config *tls.Config) { + c.tlsConfig = config +} + func (c *Conn) UseDB(dbName string) error { if c.db == dbName { return nil diff --git a/vendor/github.com/siddontang/go-mysql/client/resp.go b/vendor/github.com/siddontang/go-mysql/client/resp.go index 4e5f855..71aa1bc 100644 --- a/vendor/github.com/siddontang/go-mysql/client/resp.go +++ b/vendor/github.com/siddontang/go-mysql/client/resp.go @@ -1,8 +1,14 @@ package client +import "C" import ( "encoding/binary" + "bytes" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "github.com/juju/errors" . "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go/hack" @@ -32,7 +38,7 @@ func (c *Conn) isEOFPacket(data []byte) bool { func (c *Conn) handleOKPacket(data []byte) (*Result, error) { var n int - var pos int = 1 + var pos = 1 r := new(Result) @@ -64,7 +70,7 @@ func (c *Conn) handleOKPacket(data []byte) (*Result, error) { func (c *Conn) handleErrorPacket(data []byte) error { e := new(MyError) - var pos int = 1 + var pos = 1 e.Code = binary.LittleEndian.Uint16(data[pos:]) pos += 2 @@ -81,6 +87,116 @@ func (c *Conn) handleErrorPacket(data []byte) error { return e } +func (c *Conn) handleAuthResult() error { + data, switchToPlugin, err := c.readAuthResult() + if err != nil { + return err + } + // handle auth switch, only support 'sha256_password', and 'caching_sha2_password' + if switchToPlugin != "" { + //fmt.Printf("now switching auth plugin to '%s'\n", switchToPlugin) + if data == nil { + data = c.salt + } else { + copy(c.salt, data) + } + c.authPluginName = switchToPlugin + auth, addNull, err := c.genAuthResponse(data) + if err = c.WriteAuthSwitchPacket(auth, addNull); err != nil { + return err + } + + // Read Result Packet + data, switchToPlugin, err = c.readAuthResult() + if err != nil { + return err + } + + // Do not allow to change the auth plugin more than once + if switchToPlugin != "" { + return errors.Errorf("can not switch auth plugin more than once") + } + } + + // handle caching_sha2_password + if c.authPluginName == AUTH_CACHING_SHA2_PASSWORD { + if data == nil { + return nil // auth already succeeded + } + if data[0] == CACHE_SHA2_FAST_AUTH { + if _, err = c.readOK(); err == nil { + return nil // auth successful + } + } else if data[0] == CACHE_SHA2_FULL_AUTH { + // need full authentication + if c.tlsConfig != nil || c.proto == "unix" { + if err = c.WriteClearAuthPacket(c.password); err != nil { + return err + } + } else { + if err = c.WritePublicKeyAuthPacket(c.password, c.salt); err != nil { + return err + } + } + } else { + errors.Errorf("invalid packet") + } + } else if c.authPluginName == AUTH_SHA256_PASSWORD { + if len(data) == 0 { + return nil // auth already succeeded + } + block, _ := pem.Decode(data) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + // send encrypted password + err = c.WriteEncryptedPassword(c.password, c.salt, pub.(*rsa.PublicKey)) + if err != nil { + return err + } + _, err = c.readOK() + return err + } + return nil +} + +func (c *Conn) readAuthResult() ([]byte, string, error) { + data, err := c.ReadPacket() + if err != nil { + return nil, "", err + } + + // see: https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/ + // packet indicator + switch data[0] { + + case OK_HEADER: + _, err := c.handleOKPacket(data) + return nil, "", err + + case MORE_DATE_HEADER: + return data[1:], "", err + + case EOF_HEADER: + // server wants to switch auth + if len(data) < 1 { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest + return nil, AUTH_MYSQL_OLD_PASSWORD, nil + } + pluginEndIndex := bytes.IndexByte(data, 0x00) + if pluginEndIndex < 0 { + return nil, "", errors.New("invalid packet") + } + plugin := string(data[1:pluginEndIndex]) + authData := data[pluginEndIndex+1:] + return authData, plugin, nil + + default: // Error otherwise + return nil, "", c.handleErrorPacket(data) + } +} + func (c *Conn) readOK() (*Result, error) { data, err := c.ReadPacket() if err != nil { diff --git a/vendor/github.com/siddontang/go-mysql/cmd/go-binlogparser/main.go b/vendor/github.com/siddontang/go-mysql/cmd/go-binlogparser/main.go index 3990d00..fa5b5cf 100644 --- a/vendor/github.com/siddontang/go-mysql/cmd/go-binlogparser/main.go +++ b/vendor/github.com/siddontang/go-mysql/cmd/go-binlogparser/main.go @@ -23,6 +23,6 @@ func main() { err := p.ParseFile(*name, *offset, f) if err != nil { - println(err) + println(err.Error()) } } diff --git a/vendor/github.com/siddontang/go-mysql/cmd/go-canal/main.go b/vendor/github.com/siddontang/go-mysql/cmd/go-canal/main.go index 4d0c1df..6f60a4a 100644 --- a/vendor/github.com/siddontang/go-mysql/cmd/go-canal/main.go +++ b/vendor/github.com/siddontang/go-mysql/cmd/go-canal/main.go @@ -7,8 +7,10 @@ import ( "os/signal" "strings" "syscall" + "time" "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" ) var host = flag.String("host", "127.0.0.1", "MySQL host") @@ -18,8 +20,6 @@ var password = flag.String("password", "", "MySQL password") var flavor = flag.String("flavor", "mysql", "Flavor: mysql or mariadb") -var dataDir = flag.String("data-dir", "./var", "Path to store data, like master.info") - var serverID = flag.Int("server-id", 101, "Unique Server ID") var mysqldump = flag.String("mysqldump", "mysqldump", "mysqldump execution path") @@ -28,6 +28,12 @@ var tables = flag.String("tables", "", "dump tables, seperated by comma, will ov var tableDB = flag.String("table_db", "test", "database for dump tables") var ignoreTables = flag.String("ignore_tables", "", "ignore tables, must be database.table format, separated by comma") +var startName = flag.String("bin_name", "", "start sync from binlog name") +var startPos = flag.Uint("bin_pos", 0, "start sync from binlog position of") + +var heartbeatPeriod = flag.Duration("heartbeat", 60*time.Second, "master heartbeat period") +var readTimeout = flag.Duration("read_timeout", 90*time.Second, "connection read timeout") + func main() { flag.Parse() @@ -36,8 +42,10 @@ func main() { cfg.User = *user cfg.Password = *password cfg.Flavor = *flavor - cfg.DataDir = *dataDir + cfg.UseDecimal = true + cfg.ReadTimeout = *readTimeout + cfg.HeartbeatPeriod = *heartbeatPeriod cfg.ServerID = uint32(*serverID) cfg.Dump.ExecutionPath = *mysqldump cfg.Dump.DiscardErr = false @@ -65,14 +73,20 @@ func main() { c.AddDumpDatabases(subs...) } - c.RegRowsEventHandler(&handler{}) + c.SetEventHandler(&handler{}) - err = c.Start() - if err != nil { - fmt.Printf("start canal err %V", err) - os.Exit(1) + startPos := mysql.Position{ + Name: *startName, + Pos: uint32(*startPos), } + go func() { + err = c.RunFrom(startPos) + if err != nil { + fmt.Printf("start canal err %v", err) + } + }() + sc := make(chan os.Signal, 1) signal.Notify(sc, os.Kill, @@ -88,9 +102,10 @@ func main() { } type handler struct { + canal.DummyEventHandler } -func (h *handler) Do(e *canal.RowsEvent) error { +func (h *handler) OnRow(e *canal.RowsEvent) error { fmt.Printf("%v\n", e) return nil diff --git a/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go b/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go index 80fa73d..2c19c87 100644 --- a/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go +++ b/vendor/github.com/siddontang/go-mysql/cmd/go-mysqlbinlog/main.go @@ -4,12 +4,11 @@ package main import ( + "context" "flag" "fmt" "os" - "golang.org/x/net/context" - "github.com/juju/errors" "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" @@ -43,11 +42,12 @@ func main() { Password: *password, RawModeEnabled: *rawMode, SemiSyncEnabled: *semiSync, + UseDecimal: true, } b := replication.NewBinlogSyncer(cfg) - pos := mysql.Position{*file, uint32(*pos)} + pos := mysql.Position{Name: *file, Pos: uint32(*pos)} if len(*backupPath) > 0 { // Backup will always use RawMode. err := b.StartBackup(*backupPath, pos, 0) @@ -65,6 +65,11 @@ func main() { for { e, err := s.GetEvent(context.Background()) if err != nil { + // Try to output all left events + events := s.DumpEvents() + for _, e := range events { + e.Dump(os.Stdout) + } fmt.Printf("Get event error: %v\n", errors.ErrorStack(err)) return } diff --git a/vendor/github.com/siddontang/go-mysql/driver/dirver_test.go b/vendor/github.com/siddontang/go-mysql/driver/dirver_test.go index 54a72cb..d43580f 100644 --- a/vendor/github.com/siddontang/go-mysql/driver/dirver_test.go +++ b/vendor/github.com/siddontang/go-mysql/driver/dirver_test.go @@ -11,6 +11,11 @@ import ( // Use docker mysql to test, mysql is 3306 var testHost = flag.String("host", "127.0.0.1", "MySQL master host") +// possible choices for different MySQL versions are: 5561,5641,3306,5722,8003,8012 +var testPort = flag.Int("port", 3306, "MySQL server port") +var testUser = flag.String("user", "root", "MySQL user") +var testPassword = flag.String("pass", "", "MySQL password") +var testDB = flag.String("db", "test", "MySQL test database") func TestDriver(t *testing.T) { TestingT(t) @@ -23,7 +28,8 @@ type testDriverSuite struct { var _ = Suite(&testDriverSuite{}) func (s *testDriverSuite) SetUpSuite(c *C) { - dsn := fmt.Sprintf("root@%s:3306?test", *testHost) + addr := fmt.Sprintf("%s:%d", *testHost, *testPort) + dsn := fmt.Sprintf("%s:%s@%s?%s", *testUser, *testPassword, addr, *testDB) var err error s.db, err = sqlx.Open("mysql", dsn) diff --git a/vendor/github.com/siddontang/go-mysql/driver/driver.go b/vendor/github.com/siddontang/go-mysql/driver/driver.go index 6263929..e131548 100644 --- a/vendor/github.com/siddontang/go-mysql/driver/driver.go +++ b/vendor/github.com/siddontang/go-mysql/driver/driver.go @@ -20,7 +20,8 @@ type driver struct { // DSN user:password@addr[?db] func (d driver) Open(dsn string) (sqldriver.Conn, error) { - seps := strings.Split(dsn, "@") + lastIndex := strings.LastIndex(dsn, "@") + seps := []string{dsn[:lastIndex], dsn[lastIndex+1:]} if len(seps) != 2 { return nil, errors.Errorf("invalid dsn, must user:password@addr[?db]") } diff --git a/vendor/github.com/siddontang/go-mysql/dump/dump.go b/vendor/github.com/siddontang/go-mysql/dump/dump.go index a6ff209..1f8384d 100644 --- a/vendor/github.com/siddontang/go-mysql/dump/dump.go +++ b/vendor/github.com/siddontang/go-mysql/dump/dump.go @@ -8,6 +8,8 @@ import ( "strings" "github.com/juju/errors" + "github.com/siddontang/go-log/log" + . "github.com/siddontang/go-mysql/mysql" ) // Unlick mysqldump, Dumper is designed for parsing and syning data easily. @@ -25,9 +27,16 @@ type Dumper struct { Databases []string + Where string + Charset string + IgnoreTables map[string][]string ErrOut io.Writer + + masterDataSkipped bool + maxAllowedPacket int + hexBlob bool } func NewDumper(executionPath string, addr string, user string, password string) (*Dumper, error) { @@ -47,17 +56,40 @@ func NewDumper(executionPath string, addr string, user string, password string) d.Password = password d.Tables = make([]string, 0, 16) d.Databases = make([]string, 0, 16) + d.Charset = DEFAULT_CHARSET d.IgnoreTables = make(map[string][]string) + d.masterDataSkipped = false d.ErrOut = os.Stderr return d, nil } +func (d *Dumper) SetCharset(charset string) { + d.Charset = charset +} + +func (d *Dumper) SetWhere(where string) { + d.Where = where +} + func (d *Dumper) SetErrOut(o io.Writer) { d.ErrOut = o } +// In some cloud MySQL, we have no privilege to use `--master-data`. +func (d *Dumper) SkipMasterData(v bool) { + d.masterDataSkipped = v +} + +func (d *Dumper) SetMaxAllowedPacket(i int) { + d.maxAllowedPacket = i +} + +func (d *Dumper) SetHexBlob(v bool) { + d.hexBlob = v +} + func (d *Dumper) AddDatabases(dbs ...string) { d.Databases = append(d.Databases, dbs...) } @@ -82,22 +114,35 @@ func (d *Dumper) Reset() { d.TableDB = "" d.IgnoreTables = make(map[string][]string) d.Databases = d.Databases[0:0] + d.Where = "" } func (d *Dumper) Dump(w io.Writer) error { args := make([]string, 0, 16) // Common args - seps := strings.Split(d.Addr, ":") - args = append(args, fmt.Sprintf("--host=%s", seps[0])) - if len(seps) > 1 { - args = append(args, fmt.Sprintf("--port=%s", seps[1])) + if strings.Contains(d.Addr, "/") { + args = append(args, fmt.Sprintf("--socket=%s", d.Addr)) + } else { + seps := strings.SplitN(d.Addr, ":", 2) + args = append(args, fmt.Sprintf("--host=%s", seps[0])) + if len(seps) > 1 { + args = append(args, fmt.Sprintf("--port=%s", seps[1])) + } } args = append(args, fmt.Sprintf("--user=%s", d.User)) args = append(args, fmt.Sprintf("--password=%s", d.Password)) - args = append(args, "--master-data") + if !d.masterDataSkipped { + args = append(args, "--master-data") + } + + if d.maxAllowedPacket > 0 { + // mysqldump param should be --max-allowed-packet=%dM not be --max_allowed_packet=%dM + args = append(args, fmt.Sprintf("--max-allowed-packet=%dM", d.maxAllowedPacket)) + } + args = append(args, "--single-transaction") args = append(args, "--skip-lock-tables") @@ -112,12 +157,25 @@ func (d *Dumper) Dump(w io.Writer) error { // Multi row is easy for us to parse the data args = append(args, "--skip-extended-insert") + if d.hexBlob { + // Use hex for the binary type + args = append(args, "--hex-blob") + } + for db, tables := range d.IgnoreTables { for _, table := range tables { args = append(args, fmt.Sprintf("--ignore-table=%s.%s", db, table)) } } + if len(d.Charset) != 0 { + args = append(args, fmt.Sprintf("--default-character-set=%s", d.Charset)) + } + + if len(d.Where) != 0 { + args = append(args, fmt.Sprintf("--where=%s", d.Where)) + } + if len(d.Tables) == 0 && len(d.Databases) == 0 { args = append(args, "--all-databases") } else if len(d.Tables) == 0 { @@ -133,6 +191,7 @@ func (d *Dumper) Dump(w io.Writer) error { w.Write([]byte(fmt.Sprintf("USE `%s`;\n", d.TableDB))) } + log.Infof("exec mysqldump with %v", args) cmd := exec.Command(d.ExecutionPath, args...) cmd.Stderr = d.ErrOut @@ -147,7 +206,7 @@ func (d *Dumper) DumpAndParse(h ParseHandler) error { done := make(chan error, 1) go func() { - err := Parse(r, h) + err := Parse(r, h, !d.masterDataSkipped) r.CloseWithError(err) done <- err }() diff --git a/vendor/github.com/siddontang/go-mysql/dump/dump_test.go b/vendor/github.com/siddontang/go-mysql/dump/dump_test.go index 39e430f..eed4c75 100644 --- a/vendor/github.com/siddontang/go-mysql/dump/dump_test.go +++ b/vendor/github.com/siddontang/go-mysql/dump/dump_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "os" + "strings" "testing" . "github.com/pingcap/check" @@ -38,6 +39,7 @@ func (s *schemaTestSuite) SetUpSuite(c *C) { c.Assert(err, IsNil) c.Assert(s.d, NotNil) + s.d.SetCharset("utf8") s.d.SetErrOut(os.Stderr) _, err = s.conn.Execute("CREATE DATABASE IF NOT EXISTS test1") @@ -177,7 +179,7 @@ func (s *schemaTestSuite) TestParse(c *C) { err := s.d.Dump(&buf) c.Assert(err, IsNil) - err = Parse(&buf, new(testParseHandler)) + err = Parse(&buf, new(testParseHandler), true) c.Assert(err, IsNil) } @@ -196,3 +198,29 @@ func (s *parserTestSuite) TestParseValue(c *C) { values, err = parseValues(str) c.Assert(err, NotNil) } + +func (s *parserTestSuite) TestParseLine(c *C) { + lines := []struct { + line string + expected string + }{ + {line: "INSERT INTO `test` VALUES (1, 'first', 'hello mysql; 2', 'e1', 'a,b');", + expected: "1, 'first', 'hello mysql; 2', 'e1', 'a,b'"}, + {line: "INSERT INTO `test` VALUES (0x22270073646661736661736466, 'first', 'hello mysql; 2', 'e1', 'a,b');", + expected: "0x22270073646661736661736466, 'first', 'hello mysql; 2', 'e1', 'a,b'"}, + } + + f := func(c rune) bool { + return c == '\r' || c == '\n' + } + + for _, t := range lines { + l := strings.TrimRightFunc(t.line, f) + + m := valuesExp.FindAllStringSubmatch(l, -1) + + c.Assert(m, HasLen, 1) + c.Assert(m[0][1], Matches, "test") + c.Assert(m[0][2], Matches, t.expected) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/dump/parser.go b/vendor/github.com/siddontang/go-mysql/dump/parser.go index 69f65c7..ad40925 100644 --- a/vendor/github.com/siddontang/go-mysql/dump/parser.go +++ b/vendor/github.com/siddontang/go-mysql/dump/parser.go @@ -6,6 +6,7 @@ import ( "io" "regexp" "strconv" + "strings" "github.com/juju/errors" "github.com/siddontang/go-mysql/mysql" @@ -34,7 +35,7 @@ func init() { // Parse the dump data with Dumper generate. // It can not parse all the data formats with mysqldump outputs -func Parse(r io.Reader, h ParseHandler) error { +func Parse(r io.Reader, h ParseHandler, parseBinlogPos bool) error { rb := bufio.NewReaderSize(r, 1024*16) var db string @@ -48,9 +49,12 @@ func Parse(r io.Reader, h ParseHandler) error { break } - line = line[0 : len(line)-1] + // Ignore '\n' on Linux or '\r\n' on Windows + line = strings.TrimRightFunc(line, func(c rune) bool { + return c == '\r' || c == '\n' + }) - if !binlogParsed { + if parseBinlogPos && !binlogParsed { if m := binlogExp.FindAllStringSubmatch(line, -1); len(m) == 1 { name := m[0][1] pos, err := strconv.ParseUint(m[0][2], 10, 64) diff --git a/vendor/github.com/siddontang/go-mysql/failover/mariadb_gtid_handler.go b/vendor/github.com/siddontang/go-mysql/failover/mariadb_gtid_handler.go index 5c8bd64..2798241 100644 --- a/vendor/github.com/siddontang/go-mysql/failover/mariadb_gtid_handler.go +++ b/vendor/github.com/siddontang/go-mysql/failover/mariadb_gtid_handler.go @@ -46,12 +46,12 @@ func (h *MariadbGTIDHandler) FindBestSlaves(slaves []*Server) ([]*Server, error) if len(str) == 0 { seq = 0 } else { - g, err := ParseMariadbGTIDSet(str) + g, err := ParseMariadbGTID(str) if err != nil { return nil, errors.Trace(err) } - seq = g.(MariadbGTID).SequenceNumber + seq = g.SequenceNumber } ps[i] = seq @@ -118,7 +118,7 @@ func (h *MariadbGTIDHandler) WaitRelayLogDone(s *Server) error { fname, _ := r.GetStringByName(0, "Master_Log_File") pos, _ := r.GetIntByName(0, "Read_Master_Log_Pos") - return s.MasterPosWait(Position{fname, uint32(pos)}, 0) + return s.MasterPosWait(Position{Name: fname, Pos: uint32(pos)}, 0) } func (h *MariadbGTIDHandler) WaitCatchMaster(s *Server, m *Server) error { diff --git a/vendor/github.com/siddontang/go-mysql/failover/server.go b/vendor/github.com/siddontang/go-mysql/failover/server.go index ff87464..c02d6c8 100644 --- a/vendor/github.com/siddontang/go-mysql/failover/server.go +++ b/vendor/github.com/siddontang/go-mysql/failover/server.go @@ -152,7 +152,7 @@ func (s *Server) FetchSlaveReadPos() (Position, error) { fname, _ := r.GetStringByName(0, "Master_Log_File") pos, _ := r.GetIntByName(0, "Read_Master_Log_Pos") - return Position{fname, uint32(pos)}, nil + return Position{Name: fname, Pos: uint32(pos)}, nil } // Get current executed binlog filename and position from master @@ -165,7 +165,7 @@ func (s *Server) FetchSlaveExecutePos() (Position, error) { fname, _ := r.GetStringByName(0, "Relay_Master_Log_File") pos, _ := r.GetIntByName(0, "Exec_Master_Log_Pos") - return Position{fname, uint32(pos)}, nil + return Position{Name: fname, Pos: uint32(pos)}, nil } func (s *Server) MasterPosWait(pos Position, timeout int) error { diff --git a/vendor/github.com/siddontang/go-mysql/glide.lock b/vendor/github.com/siddontang/go-mysql/glide.lock deleted file mode 100644 index 7d71953..0000000 --- a/vendor/github.com/siddontang/go-mysql/glide.lock +++ /dev/null @@ -1,30 +0,0 @@ -hash: 1a3d05afef96cd7601a004e573b128db9051eecd1b5d0a3d69d3fa1ee1a3e3b8 -updated: 2016-09-03T12:30:00.028685232+08:00 -imports: -- name: github.com/BurntSushi/toml - version: 056c9bc7be7190eaa7715723883caffa5f8fa3e4 -- name: github.com/go-sql-driver/mysql - version: 3654d25ec346ee8ce71a68431025458d52a38ac0 -- name: github.com/jmoiron/sqlx - version: 54aec3fd91a2b2129ffaca0d652b8a9223ee2d9e - subpackages: - - reflectx -- name: github.com/juju/errors - version: 6f54ff6318409d31ff16261533ce2c8381a4fd5d -- name: github.com/ngaut/log - version: cec23d3e10b016363780d894a0eb732a12c06e02 -- name: github.com/pingcap/check - version: ce8a2f822ab1e245a4eefcef2996531c79c943f1 -- name: github.com/satori/go.uuid - version: 879c5887cd475cd7864858769793b2ceb0d44feb -- name: github.com/siddontang/go - version: 354e14e6c093c661abb29fd28403b3c19cff5514 - subpackages: - - hack - - ioutil2 - - sync2 -- name: golang.org/x/net - version: 6acef71eb69611914f7a30939ea9f6e194c78172 - subpackages: - - context -testImports: [] diff --git a/vendor/github.com/siddontang/go-mysql/glide.yaml b/vendor/github.com/siddontang/go-mysql/glide.yaml deleted file mode 100644 index 3561648..0000000 --- a/vendor/github.com/siddontang/go-mysql/glide.yaml +++ /dev/null @@ -1,26 +0,0 @@ -package: github.com/siddontang/go-mysql -import: -- package: github.com/BurntSushi/toml - version: 056c9bc7be7190eaa7715723883caffa5f8fa3e4 -- package: github.com/go-sql-driver/mysql - version: 3654d25ec346ee8ce71a68431025458d52a38ac0 -- package: github.com/jmoiron/sqlx - version: 54aec3fd91a2b2129ffaca0d652b8a9223ee2d9e - subpackages: - - reflectx -- package: github.com/juju/errors - version: 6f54ff6318409d31ff16261533ce2c8381a4fd5d -- package: github.com/ngaut/log - version: cec23d3e10b016363780d894a0eb732a12c06e02 -- package: github.com/pingcap/check - version: ce8a2f822ab1e245a4eefcef2996531c79c943f1 -- package: github.com/satori/go.uuid - version: ^1.1.0 -- package: github.com/siddontang/go - version: 354e14e6c093c661abb29fd28403b3c19cff5514 - subpackages: - - hack - - ioutil2 - - sync2 -- package: golang.org/x/net - version: 6acef71eb69611914f7a30939ea9f6e194c78172 diff --git a/vendor/github.com/siddontang/go-mysql/mysql/const.go b/vendor/github.com/siddontang/go-mysql/mysql/const.go index 2f4ab63..256d163 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/const.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/const.go @@ -6,16 +6,22 @@ const ( TimeFormat string = "2006-01-02 15:04:05" ) -var ( - // maybe you can change for your specified name - ServerVersion string = "go-mysql-0.1" -) - const ( OK_HEADER byte = 0x00 + MORE_DATE_HEADER byte = 0x01 ERR_HEADER byte = 0xff EOF_HEADER byte = 0xfe LocalInFile_HEADER byte = 0xfb + + CACHE_SHA2_FAST_AUTH byte = 0x03 + CACHE_SHA2_FULL_AUTH byte = 0x04 +) + +const ( + AUTH_MYSQL_OLD_PASSWORD = "mysql_old_password" + AUTH_NATIVE_PASSWORD = "mysql_native_password" + AUTH_CACHING_SHA2_PASSWORD = "caching_sha2_password" + AUTH_SHA256_PASSWORD = "sha256_password" ) const ( @@ -151,7 +157,6 @@ const ( ) const ( - AUTH_NAME = "mysql_native_password" DEFAULT_CHARSET = "utf8" DEFAULT_COLLATION_ID uint8 = 33 DEFAULT_COLLATION_NAME string = "utf8_general_ci" diff --git a/vendor/github.com/siddontang/go-mysql/mysql/error.go b/vendor/github.com/siddontang/go-mysql/mysql/error.go index 227e70e..876a408 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/error.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/error.go @@ -57,3 +57,10 @@ func NewError(errCode uint16, message string) *MyError { return e } + +func ErrorCode(errMsg string) (code int) { + var tmpStr string + // golang scanf doesn't support %*,so I used a temporary variable + fmt.Sscanf(errMsg, "%s%d", &tmpStr, &code) + return +} diff --git a/vendor/github.com/siddontang/go-mysql/mysql/field.go b/vendor/github.com/siddontang/go-mysql/mysql/field.go index c26f6a2..891f00b 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/field.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/field.go @@ -31,42 +31,42 @@ func (p FieldData) Parse() (f *Field, err error) { var n int pos := 0 //skip catelog, always def - n, err = SkipLengthEnodedString(p) + n, err = SkipLengthEncodedString(p) if err != nil { return } pos += n //schema - f.Schema, _, n, err = LengthEnodedString(p[pos:]) + f.Schema, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return } pos += n //table - f.Table, _, n, err = LengthEnodedString(p[pos:]) + f.Table, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return } pos += n //org_table - f.OrgTable, _, n, err = LengthEnodedString(p[pos:]) + f.OrgTable, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return } pos += n //name - f.Name, _, n, err = LengthEnodedString(p[pos:]) + f.Name, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return } pos += n //org_name - f.OrgName, _, n, err = LengthEnodedString(p[pos:]) + f.OrgName, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return } diff --git a/vendor/github.com/siddontang/go-mysql/mysql/gtid.go b/vendor/github.com/siddontang/go-mysql/mysql/gtid.go index db0d638..cde9901 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/gtid.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/gtid.go @@ -11,6 +11,10 @@ type GTIDSet interface { Equal(o GTIDSet) bool Contain(o GTIDSet) bool + + Update(GTIDStr string) error + + Clone() GTIDSet } func ParseGTIDSet(flavor string, s string) (GTIDSet, error) { diff --git a/vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid.go b/vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid.go index ea89458..09fe7ac 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid.go @@ -1,28 +1,32 @@ package mysql import ( + "bytes" "fmt" "strconv" "strings" "github.com/juju/errors" + "github.com/siddontang/go-log/log" + "github.com/siddontang/go/hack" ) +// MariadbGTID represent mariadb gtid, [domain ID]-[server-id]-[sequence] type MariadbGTID struct { DomainID uint32 ServerID uint32 SequenceNumber uint64 } -// We don't support multi source replication, so the mariadb gtid set may have only domain-server-sequence -func ParseMariadbGTIDSet(str string) (GTIDSet, error) { +// ParseMariadbGTID parses mariadb gtid, [domain ID]-[server-id]-[sequence] +func ParseMariadbGTID(str string) (*MariadbGTID, error) { if len(str) == 0 { - return MariadbGTID{0, 0, 0}, nil + return &MariadbGTID{0, 0, 0}, nil } seps := strings.Split(str, "-") - var gtid MariadbGTID + gtid := new(MariadbGTID) if len(seps) != 3 { return gtid, errors.Errorf("invalid Mariadb GTID %v, must domain-server-sequence", str) @@ -43,13 +47,13 @@ func ParseMariadbGTIDSet(str string) (GTIDSet, error) { return gtid, errors.Errorf("invalid MariaDB GTID Sequence number (%v): %v", seps[2], err) } - return MariadbGTID{ + return &MariadbGTID{ DomainID: uint32(domainID), ServerID: uint32(serverID), SequenceNumber: sequenceID}, nil } -func (gtid MariadbGTID) String() string { +func (gtid *MariadbGTID) String() string { if gtid.DomainID == 0 && gtid.ServerID == 0 && gtid.SequenceNumber == 0 { return "" } @@ -57,24 +61,172 @@ func (gtid MariadbGTID) String() string { return fmt.Sprintf("%d-%d-%d", gtid.DomainID, gtid.ServerID, gtid.SequenceNumber) } -func (gtid MariadbGTID) Encode() []byte { - return []byte(gtid.String()) -} - -func (gtid MariadbGTID) Equal(o GTIDSet) bool { - other, ok := o.(MariadbGTID) - if !ok { - return false - } - - return gtid == other -} - -func (gtid MariadbGTID) Contain(o GTIDSet) bool { - other, ok := o.(MariadbGTID) - if !ok { - return false - } - +// Contain return whether one mariadb gtid covers another mariadb gtid +func (gtid *MariadbGTID) Contain(other *MariadbGTID) bool { return gtid.DomainID == other.DomainID && gtid.SequenceNumber >= other.SequenceNumber } + +// Clone clones a mariadb gtid +func (gtid *MariadbGTID) Clone() *MariadbGTID { + o := new(MariadbGTID) + *o = *gtid + return o +} + +func (gtid *MariadbGTID) forward(newer *MariadbGTID) error { + if newer.DomainID != gtid.DomainID { + return errors.Errorf("%s is not same with doamin of %s", newer, gtid) + } + + /* + Here's a simplified example of binlog events. + Although I think one domain should have only one update at same time, we can't limit the user's usage. + we just output a warn log and let it go on + | mysqld-bin.000001 | 1453 | Gtid | 112 | 1495 | BEGIN GTID 0-112-6 | + | mysqld-bin.000001 | 1624 | Xid | 112 | 1655 | COMMIT xid=74 | + | mysqld-bin.000001 | 1655 | Gtid | 112 | 1697 | BEGIN GTID 0-112-7 | + | mysqld-bin.000001 | 1826 | Xid | 112 | 1857 | COMMIT xid=75 | + | mysqld-bin.000001 | 1857 | Gtid | 111 | 1899 | BEGIN GTID 0-111-5 | + | mysqld-bin.000001 | 1981 | Xid | 111 | 2012 | COMMIT xid=77 | + | mysqld-bin.000001 | 2012 | Gtid | 112 | 2054 | BEGIN GTID 0-112-8 | + | mysqld-bin.000001 | 2184 | Xid | 112 | 2215 | COMMIT xid=116 | + | mysqld-bin.000001 | 2215 | Gtid | 111 | 2257 | BEGIN GTID 0-111-6 | + */ + if newer.SequenceNumber <= gtid.SequenceNumber { + log.Warnf("out of order binlog appears with gtid %s vs current position gtid %s", newer, gtid) + } + + gtid.ServerID = newer.ServerID + gtid.SequenceNumber = newer.SequenceNumber + return nil +} + +// MariadbGTIDSet is a set of mariadb gtid +type MariadbGTIDSet struct { + Sets map[uint32]*MariadbGTID +} + +// ParseMariadbGTIDSet parses str into mariadb gtid sets +func ParseMariadbGTIDSet(str string) (GTIDSet, error) { + s := new(MariadbGTIDSet) + s.Sets = make(map[uint32]*MariadbGTID) + if str == "" { + return s, nil + } + + sp := strings.Split(str, ",") + + //todo, handle redundant same uuid + for i := 0; i < len(sp); i++ { + err := s.Update(sp[i]) + if err != nil { + return nil, errors.Trace(err) + } + } + return s, nil +} + +// AddSet adds mariadb gtid into mariadb gtid set +func (s *MariadbGTIDSet) AddSet(gtid *MariadbGTID) error { + if gtid == nil { + return nil + } + + o, ok := s.Sets[gtid.DomainID] + if ok { + err := o.forward(gtid) + if err != nil { + return errors.Trace(err) + } + } else { + s.Sets[gtid.DomainID] = gtid + } + + return nil +} + +// Update updates mariadb gtid set +func (s *MariadbGTIDSet) Update(GTIDStr string) error { + gtid, err := ParseMariadbGTID(GTIDStr) + if err != nil { + return err + } + + err = s.AddSet(gtid) + return errors.Trace(err) +} + +func (s *MariadbGTIDSet) String() string { + return hack.String(s.Encode()) +} + +// Encode encodes mariadb gtid set +func (s *MariadbGTIDSet) Encode() []byte { + var buf bytes.Buffer + sep := "" + for _, gtid := range s.Sets { + buf.WriteString(sep) + buf.WriteString(gtid.String()) + sep = "," + } + + return buf.Bytes() +} + +// Clone clones a mariadb gtid set +func (s *MariadbGTIDSet) Clone() GTIDSet { + clone := &MariadbGTIDSet{ + Sets: make(map[uint32]*MariadbGTID), + } + for domainID, gtid := range s.Sets { + clone.Sets[domainID] = gtid.Clone() + } + + return clone +} + +// Equal returns true if two mariadb gtid set is same, otherwise return false +func (s *MariadbGTIDSet) Equal(o GTIDSet) bool { + other, ok := o.(*MariadbGTIDSet) + if !ok { + return false + } + + if len(other.Sets) != len(s.Sets) { + return false + } + + for domainID, gtid := range other.Sets { + o, ok := s.Sets[domainID] + if !ok { + return false + } + + if *gtid != *o { + return false + } + } + + return true +} + +// Contain return whether one mariadb gtid set covers another mariadb gtid set +func (s *MariadbGTIDSet) Contain(o GTIDSet) bool { + other, ok := o.(*MariadbGTIDSet) + if !ok { + return false + } + + for doaminID, gtid := range other.Sets { + o, ok := s.Sets[doaminID] + if !ok { + return false + } + + if !o.Contain(gtid) { + return false + } + } + + return true +} diff --git a/vendor/github.com/siddontang/go-mysql/mysql/mysql_gtid.go b/vendor/github.com/siddontang/go-mysql/mysql/mysql_gtid.go index c54ded0..a937cb8 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/mysql_gtid.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/mysql_gtid.go @@ -97,7 +97,11 @@ func (s IntervalSlice) Normalize() IntervalSlice { n = append(n, s[i]) continue } else { - n[len(n)-1] = Interval{last.Start, s[i].Stop} + stop := s[i].Stop + if last.Stop > stop { + stop = last.Stop + } + n[len(n)-1] = Interval{last.Start, stop} } } @@ -285,17 +289,28 @@ func (s *UUIDSet) Decode(data []byte) error { return err } +func (s *UUIDSet) Clone() *UUIDSet { + clone := new(UUIDSet) + + clone.SID, _ = uuid.FromString(s.SID.String()) + clone.Intervals = s.Intervals.Normalize() + + return clone +} + type MysqlGTIDSet struct { Sets map[string]*UUIDSet } func ParseMysqlGTIDSet(str string) (GTIDSet, error) { s := new(MysqlGTIDSet) + s.Sets = make(map[string]*UUIDSet) + if str == "" { + return s, nil + } sp := strings.Split(str, ",") - s.Sets = make(map[string]*UUIDSet, len(sp)) - //todo, handle redundant same uuid for i := 0; i < len(sp); i++ { if set, err := ParseUUIDSet(sp[i]); err != nil { @@ -334,6 +349,9 @@ func DecodeMysqlGTIDSet(data []byte) (*MysqlGTIDSet, error) { } func (s *MysqlGTIDSet) AddSet(set *UUIDSet) { + if set == nil { + return + } sid := set.SID.String() o, ok := s.Sets[sid] if ok { @@ -343,6 +361,17 @@ func (s *MysqlGTIDSet) AddSet(set *UUIDSet) { } } +func (s *MysqlGTIDSet) Update(GTIDStr string) error { + uuidSet, err := ParseUUIDSet(GTIDStr) + if err != nil { + return err + } + + s.AddSet(uuidSet) + + return nil +} + func (s *MysqlGTIDSet) Contain(o GTIDSet) bool { sub, ok := o.(*MysqlGTIDSet) if !ok { @@ -407,3 +436,14 @@ func (s *MysqlGTIDSet) Encode() []byte { return buf.Bytes() } + +func (gtid *MysqlGTIDSet) Clone() GTIDSet { + clone := &MysqlGTIDSet{ + Sets: make(map[string]*UUIDSet), + } + for sid, uuidSet := range gtid.Sets { + clone.Sets[sid] = uuidSet.Clone() + } + + return clone +} diff --git a/vendor/github.com/siddontang/go-mysql/mysql/mysql_test.go b/vendor/github.com/siddontang/go-mysql/mysql/mysql_test.go index 8fbaaab..df4b206 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/mysql_test.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/mysql_test.go @@ -1,6 +1,7 @@ package mysql import ( + "strings" "testing" "github.com/pingcap/check" @@ -15,11 +16,11 @@ type mysqlTestSuite struct { var _ = check.Suite(&mysqlTestSuite{}) -func (s *mysqlTestSuite) SetUpSuite(c *check.C) { +func (t *mysqlTestSuite) SetUpSuite(c *check.C) { } -func (s *mysqlTestSuite) TearDownSuite(c *check.C) { +func (t *mysqlTestSuite) TearDownSuite(c *check.C) { } @@ -59,6 +60,12 @@ func (t *mysqlTestSuite) TestMysqlGTIDIntervalSlice(c *check.C) { n = i.Normalize() c.Assert(n, check.DeepEquals, IntervalSlice{Interval{1, 3}, Interval{4, 5}}) + i = IntervalSlice{Interval{1, 4}, Interval{2, 3}} + i.Sort() + c.Assert(i, check.DeepEquals, IntervalSlice{Interval{1, 4}, Interval{2, 3}}) + n = i.Normalize() + c.Assert(n, check.DeepEquals, IntervalSlice{Interval{1, 4}}) + n1 := IntervalSlice{Interval{1, 3}, Interval{4, 5}} n2 := IntervalSlice{Interval{1, 2}} @@ -91,6 +98,15 @@ func (t *mysqlTestSuite) TestMysqlGTIDCodec(c *check.C) { c.Assert(gs, check.DeepEquals, o) } +func (t *mysqlTestSuite) TestMysqlUpdate(c *check.C) { + g1, err := ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:21-57") + c.Assert(err, check.IsNil) + + g1.Update("3E11FA47-71CA-11E1-9E33-C80AA9429562:21-58") + + c.Assert(strings.ToUpper(g1.String()), check.Equals, "3E11FA47-71CA-11E1-9E33-C80AA9429562:21-58") +} + func (t *mysqlTestSuite) TestMysqlGTIDContain(c *check.C) { g1, err := ParseMysqlGTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:23") c.Assert(err, check.IsNil) @@ -151,3 +167,26 @@ func (t *mysqlTestSuite) TestMysqlParseBinaryUint64(c *check.C) { u64 := ParseBinaryUint64([]byte{1, 2, 3, 4, 5, 6, 7, 128}) c.Assert(u64, check.Equals, 128*uint64(72057594037927936)+7*uint64(281474976710656)+6*uint64(1099511627776)+5*uint64(4294967296)+4*16777216+3*65536+2*256+1) } + +func (t *mysqlTestSuite) TestErrorCode(c *check.C) { + tbls := []struct { + msg string + code int + }{ + {"ERROR 1094 (HY000): Unknown thread id: 1094", 1094}, + {"error string", 0}, + {"abcdefg", 0}, + {"123455 ks094", 0}, + {"ERROR 1046 (3D000): Unknown error 1046", 1046}, + } + for _, v := range tbls { + c.Assert(ErrorCode(v.msg), check.Equals, v.code) + } +} + +func (t *mysqlTestSuite) TestMysqlNullDecode(c *check.C) { + _, isNull, n := LengthEncodedInt([]byte{0xfb}) + + c.Assert(isNull, check.IsTrue) + c.Assert(n, check.Equals, 1) +} diff --git a/vendor/github.com/siddontang/go-mysql/mysql/resultset.go b/vendor/github.com/siddontang/go-mysql/mysql/resultset.go index a50d9f8..b01e1a5 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/resultset.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/resultset.go @@ -28,7 +28,7 @@ func (p RowData) ParseText(f []*Field) ([]interface{}, error) { var n int = 0 for i := range f { - v, isNull, n, err = LengthEnodedString(p[pos:]) + v, isNull, n, err = LengthEncodedString(p[pos:]) if err != nil { return nil, errors.Trace(err) } @@ -115,7 +115,8 @@ func (p RowData) ParseBinary(f []*Field) ([]interface{}, error) { } else { data[i] = ParseBinaryInt24(p[pos : pos+3]) } - pos += 4 + //3 byte + pos += 3 continue case MYSQL_TYPE_LONG: @@ -150,7 +151,7 @@ func (p RowData) ParseBinary(f []*Field) ([]interface{}, error) { MYSQL_TYPE_BIT, MYSQL_TYPE_ENUM, MYSQL_TYPE_SET, MYSQL_TYPE_TINY_BLOB, MYSQL_TYPE_MEDIUM_BLOB, MYSQL_TYPE_LONG_BLOB, MYSQL_TYPE_BLOB, MYSQL_TYPE_VAR_STRING, MYSQL_TYPE_STRING, MYSQL_TYPE_GEOMETRY: - v, isNull, n, err = LengthEnodedString(p[pos:]) + v, isNull, n, err = LengthEncodedString(p[pos:]) pos += n if err != nil { return nil, errors.Trace(err) diff --git a/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go b/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go index 488d253..307684d 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/resultset_helper.go @@ -38,6 +38,8 @@ func formatTextValue(value interface{}) ([]byte, error) { return v, nil case string: return hack.Slice(v), nil + case nil: + return nil, nil default: return nil, errors.Errorf("invalid type %T", value) } @@ -77,23 +79,40 @@ func formatBinaryValue(value interface{}) ([]byte, error) { return nil, errors.Errorf("invalid type %T", value) } } + +func fieldType(value interface{}) (typ uint8, err error) { + switch value.(type) { + case int8, int16, int32, int64, int: + typ = MYSQL_TYPE_LONGLONG + case uint8, uint16, uint32, uint64, uint: + typ = MYSQL_TYPE_LONGLONG + case float32, float64: + typ = MYSQL_TYPE_DOUBLE + case string, []byte: + typ = MYSQL_TYPE_VAR_STRING + case nil: + typ = MYSQL_TYPE_NULL + default: + err = errors.Errorf("unsupport type %T for resultset", value) + } + return +} + func formatField(field *Field, value interface{}) error { switch value.(type) { case int8, int16, int32, int64, int: field.Charset = 63 - field.Type = MYSQL_TYPE_LONGLONG field.Flag = BINARY_FLAG | NOT_NULL_FLAG case uint8, uint16, uint32, uint64, uint: field.Charset = 63 - field.Type = MYSQL_TYPE_LONGLONG field.Flag = BINARY_FLAG | NOT_NULL_FLAG | UNSIGNED_FLAG case float32, float64: field.Charset = 63 - field.Type = MYSQL_TYPE_DOUBLE field.Flag = BINARY_FLAG | NOT_NULL_FLAG case string, []byte: field.Charset = 33 - field.Type = MYSQL_TYPE_VAR_STRING + case nil: + field.Charset = 33 default: return errors.Errorf("unsupport type %T for resultset", value) } @@ -106,7 +125,13 @@ func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultse r.Fields = make([]*Field, len(names)) var b []byte - var err error + + if len(values) == 0 { + for i, name := range names { + r.Fields[i] = &Field{Name: hack.Slice(name), Charset: 33, Type: MYSQL_TYPE_NULL} + } + return r, nil + } for i, vs := range values { if len(vs) != len(r.Fields) { @@ -115,13 +140,23 @@ func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultse var row []byte for j, value := range vs { - if i == 0 { - field := &Field{} - r.Fields[j] = field - field.Name = hack.Slice(names[j]) - - if err = formatField(field, value); err != nil { - return nil, errors.Trace(err) + typ, err := fieldType(value) + if err != nil { + return nil, errors.Trace(err) + } + if r.Fields[j] == nil { + r.Fields[j] = &Field{Name: hack.Slice(names[j]), Type: typ} + formatField(r.Fields[j], value) + } else if typ != r.Fields[j].Type { + // we got another type in the same column. in general, we treat it as an error, except + // the case, when old value was null, and the new one isn't null, so we can update + // type info for fields. + oldIsNull, newIsNull := r.Fields[j].Type == MYSQL_TYPE_NULL, typ == MYSQL_TYPE_NULL + if oldIsNull && !newIsNull { // old is null, new isn't, update type info. + r.Fields[j].Type = typ + formatField(r.Fields[j], value) + } else if !oldIsNull && !newIsNull { // different non-null types, that's an error. + return nil, errors.Errorf("row types aren't consistent") } } b, err = formatTextValue(value) @@ -130,7 +165,12 @@ func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultse return nil, errors.Trace(err) } - row = append(row, PutLengthEncodedString(b)...) + if b == nil { + // NULL value is encoded as 0xfb here (without additional info about length) + row = append(row, 0xfb) + } else { + row = append(row, PutLengthEncodedString(b)...) + } } r.RowDatas = append(r.RowDatas, row) @@ -145,7 +185,6 @@ func BuildSimpleBinaryResultset(names []string, values [][]interface{}) (*Result r.Fields = make([]*Field, len(names)) var b []byte - var err error bitmapLen := ((len(names) + 7 + 2) >> 3) @@ -161,8 +200,12 @@ func BuildSimpleBinaryResultset(names []string, values [][]interface{}) (*Result row = append(row, nullBitmap...) for j, value := range vs { + typ, err := fieldType(value) + if err != nil { + return nil, errors.Trace(err) + } if i == 0 { - field := &Field{} + field := &Field{Type: typ} r.Fields[j] = field field.Name = hack.Slice(names[j]) diff --git a/vendor/github.com/siddontang/go-mysql/mysql/util.go b/vendor/github.com/siddontang/go-mysql/mysql/util.go index 7fe41fa..757910e 100644 --- a/vendor/github.com/siddontang/go-mysql/mysql/util.go +++ b/vendor/github.com/siddontang/go-mysql/mysql/util.go @@ -11,6 +11,8 @@ import ( "github.com/juju/errors" "github.com/siddontang/go/hack" + "crypto/sha256" + "crypto/rsa" ) func Pstack() string { @@ -48,6 +50,62 @@ func CalcPassword(scramble, password []byte) []byte { return scramble } +// Hash password using MySQL 8+ method (SHA256) +func CalcCachingSha2Password(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + // XOR(SHA256(password), SHA256(SHA256(SHA256(password)), scramble)) + + crypt := sha256.New() + crypt.Write([]byte(password)) + message1 := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1) + message1Hash := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1Hash) + crypt.Write(scramble) + message2 := crypt.Sum(nil) + + for i := range message1 { + message1[i] ^= message2[i] + } + + return message1 +} + + +func EncryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, error) { + plain := make([]byte, len(password)+1) + copy(plain, password) + for i := range plain { + j := i % len(seed) + plain[i] ^= seed[j] + } + sha1v := sha1.New() + return rsa.EncryptOAEP(sha1v, rand.Reader, pub, plain, nil) +} + +// encodes a uint64 value and appends it to the given bytes slice +func AppendLengthEncodedInteger(b []byte, n uint64) []byte { + switch { + case n <= 250: + return append(b, byte(n)) + + case n <= 0xffff: + return append(b, 0xfc, byte(n), byte(n>>8)) + + case n <= 0xffffff: + return append(b, 0xfd, byte(n), byte(n>>8), byte(n>>16)) + } + return append(b, 0xfe, byte(n), byte(n>>8), byte(n>>16), byte(n>>24), + byte(n>>32), byte(n>>40), byte(n>>48), byte(n>>56)) +} + func RandomBuf(size int) ([]byte, error) { buf := make([]byte, size) @@ -84,39 +142,33 @@ func BFixedLengthInt(buf []byte) uint64 { } func LengthEncodedInt(b []byte) (num uint64, isNull bool, n int) { - switch b[0] { + if len(b) == 0 { + return 0, true, 1 + } + switch b[0] { // 251: NULL case 0xfb: - n = 1 - isNull = true - return + return 0, true, 1 - // 252: value of following 2 + // 252: value of following 2 case 0xfc: - num = uint64(b[1]) | uint64(b[2])<<8 - n = 3 - return + return uint64(b[1]) | uint64(b[2])<<8, false, 3 - // 253: value of following 3 + // 253: value of following 3 case 0xfd: - num = uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16 - n = 4 - return + return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16, false, 4 - // 254: value of following 8 + // 254: value of following 8 case 0xfe: - num = uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16 | + return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16 | uint64(b[4])<<24 | uint64(b[5])<<32 | uint64(b[6])<<40 | - uint64(b[7])<<48 | uint64(b[8])<<56 - n = 9 - return + uint64(b[7])<<48 | uint64(b[8])<<56, + false, 9 } // 0-250: value of first byte - num = uint64(b[0]) - n = 1 - return + return uint64(b[0]), false, 1 } func PutLengthEncodedInt(n uint64) []byte { @@ -137,23 +189,26 @@ func PutLengthEncodedInt(n uint64) []byte { return nil } -func LengthEnodedString(b []byte) ([]byte, bool, int, error) { +// returns the string read as a bytes slice, whether the value is NULL, +// the number of bytes read and an error, in case the string is longer than +// the input slice +func LengthEncodedString(b []byte) ([]byte, bool, int, error) { // Get length num, isNull, n := LengthEncodedInt(b) if num < 1 { - return nil, isNull, n, nil + return b[n:n], isNull, n, nil } n += int(num) // Check data length if len(b) >= n { - return b[n-int(num) : n], false, n, nil + return b[n-int(num) : n : n], false, n, nil } return nil, false, n, io.EOF } -func SkipLengthEnodedString(b []byte) (int, error) { +func SkipLengthEncodedString(b []byte) (int, error) { // Get length num, _, n := LengthEncodedInt(b) if num < 1 { diff --git a/vendor/github.com/siddontang/go-mysql/packet/conn.go b/vendor/github.com/siddontang/go-mysql/packet/conn.go index 3772e1a..41b1bf1 100644 --- a/vendor/github.com/siddontang/go-mysql/packet/conn.go +++ b/vendor/github.com/siddontang/go-mysql/packet/conn.go @@ -1,11 +1,17 @@ package packet +import "C" import ( - "bufio" "bytes" "io" "net" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "encoding/pem" + "github.com/juju/errors" . "github.com/siddontang/go-mysql/mysql" ) @@ -15,7 +21,9 @@ import ( */ type Conn struct { net.Conn - br *bufio.Reader + + // we removed the buffer reader because it will cause the SSLRequest to block (tls connection handshake won't be + // able to read the "Client Hello" data since it has been buffered into the buffer reader) Sequence uint8 } @@ -23,7 +31,6 @@ type Conn struct { func NewConn(conn net.Conn) *Conn { c := new(Conn) - c.br = bufio.NewReaderSize(conn, 4096) c.Conn = conn return c @@ -37,55 +44,20 @@ func (c *Conn) ReadPacket() ([]byte, error) { } else { return buf.Bytes(), nil } - - // header := []byte{0, 0, 0, 0} - - // if _, err := io.ReadFull(c.br, header); err != nil { - // return nil, ErrBadConn - // } - - // length := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) - // if length < 1 { - // return nil, fmt.Errorf("invalid payload length %d", length) - // } - - // sequence := uint8(header[3]) - - // if sequence != c.Sequence { - // return nil, fmt.Errorf("invalid sequence %d != %d", sequence, c.Sequence) - // } - - // c.Sequence++ - - // data := make([]byte, length) - // if _, err := io.ReadFull(c.br, data); err != nil { - // return nil, ErrBadConn - // } else { - // if length < MaxPayloadLen { - // return data, nil - // } - - // var buf []byte - // buf, err = c.ReadPacket() - // if err != nil { - // return nil, ErrBadConn - // } else { - // return append(data, buf...), nil - // } - // } } func (c *Conn) ReadPacketTo(w io.Writer) error { header := []byte{0, 0, 0, 0} - if _, err := io.ReadFull(c.br, header); err != nil { + if _, err := io.ReadFull(c.Conn, header); err != nil { return ErrBadConn } length := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) - if length < 1 { - return errors.Errorf("invalid payload length %d", length) - } + // bug fixed: caching_sha2_password will send 0-length payload (the unscrambled password) when the password is empty + //if length < 1 { + // return errors.Errorf("invalid payload length %d", length) + //} sequence := uint8(header[3]) @@ -95,7 +67,7 @@ func (c *Conn) ReadPacketTo(w io.Writer) error { c.Sequence++ - if n, err := io.CopyN(w, c.br, int64(length)); err != nil { + if n, err := io.CopyN(w, c.Conn, int64(length)); err != nil { return ErrBadConn } else if n != int64(length) { return ErrBadConn @@ -150,6 +122,77 @@ func (c *Conn) WritePacket(data []byte) error { } } +// Client clear text authentication packet +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (c *Conn) WriteClearAuthPacket(password string) error { + // Calculate the packet length and add a tailing 0 + pktLen := len(password) + 1 + data := make([]byte, 4 + pktLen) + + // Add the clear password [null terminated string] + copy(data[4:], password) + data[4+pktLen-1] = 0x00 + + return c.WritePacket(data) +} + +// Caching sha2 authentication. Public key request and send encrypted password +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (c *Conn) WritePublicKeyAuthPacket(password string, cipher []byte) error { + // request public key + data := make([]byte, 4 + 1) + data[4] = 2 // cachingSha2PasswordRequestPublicKey + c.WritePacket(data) + + data, err := c.ReadPacket() + if err != nil { + return err + } + + block, _ := pem.Decode(data[1:]) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + + plain := make([]byte, len(password)+1) + copy(plain, password) + for i := range plain { + j := i % len(cipher) + plain[i] ^= cipher[j] + } + sha1v := sha1.New() + enc, _ := rsa.EncryptOAEP(sha1v, rand.Reader, pub.(*rsa.PublicKey), plain, nil) + data = make([]byte, 4 + len(enc)) + copy(data[4:], enc) + return c.WritePacket(data) +} + +func (c *Conn) WriteEncryptedPassword(password string, seed []byte, pub *rsa.PublicKey) error { + enc, err := EncryptPassword(password, seed, pub) + if err != nil { + return err + } + return c.WriteAuthSwitchPacket(enc, false) +} + +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (c *Conn) WriteAuthSwitchPacket(authData []byte, addNUL bool) error { + pktLen := 4 + len(authData) + if addNUL { + pktLen++ + } + data := make([]byte, pktLen) + + // Add the auth data [EOF] + copy(data[4:], authData) + if addNUL { + data[pktLen-1] = 0x00 + } + + return c.WritePacket(data) +} + func (c *Conn) ResetSequence() { c.Sequence = 0 } diff --git a/vendor/github.com/siddontang/go-mysql/replication/backup.go b/vendor/github.com/siddontang/go-mysql/replication/backup.go index 744c38c..24a25ae 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/backup.go +++ b/vendor/github.com/siddontang/go-mysql/replication/backup.go @@ -1,13 +1,12 @@ package replication import ( + "context" "io" "os" "path" "time" - "golang.org/x/net/context" - "github.com/juju/errors" . "github.com/siddontang/go-mysql/mysql" ) @@ -41,7 +40,7 @@ func (b *BinlogSyncer) StartBackup(backupDir string, p Position, timeout time.Du }() for { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), timeout) e, err := s.GetEvent(ctx) cancel() diff --git a/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go b/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go index e5b165c..c1e4057 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go +++ b/vendor/github.com/siddontang/go-mysql/replication/binlogstreamer.go @@ -1,10 +1,10 @@ package replication import ( - "golang.org/x/net/context" - + "context" + "time" "github.com/juju/errors" - "github.com/ngaut/log" + "github.com/siddontang/go-log/log" ) var ( @@ -36,6 +36,36 @@ func (s *BinlogStreamer) GetEvent(ctx context.Context) (*BinlogEvent, error) { } } +// Get the binlog event with starttime, if current binlog event timestamp smaller than specify starttime +// return nil event +func (s *BinlogStreamer) GetEventWithStartTime(ctx context.Context,startTime time.Time) (*BinlogEvent, error) { + if s.err != nil { + return nil, ErrNeedSyncAgain + } + startUnix := startTime.Unix() + select { + case c := <-s.ch: + if int64(c.Header.Timestamp) >= startUnix { + return c, nil + } + return nil,nil + case s.err = <-s.ech: + return nil, s.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// DumpEvents dumps all left events +func (s *BinlogStreamer) DumpEvents() []*BinlogEvent { + count := len(s.ch) + events := make([]*BinlogEvent, 0, count) + for i := 0; i < count; i++ { + events = append(events, <-s.ch) + } + return events +} + func (s *BinlogStreamer) close() { s.closeWithError(ErrSyncClosed) } diff --git a/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go b/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go index 2a72c6d..552798b 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go +++ b/vendor/github.com/siddontang/go-mysql/replication/binlogsyncer.go @@ -1,17 +1,19 @@ package replication import ( + "context" "crypto/tls" "encoding/binary" "fmt" + "net" "os" + "strings" "sync" "time" - "golang.org/x/net/context" - "github.com/juju/errors" - "github.com/ngaut/log" + "github.com/satori/go.uuid" + "github.com/siddontang/go-log/log" "github.com/siddontang/go-mysql/client" . "github.com/siddontang/go-mysql/mysql" ) @@ -40,6 +42,9 @@ type BinlogSyncerConfig struct { // If not set, use os.Hostname() instead. Localhost string + // Charset is for MySQL client character set + Charset string + // SemiSyncEnabled enables semi-sync or not. SemiSyncEnabled bool @@ -48,6 +53,46 @@ type BinlogSyncerConfig struct { // If not nil, use the provided tls.Config to connect to the database using TLS/SSL. TLSConfig *tls.Config + + // Use replication.Time structure for timestamp and datetime. + // We will use Local location for timestamp and UTC location for datatime. + ParseTime bool + + // If ParseTime is false, convert TIMESTAMP into this specified timezone. If + // ParseTime is true, this option will have no effect and TIMESTAMP data will + // be parsed into the local timezone and a full time.Time struct will be + // returned. + // + // Note that MySQL TIMESTAMP columns are offset from the machine local + // timezone while DATETIME columns are offset from UTC. This is consistent + // with documented MySQL behaviour as it return TIMESTAMP in local timezone + // and DATETIME in UTC. + // + // Setting this to UTC effectively equalizes the TIMESTAMP and DATETIME time + // strings obtained from MySQL. + TimestampStringLocation *time.Location + + // Use decimal.Decimal structure for decimals. + UseDecimal bool + + // RecvBufferSize sets the size in bytes of the operating system's receive buffer associated with the connection. + RecvBufferSize int + + // master heartbeat period + HeartbeatPeriod time.Duration + + // read timeout + ReadTimeout time.Duration + + // maximum number of attempts to re-establish a broken connection + MaxReconnectAttempts int + + // Only works when MySQL/MariaDB variable binlog_checksum=CRC32. + // For MySQL, binlog_checksum was introduced since 5.6.2, but CRC32 was set as default value since 5.6.6 . + // https://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html#option_mysqld_binlog-checksum + // For MariaDB, binlog_checksum was introduced since MariaDB 5.3, but CRC32 was set as default value since MariaDB 10.2.1 . + // https://mariadb.com/kb/en/library/replication-and-binary-log-server-system-variables/#binlog_checksum + VerifyChecksum bool } // BinlogSyncer syncs binlog event from server. @@ -64,14 +109,23 @@ type BinlogSyncer struct { nextPos Position + gset GTIDSet + running bool ctx context.Context cancel context.CancelFunc + + lastConnectionID uint32 + + retryCount int } // NewBinlogSyncer creates the BinlogSyncer with cfg. func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { + if cfg.ServerID == 0 { + log.Fatal("can't use 0 as the server ID") + } // Clear the Password to avoid outputing it in log. pass := cfg.Password @@ -84,7 +138,10 @@ func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { b.cfg = cfg b.parser = NewBinlogParser() b.parser.SetRawMode(b.cfg.RawModeEnabled) - + b.parser.SetParseTime(b.cfg.ParseTime) + b.parser.SetTimestampStringLocation(b.cfg.TimestampStringLocation) + b.parser.SetUseDecimal(b.cfg.UseDecimal) + b.parser.SetVerifyChecksum(b.cfg.VerifyChecksum) b.running = false b.ctx, b.cancel = context.WithCancel(context.Background()) @@ -136,15 +193,53 @@ func (b *BinlogSyncer) registerSlave() error { b.c.Close() } - log.Infof("register slave for master server %s:%d", b.cfg.Host, b.cfg.Port) + addr := "" + if strings.Contains(b.cfg.Host, "/") { + addr = b.cfg.Host + } else { + addr = fmt.Sprintf("%s:%d", b.cfg.Host, b.cfg.Port) + } + + log.Infof("register slave for master server %s", addr) var err error - b.c, err = client.Connect(fmt.Sprintf("%s:%d", b.cfg.Host, b.cfg.Port), b.cfg.User, b.cfg.Password, "", func(c *client.Conn) { - c.TLSConfig = b.cfg.TLSConfig + b.c, err = client.Connect(addr, b.cfg.User, b.cfg.Password, "", func(c *client.Conn) { + c.SetTLSConfig(b.cfg.TLSConfig) }) if err != nil { return errors.Trace(err) } + if len(b.cfg.Charset) != 0 { + b.c.SetCharset(b.cfg.Charset) + } + + //set read timeout + if b.cfg.ReadTimeout > 0 { + b.c.SetReadDeadline(time.Now().Add(b.cfg.ReadTimeout)) + } + + if b.cfg.RecvBufferSize > 0 { + if tcp, ok := b.c.Conn.Conn.(*net.TCPConn); ok { + tcp.SetReadBuffer(b.cfg.RecvBufferSize) + } + } + + // kill last connection id + if b.lastConnectionID > 0 { + cmd := fmt.Sprintf("KILL %d", b.lastConnectionID) + if _, err := b.c.Execute(cmd); err != nil { + log.Errorf("kill connection %d error %v", b.lastConnectionID, err) + // Unknown thread id + if code := ErrorCode(err.Error()); code != ER_NO_SUCH_THREAD { + return errors.Trace(err) + } + } + log.Infof("kill last connection id %d", b.lastConnectionID) + } + + // save last last connection id for kill + b.lastConnectionID = b.c.GetConnectionID() + //for mysql 5.6+, binlog has a crc32 checksum //before mysql 5.6, this will not work, don't matter.:-) if r, err := b.c.Execute("SHOW GLOBAL VARIABLES LIKE 'BINLOG_CHECKSUM'"); err != nil { @@ -180,6 +275,14 @@ func (b *BinlogSyncer) registerSlave() error { } } + if b.cfg.HeartbeatPeriod > 0 { + _, err = b.c.Execute(fmt.Sprintf("SET @master_heartbeat_period=%d;", b.cfg.HeartbeatPeriod)) + if err != nil { + log.Errorf("failed to set @master_heartbeat_period=%d, err: %v", b.cfg.HeartbeatPeriod, err) + return errors.Trace(err) + } + } + if err = b.writeRegisterSlaveCommand(); err != nil { return errors.Trace(err) } @@ -191,7 +294,7 @@ func (b *BinlogSyncer) registerSlave() error { return nil } -func (b *BinlogSyncer) enalbeSemiSync() error { +func (b *BinlogSyncer) enableSemiSync() error { if !b.cfg.SemiSyncEnabled { return nil } @@ -224,7 +327,7 @@ func (b *BinlogSyncer) prepare() error { return errors.Trace(err) } - if err := b.enalbeSemiSync(); err != nil { + if err := b.enableSemiSync(); err != nil { return errors.Trace(err) } @@ -241,6 +344,11 @@ func (b *BinlogSyncer) startDumpStream() *BinlogStreamer { return s } +// GetNextPosition returns the next position of the syncer +func (b *BinlogSyncer) GetNextPosition() Position { + return b.nextPos +} + // StartSync starts syncing from the `pos` position. func (b *BinlogSyncer) StartSync(pos Position) (*BinlogStreamer, error) { log.Infof("begin to sync binlog from position %s", pos) @@ -261,7 +369,9 @@ func (b *BinlogSyncer) StartSync(pos Position) (*BinlogStreamer, error) { // StartSyncGTID starts syncing from the `gset` GTIDSet. func (b *BinlogSyncer) StartSyncGTID(gset GTIDSet) (*BinlogStreamer, error) { - log.Infof("begin to sync binlog from GTID %s", gset) + log.Infof("begin to sync binlog from GTID set %s", gset) + + b.gset = gset b.m.Lock() defer b.m.Unlock() @@ -275,11 +385,12 @@ func (b *BinlogSyncer) StartSyncGTID(gset GTIDSet) (*BinlogStreamer, error) { } var err error - if b.cfg.Flavor != MariaDBFlavor { + switch b.cfg.Flavor { + case MariaDBFlavor: + err = b.writeBinlogDumpMariadbGTIDCommand(gset) + default: // default use MySQL err = b.writeBinlogDumpMysqlGTIDCommand(gset) - } else { - err = b.writeBinlogDumpMariadbGTIDCommand(gset) } if err != nil { @@ -289,7 +400,7 @@ func (b *BinlogSyncer) StartSyncGTID(gset GTIDSet) (*BinlogStreamer, error) { return b.startDumpStream(), nil } -func (b *BinlogSyncer) writeBinglogDumpCommand(p Position) error { +func (b *BinlogSyncer) writeBinlogDumpCommand(p Position) error { b.c.ResetSequence() data := make([]byte, 4+1+4+2+4+len(p.Name)) @@ -313,7 +424,7 @@ func (b *BinlogSyncer) writeBinglogDumpCommand(p Position) error { } func (b *BinlogSyncer) writeBinlogDumpMysqlGTIDCommand(gset GTIDSet) error { - p := Position{"", 4} + p := Position{Name: "", Pos: 4} gtidData := gset.Encode() b.c.ResetSequence() @@ -369,7 +480,7 @@ func (b *BinlogSyncer) writeBinlogDumpMariadbGTIDCommand(gset GTIDSet) error { } // Since we use @slave_connect_state, the file and position here are ignored. - return b.writeBinglogDumpCommand(Position{"", 0}) + return b.writeBinlogDumpCommand(Position{Name: "", Pos: 0}) } // localHostname returns the hostname that register slave would register as. @@ -444,21 +555,25 @@ func (b *BinlogSyncer) replySemiSyncACK(p Position) error { return errors.Trace(err) } - _, err = b.c.ReadOKPacket() - if err != nil { - } - return errors.Trace(err) + return nil } func (b *BinlogSyncer) retrySync() error { b.m.Lock() defer b.m.Unlock() - log.Infof("begin to re-sync from %s", b.nextPos) - b.parser.Reset() - if err := b.prepareSyncPos(b.nextPos); err != nil { - return errors.Trace(err) + + if b.gset != nil { + log.Infof("begin to re-sync from %s", b.gset.String()) + if err := b.prepareSyncGTID(b.gset); err != nil { + return errors.Trace(err) + } + } else { + log.Infof("begin to re-sync from %s", b.nextPos) + if err := b.prepareSyncPos(b.nextPos); err != nil { + return errors.Trace(err) + } } return nil @@ -474,13 +589,34 @@ func (b *BinlogSyncer) prepareSyncPos(pos Position) error { return errors.Trace(err) } - if err := b.writeBinglogDumpCommand(pos); err != nil { + if err := b.writeBinlogDumpCommand(pos); err != nil { return errors.Trace(err) } return nil } +func (b *BinlogSyncer) prepareSyncGTID(gset GTIDSet) error { + var err error + + if err = b.prepare(); err != nil { + return errors.Trace(err) + } + + switch b.cfg.Flavor { + case MariaDBFlavor: + err = b.writeBinlogDumpMariadbGTIDCommand(gset) + default: + // default use MySQL + err = b.writeBinlogDumpMysqlGTIDCommand(gset) + } + + if err != nil { + return err + } + return nil +} + func (b *BinlogSyncer) onStream(s *BinlogStreamer) { defer func() { if e := recover(); e != nil { @@ -495,21 +631,27 @@ func (b *BinlogSyncer) onStream(s *BinlogStreamer) { log.Error(err) // we meet connection error, should re-connect again with - // last nextPos we got. - if len(b.nextPos.Name) == 0 { + // last nextPos or nextGTID we got. + if len(b.nextPos.Name) == 0 && b.gset == nil { // we can't get the correct position, close. s.closeWithError(err) return } - // TODO: add a max retry count. for { select { case <-b.ctx.Done(): s.close() return case <-time.After(time.Second): + b.retryCount++ if err = b.retrySync(); err != nil { + if b.cfg.MaxReconnectAttempts > 0 && b.retryCount >= b.cfg.MaxReconnectAttempts { + log.Errorf("retry sync err: %v, exceeded max retries (%d)", err, b.cfg.MaxReconnectAttempts) + s.closeWithError(err) + return + } + log.Errorf("retry sync err: %v, wait 1s and retry again", err) continue } @@ -522,6 +664,14 @@ func (b *BinlogSyncer) onStream(s *BinlogStreamer) { continue } + //set read timeout + if b.cfg.ReadTimeout > 0 { + b.c.SetReadDeadline(time.Now().Add(b.cfg.ReadTimeout)) + } + + // Reset retry count on successful packet receieve + b.retryCount = 0 + switch data[0] { case OK_HEADER: if err = b.parseEvent(s, data); err != nil { @@ -557,7 +707,7 @@ func (b *BinlogSyncer) parseEvent(s *BinlogStreamer, data []byte) error { data = data[2:] } - e, err := b.parser.parse(data) + e, err := b.parser.Parse(data) if err != nil { return errors.Trace(err) } @@ -566,11 +716,33 @@ func (b *BinlogSyncer) parseEvent(s *BinlogStreamer, data []byte) error { // Some events like FormatDescriptionEvent return 0, ignore. b.nextPos.Pos = e.Header.LogPos } - - if re, ok := e.Event.(*RotateEvent); ok { - b.nextPos.Name = string(re.NextLogName) - b.nextPos.Pos = uint32(re.Position) + switch event := e.Event.(type) { + case *RotateEvent: + b.nextPos.Name = string(event.NextLogName) + b.nextPos.Pos = uint32(event.Position) log.Infof("rotate to %s", b.nextPos) + case *GTIDEvent: + if b.gset == nil { + break + } + u, _ := uuid.FromBytes(event.SID) + err := b.gset.Update(fmt.Sprintf("%s:%d", u.String(), event.GNO)) + if err != nil { + return errors.Trace(err) + } + case *MariadbGTIDEvent: + if b.gset == nil { + break + } + GTID := event.GTID + err := b.gset.Update(fmt.Sprintf("%d-%d-%d", GTID.DomainID, GTID.ServerID, GTID.SequenceNumber)) + if err != nil { + return errors.Trace(err) + } + case *XIDEvent: + event.GSet = b.getGtidSet() + case *QueryEvent: + event.GSet = b.getGtidSet() } needStop := false @@ -593,3 +765,15 @@ func (b *BinlogSyncer) parseEvent(s *BinlogStreamer, data []byte) error { return nil } + +func (b *BinlogSyncer) getGtidSet() GTIDSet { + if b.gset == nil { + return nil + } + return b.gset.Clone() +} + +// LastConnectionID returns last connectionID. +func (b *BinlogSyncer) LastConnectionID() uint32 { + return b.lastConnectionID +} diff --git a/vendor/github.com/siddontang/go-mysql/replication/event.go b/vendor/github.com/siddontang/go-mysql/replication/event.go index 1a0d2c6..737b431 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/event.go @@ -2,7 +2,6 @@ package replication import ( "encoding/binary" - //"encoding/hex" "fmt" "io" "strconv" @@ -16,11 +15,15 @@ import ( ) const ( - EventHeaderSize = 19 + EventHeaderSize = 19 + SidLength = 16 + LogicalTimestampTypeCode = 2 + PartLogicalTimestampLength = 8 + BinlogChecksumLength = 4 ) type BinlogEvent struct { - // raw binlog data, including crc32 checksum if exists + // raw binlog data which contains all data, including binlog header and event body, and including crc32 checksum if exists RawData []byte Header *EventHeader @@ -50,7 +53,7 @@ type EventError struct { } func (e *EventError) Error() string { - return e.Err + return fmt.Sprintf("Header %#v, Data %q, Err: %v", e.Header, e.Data, e.Err) } type EventHeader struct { @@ -216,6 +219,9 @@ func (e *RotateEvent) Dump(w io.Writer) { type XIDEvent struct { XID uint64 + + // in fact XIDEvent dosen't have the GTIDSet information, just for beneficial to use + GSet GTIDSet } func (e *XIDEvent) Decode(data []byte) error { @@ -225,6 +231,9 @@ func (e *XIDEvent) Decode(data []byte) error { func (e *XIDEvent) Dump(w io.Writer) { fmt.Fprintf(w, "XID: %d\n", e.XID) + if e.GSet != nil { + fmt.Fprintf(w, "GTIDSet: %s\n", e.GSet.String()) + } fmt.Fprintln(w) } @@ -235,6 +244,9 @@ type QueryEvent struct { StatusVars []byte Schema []byte Query []byte + + // in fact QueryEvent dosen't have the GTIDSet information, just for beneficial to use + GSet GTIDSet } func (e *QueryEvent) Decode(data []byte) error { @@ -275,21 +287,36 @@ func (e *QueryEvent) Dump(w io.Writer) { //fmt.Fprintf(w, "Status vars: \n%s", hex.Dump(e.StatusVars)) fmt.Fprintf(w, "Schema: %s\n", e.Schema) fmt.Fprintf(w, "Query: %s\n", e.Query) + if e.GSet != nil { + fmt.Fprintf(w, "GTIDSet: %s\n", e.GSet.String()) + } fmt.Fprintln(w) } type GTIDEvent struct { - CommitFlag uint8 - SID []byte - GNO int64 + CommitFlag uint8 + SID []byte + GNO int64 + LastCommitted int64 + SequenceNumber int64 } func (e *GTIDEvent) Decode(data []byte) error { - e.CommitFlag = uint8(data[0]) - - e.SID = data[1:17] - - e.GNO = int64(binary.LittleEndian.Uint64(data[17:])) + pos := 0 + e.CommitFlag = uint8(data[pos]) + pos++ + e.SID = data[pos : pos+SidLength] + pos += SidLength + e.GNO = int64(binary.LittleEndian.Uint64(data[pos:])) + pos += 8 + if len(data) >= 42 { + if uint8(data[pos]) == LogicalTimestampTypeCode { + pos++ + e.LastCommitted = int64(binary.LittleEndian.Uint64(data[pos:])) + pos += PartLogicalTimestampLength + e.SequenceNumber = int64(binary.LittleEndian.Uint64(data[pos:])) + } + } return nil } @@ -297,6 +324,8 @@ func (e *GTIDEvent) Dump(w io.Writer) { fmt.Fprintf(w, "Commit flag: %d\n", e.CommitFlag) u, _ := uuid.FromBytes(e.SID) fmt.Fprintf(w, "GTID_NEXT: %s:%d\n", u.String(), e.GNO) + fmt.Fprintf(w, "LAST_COMMITTED: %d\n", e.LastCommitted) + fmt.Fprintf(w, "SEQUENCE_NUMBER: %d\n", e.SequenceNumber) fmt.Fprintln(w) } @@ -382,16 +411,16 @@ func (e *ExecuteLoadQueryEvent) Dump(w io.Writer) { // case MARIADB_ANNOTATE_ROWS_EVENT: // return "MariadbAnnotateRowsEvent" -type MariadbAnnotaeRowsEvent struct { +type MariadbAnnotateRowsEvent struct { Query []byte } -func (e *MariadbAnnotaeRowsEvent) Decode(data []byte) error { +func (e *MariadbAnnotateRowsEvent) Decode(data []byte) error { e.Query = data return nil } -func (e *MariadbAnnotaeRowsEvent) Dump(w io.Writer) { +func (e *MariadbAnnotateRowsEvent) Dump(w io.Writer) { fmt.Fprintf(w, "Query: %s\n", e.Query) fmt.Fprintln(w) } @@ -424,7 +453,7 @@ func (e *MariadbGTIDEvent) Decode(data []byte) error { } func (e *MariadbGTIDEvent) Dump(w io.Writer) { - fmt.Fprintf(w, "GTID: %s\n", e.GTID) + fmt.Fprintf(w, "GTID: %v\n", e.GTID) fmt.Fprintln(w) } diff --git a/vendor/github.com/siddontang/go-mysql/replication/json_binary.go b/vendor/github.com/siddontang/go-mysql/replication/json_binary.go index 24a9b27..6529f01 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/json_binary.go +++ b/vendor/github.com/siddontang/go-mysql/replication/json_binary.go @@ -70,8 +70,13 @@ func jsonbGetValueEntrySize(isSmall bool) int { // decodeJsonBinary decodes the JSON binary encoding data and returns // the common JSON encoding data. -func decodeJsonBinary(data []byte) ([]byte, error) { - d := new(jsonBinaryDecoder) +func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { + // Sometimes, we can insert a NULL JSON even we set the JSON field as NOT NULL. + // If we meet this case, we can return an empty slice. + if len(data) == 0 { + return []byte{}, nil + } + d := jsonBinaryDecoder{useDecimal: e.useDecimal} if d.isDataShort(data, 1) { return nil, d.err @@ -86,7 +91,8 @@ func decodeJsonBinary(data []byte) ([]byte, error) { } type jsonBinaryDecoder struct { - err error + useDecimal bool + err error } func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { @@ -382,7 +388,7 @@ func (d *jsonBinaryDecoder) decodeDecimal(data []byte) interface{} { precision := int(data[0]) scale := int(data[1]) - v, _, err := decodeDecimal(data[2:], precision, scale) + v, _, err := decodeDecimal(data[2:], precision, scale, d.useDecimal) d.err = err return v @@ -463,7 +469,7 @@ func (d *jsonBinaryDecoder) decodeVariableLength(data []byte) (int, int) { if v&0x80 == 0 { if length > math.MaxUint32 { - d.err = errors.Errorf("variable length %d must <= %d", length, math.MaxUint32) + d.err = errors.Errorf("variable length %d must <= %d", length, int64(math.MaxUint32)) return 0, 0 } diff --git a/vendor/github.com/siddontang/go-mysql/replication/parser.go b/vendor/github.com/siddontang/go-mysql/replication/parser.go index cfc97e4..6fe1cc0 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/parser.go +++ b/vendor/github.com/siddontang/go-mysql/replication/parser.go @@ -2,13 +2,22 @@ package replication import ( "bytes" + "encoding/binary" "fmt" + "hash/crc32" "io" "os" + "sync/atomic" + "time" "github.com/juju/errors" ) +var ( + // ErrChecksumMismatch indicates binlog checksum mismatch. + ErrChecksumMismatch = errors.New("binlog checksum mismatch, data may be corrupted") +) + type BinlogParser struct { format *FormatDescriptionEvent @@ -16,6 +25,15 @@ type BinlogParser struct { // for rawMode, we only parse FormatDescriptionEvent and RotateEvent rawMode bool + + parseTime bool + timestampStringLocation *time.Location + + // used to start/stop processing + stopProcessing uint32 + + useDecimal bool + verifyChecksum bool } func NewBinlogParser() *BinlogParser { @@ -26,6 +44,14 @@ func NewBinlogParser() *BinlogParser { return p } +func (p *BinlogParser) Stop() { + atomic.StoreUint32(&p.stopProcessing, 1) +} + +func (p *BinlogParser) Resume() { + atomic.StoreUint32(&p.stopProcessing, 0) +} + func (p *BinlogParser) Reset() { p.format = nil } @@ -48,64 +74,102 @@ func (p *BinlogParser) ParseFile(name string, offset int64, onEvent OnEventFunc) if offset < 4 { offset = 4 + } else if offset > 4 { + // FORMAT_DESCRIPTION event should be read by default always (despite that fact passed offset may be higher than 4) + if _, err = f.Seek(4, os.SEEK_SET); err != nil { + return errors.Errorf("seek %s to %d error %v", name, offset, err) + } + + if err = p.parseFormatDescriptionEvent(f, onEvent); err != nil { + return errors.Annotatef(err, "parse FormatDescriptionEvent") + } } if _, err = f.Seek(offset, os.SEEK_SET); err != nil { return errors.Errorf("seek %s to %d error %v", name, offset, err) } - return p.parseReader(f, onEvent) + return p.ParseReader(f, onEvent) } -func (p *BinlogParser) parseReader(r io.Reader, onEvent OnEventFunc) error { - p.Reset() +func (p *BinlogParser) parseFormatDescriptionEvent(r io.Reader, onEvent OnEventFunc) error { + _, err := p.parseSingleEvent(r, onEvent) + return err +} +// ParseSingleEvent parses single binlog event and passes the event to onEvent function. +func (p *BinlogParser) ParseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool, error) { + return p.parseSingleEvent(r, onEvent) +} + +func (p *BinlogParser) parseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool, error) { var err error var n int64 + var buf bytes.Buffer + if n, err = io.CopyN(&buf, r, EventHeaderSize); err == io.EOF { + return true, nil + } else if err != nil { + return false, errors.Errorf("get event header err %v, need %d but got %d", err, EventHeaderSize, n) + } + + var h *EventHeader + h, err = p.parseHeader(buf.Bytes()) + if err != nil { + return false, errors.Trace(err) + } + + if h.EventSize <= uint32(EventHeaderSize) { + return false, errors.Errorf("invalid event header, event size is %d, too small", h.EventSize) + } + if n, err = io.CopyN(&buf, r, int64(h.EventSize-EventHeaderSize)); err != nil { + return false, errors.Errorf("get event err %v, need %d but got %d", err, h.EventSize, n) + } + if buf.Len() != int(h.EventSize) { + return false, errors.Errorf("invalid raw data size in event %s, need %d but got %d", h.EventType, h.EventSize, buf.Len()) + } + + rawData := buf.Bytes() + bodyLen := int(h.EventSize) - EventHeaderSize + body := rawData[EventHeaderSize:] + if len(body) != bodyLen { + return false, errors.Errorf("invalid body data size in event %s, need %d but got %d", h.EventType, bodyLen, len(body)) + } + + var e Event + e, err = p.parseEvent(h, body, rawData) + if err != nil { + if err == errMissingTableMapEvent { + return false, nil + } + return false, errors.Trace(err) + } + + if err = onEvent(&BinlogEvent{RawData: rawData, Header: h, Event: e}); err != nil { + return false, errors.Trace(err) + } + + return false, nil +} + +func (p *BinlogParser) ParseReader(r io.Reader, onEvent OnEventFunc) error { + for { - headBuf := make([]byte, EventHeaderSize) - - if _, err = io.ReadFull(r, headBuf); err == io.EOF { - return nil - } else if err != nil { - return errors.Trace(err) - } - - var h *EventHeader - h, err = p.parseHeader(headBuf) - if err != nil { - return errors.Trace(err) - } - - if h.EventSize <= uint32(EventHeaderSize) { - return errors.Errorf("invalid event header, event size is %d, too small", h.EventSize) - - } - - var buf bytes.Buffer - if n, err = io.CopyN(&buf, r, int64(h.EventSize)-int64(EventHeaderSize)); err != nil { - return errors.Errorf("get event body err %v, need %d - %d, but got %d", err, h.EventSize, EventHeaderSize, n) - } - - data := buf.Bytes() - rawData := data - - eventLen := int(h.EventSize) - EventHeaderSize - - if len(data) != eventLen { - return errors.Errorf("invalid data size %d in event %s, less event length %d", len(data), h.EventType, eventLen) - } - - var e Event - e, err = p.parseEvent(h, data) - if err != nil { + if atomic.LoadUint32(&p.stopProcessing) == 1 { break } - if err = onEvent(&BinlogEvent{rawData, h, e}); err != nil { + done, err := p.parseSingleEvent(r, onEvent) + if err != nil { + if err == errMissingTableMapEvent { + continue + } return errors.Trace(err) } + + if done { + break + } } return nil @@ -115,6 +179,22 @@ func (p *BinlogParser) SetRawMode(mode bool) { p.rawMode = mode } +func (p *BinlogParser) SetParseTime(parseTime bool) { + p.parseTime = parseTime +} + +func (p *BinlogParser) SetTimestampStringLocation(timestampStringLocation *time.Location) { + p.timestampStringLocation = timestampStringLocation +} + +func (p *BinlogParser) SetUseDecimal(useDecimal bool) { + p.useDecimal = useDecimal +} + +func (p *BinlogParser) SetVerifyChecksum(verify bool) { + p.verifyChecksum = verify +} + func (p *BinlogParser) parseHeader(data []byte) (*EventHeader, error) { h := new(EventHeader) err := h.Decode(data) @@ -125,7 +205,7 @@ func (p *BinlogParser) parseHeader(data []byte) (*EventHeader, error) { return h, nil } -func (p *BinlogParser) parseEvent(h *EventHeader, data []byte) (Event, error) { +func (p *BinlogParser) parseEvent(h *EventHeader, data []byte, rawData []byte) (Event, error) { var e Event if h.EventType == FORMAT_DESCRIPTION_EVENT { @@ -133,7 +213,11 @@ func (p *BinlogParser) parseEvent(h *EventHeader, data []byte) (Event, error) { e = p.format } else { if p.format != nil && p.format.ChecksumAlgorithm == BINLOG_CHECKSUM_ALG_CRC32 { - data = data[0 : len(data)-4] + err := p.verifyCrc32Checksum(rawData) + if err != nil { + return nil, err + } + data = data[0 : len(data)-BinlogChecksumLength] } if h.EventType == ROTATE_EVENT { @@ -166,12 +250,14 @@ func (p *BinlogParser) parseEvent(h *EventHeader, data []byte) (Event, error) { e = &RowsQueryEvent{} case GTID_EVENT: e = >IDEvent{} + case ANONYMOUS_GTID_EVENT: + e = >IDEvent{} case BEGIN_LOAD_QUERY_EVENT: e = &BeginLoadQueryEvent{} case EXECUTE_LOAD_QUERY_EVENT: e = &ExecuteLoadQueryEvent{} case MARIADB_ANNOTATE_ROWS_EVENT: - e = &MariadbAnnotaeRowsEvent{} + e = &MariadbAnnotateRowsEvent{} case MARIADB_BINLOG_CHECKPOINT_EVENT: e = &MariadbBinlogCheckPointEvent{} case MARIADB_GTID_LIST_EVENT: @@ -206,7 +292,13 @@ func (p *BinlogParser) parseEvent(h *EventHeader, data []byte) (Event, error) { return e, nil } -func (p *BinlogParser) parse(data []byte) (*BinlogEvent, error) { +// Given the bytes for a a binary log event: return the decoded event. +// With the exception of the FORMAT_DESCRIPTION_EVENT event type +// there must have previously been passed a FORMAT_DESCRIPTION_EVENT +// into the parser for this to work properly on any given event. +// Passing a new FORMAT_DESCRIPTION_EVENT into the parser will replace +// an existing one. +func (p *BinlogParser) Parse(data []byte) (*BinlogEvent, error) { rawData := data h, err := p.parseHeader(data) @@ -222,12 +314,32 @@ func (p *BinlogParser) parse(data []byte) (*BinlogEvent, error) { return nil, fmt.Errorf("invalid data size %d in event %s, less event length %d", len(data), h.EventType, eventLen) } - e, err := p.parseEvent(h, data) + e, err := p.parseEvent(h, data, rawData) if err != nil { return nil, err } - return &BinlogEvent{rawData, h, e}, nil + return &BinlogEvent{RawData: rawData, Header: h, Event: e}, nil +} + +func (p *BinlogParser) verifyCrc32Checksum(rawData []byte) error { + if !p.verifyChecksum { + return nil + } + + calculatedPart := rawData[0 : len(rawData)-BinlogChecksumLength] + expectedChecksum := rawData[len(rawData)-BinlogChecksumLength:] + + // mysql use zlib's CRC32 implementation, which uses polynomial 0xedb88320UL. + // reference: https://github.com/madler/zlib/blob/master/crc32.c + // https://github.com/madler/zlib/blob/master/doc/rfc1952.txt#L419 + checksum := crc32.ChecksumIEEE(calculatedPart) + computed := make([]byte, BinlogChecksumLength) + binary.LittleEndian.PutUint32(computed, checksum) + if !bytes.Equal(expectedChecksum, computed) { + return ErrChecksumMismatch + } + return nil } func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent { @@ -240,6 +352,9 @@ func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent { e.needBitmap2 = false e.tables = p.tables + e.parseTime = p.parseTime + e.timestampStringLocation = p.timestampStringLocation + e.useDecimal = p.useDecimal switch h.EventType { case WRITE_ROWS_EVENTv0: diff --git a/vendor/github.com/siddontang/go-mysql/replication/parser_test.go b/vendor/github.com/siddontang/go-mysql/replication/parser_test.go index 52f5f1f..d4efc98 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/parser_test.go +++ b/vendor/github.com/siddontang/go-mysql/replication/parser_test.go @@ -29,7 +29,7 @@ func (t *testSyncerSuite) TestIndexOutOfRange(c *C) { 0x3065f: &TableMapEvent{tableIDSize: 6, TableID: 0x3065f, Flags: 0x1, Schema: []uint8{0x73, 0x65, 0x69, 0x75, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72}, Table: []uint8{0x63, 0x6f, 0x6e, 0x73, 0x5f, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x70, 0x65, 0x61, 0x6b, 0x6f, 0x75, 0x74, 0x5f, 0x6c, 0x65, 0x74, 0x74, 0x65, 0x72}, ColumnCount: 0xd, ColumnType: []uint8{0x3, 0x3, 0x3, 0x3, 0x1, 0x12, 0xf, 0xf, 0x12, 0xf, 0xf, 0x3, 0xf}, ColumnMeta: []uint16{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x180, 0x180, 0x0, 0x180, 0x180, 0x0, 0x2fd}, NullBitmap: []uint8{0xe0, 0x17}}, } - _, err := parser.parse([]byte{ + _, err := parser.Parse([]byte{ /* 0x00, */ 0xc1, 0x86, 0x8e, 0x55, 0x1e, 0xa5, 0x14, 0x80, 0xa, 0x55, 0x0, 0x0, 0x0, 0x7, 0xc, 0xbf, 0xe, 0x0, 0x0, 0x5f, 0x6, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x2, 0x0, 0xd, 0xff, 0x0, 0x0, 0x19, 0x63, 0x7, 0x0, 0xca, 0x61, 0x5, 0x0, 0x5e, 0xf7, 0xc, 0x0, 0xf5, 0x7, diff --git a/vendor/github.com/siddontang/go-mysql/replication/replication_test.go b/vendor/github.com/siddontang/go-mysql/replication/replication_test.go index 82cedad..50fd1ee 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/replication_test.go +++ b/vendor/github.com/siddontang/go-mysql/replication/replication_test.go @@ -1,15 +1,15 @@ package replication import ( + "context" "flag" "fmt" "os" + "path" "sync" "testing" "time" - "golang.org/x/net/context" - . "github.com/pingcap/check" uuid "github.com/satori/go.uuid" "github.com/siddontang/go-mysql/client" @@ -158,8 +158,8 @@ func (t *testSyncerSuite) testSync(c *C, s *BinlogStreamer) { t.testExecute(c, "DROP TABLE IF EXISTS test_json_v2") str = `CREATE TABLE test_json_v2 ( - id INT, - c JSON, + id INT, + c JSON, PRIMARY KEY (id) ) ENGINE=InnoDB` @@ -230,6 +230,28 @@ func (t *testSyncerSuite) testSync(c *C, s *BinlogStreamer) { } } + str = `DROP TABLE IF EXISTS test_parse_time` + t.testExecute(c, str) + + // Must allow zero time. + t.testExecute(c, `SET sql_mode=''`) + str = `CREATE TABLE test_parse_time ( + a1 DATETIME, + a2 DATETIME(3), + a3 DATETIME(6), + b1 TIMESTAMP, + b2 TIMESTAMP(3) , + b3 TIMESTAMP(6))` + t.testExecute(c, str) + + t.testExecute(c, `INSERT INTO test_parse_time VALUES + ("2014-09-08 17:51:04.123456", "2014-09-08 17:51:04.123456", "2014-09-08 17:51:04.123456", + "2014-09-08 17:51:04.123456","2014-09-08 17:51:04.123456","2014-09-08 17:51:04.123456"), + ("0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000", + "0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000"), + ("2014-09-08 17:51:04.000456", "2014-09-08 17:51:04.000456", "2014-09-08 17:51:04.000456", + "2014-09-08 17:51:04.000456","2014-09-08 17:51:04.000456","2014-09-08 17:51:04.000456")`) + t.wg.Wait() } @@ -263,12 +285,13 @@ func (t *testSyncerSuite) setupTest(c *C, flavor string) { } cfg := BinlogSyncerConfig{ - ServerID: 100, - Flavor: flavor, - Host: *testHost, - Port: port, - User: "root", - Password: "", + ServerID: 100, + Flavor: flavor, + Host: *testHost, + Port: port, + User: "root", + Password: "", + UseDecimal: true, } t.b = NewBinlogSyncer(cfg) @@ -281,7 +304,7 @@ func (t *testSyncerSuite) testPositionSync(c *C) { binFile, _ := r.GetString(0, 0) binPos, _ := r.GetInt(0, 1) - s, err := t.b.StartSync(mysql.Position{binFile, uint32(binPos)}) + s, err := t.b.StartSync(mysql.Position{Name: binFile, Pos: uint32(binPos)}) c.Assert(err, IsNil) // Test re-sync. @@ -373,12 +396,15 @@ func (t *testSyncerSuite) TestMysqlBinlogCodec(c *C) { t.testSync(c, nil) }() - os.RemoveAll("./var") + binlogDir := "./var" - err := t.b.StartBackup("./var", mysql.Position{"", uint32(0)}, 2*time.Second) + os.RemoveAll(binlogDir) + + err := t.b.StartBackup(binlogDir, mysql.Position{Name: "", Pos: uint32(0)}, 2*time.Second) c.Assert(err, IsNil) p := NewBinlogParser() + p.SetVerifyChecksum(true) f := func(e *BinlogEvent) error { if *testOutputLogs { @@ -388,9 +414,15 @@ func (t *testSyncerSuite) TestMysqlBinlogCodec(c *C) { return nil } - err = p.ParseFile("./var/mysql.000001", 0, f) + dir, err := os.Open(binlogDir) + c.Assert(err, IsNil) + defer dir.Close() + + files, err := dir.Readdirnames(-1) c.Assert(err, IsNil) - err = p.ParseFile("./var/mysql.000002", 0, f) - c.Assert(err, IsNil) + for _, file := range files { + err = p.ParseFile(path.Join(binlogDir, file), 0, f) + c.Assert(err, IsNil) + } } diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index 85a3e43..7cdd941 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -10,11 +10,14 @@ import ( "time" "github.com/juju/errors" - "github.com/ngaut/log" + "github.com/shopspring/decimal" + "github.com/siddontang/go-log/log" . "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go/hack" ) +var errMissingTableMapEvent = errors.New("invalid table id, no corresponding table map event") + type TableMapEvent struct { tableIDSize int @@ -68,7 +71,7 @@ func (e *TableMapEvent) Decode(data []byte) error { var err error var metaData []byte - if metaData, _, n, err = LengthEnodedString(data[pos:]); err != nil { + if metaData, _, n, err = LengthEncodedString(data[pos:]); err != nil { return errors.Trace(err) } @@ -78,11 +81,14 @@ func (e *TableMapEvent) Decode(data []byte) error { pos += n - if len(data[pos:]) != bitmapByteSize(int(e.ColumnCount)) { + nullBitmapSize := bitmapByteSize(int(e.ColumnCount)) + if len(data[pos:]) < nullBitmapSize { return io.EOF } - e.NullBitmap = data[pos:] + e.NullBitmap = data[pos : pos+nullBitmapSize] + + // TODO: handle optional field meta return nil } @@ -223,6 +229,10 @@ type RowsEvent struct { //rows: invalid: int64, float64, bool, []byte, string Rows [][]interface{} + + parseTime bool + timestampStringLocation *time.Location + useDecimal bool } func (e *RowsEvent) Decode(data []byte) error { @@ -257,7 +267,11 @@ func (e *RowsEvent) Decode(data []byte) error { var ok bool e.Table, ok = e.tables[e.TableID] if !ok { - return errors.Errorf("invalid table id %d, no correspond table map event", e.TableID) + if len(e.tables) > 0 { + return errors.Errorf("invalid table id %d, no corresponding table map event", e.TableID) + } else { + return errors.Annotatef(errMissingTableMapEvent, "table id %d", e.TableID) + } } var err error @@ -336,6 +350,21 @@ func (e *RowsEvent) decodeRows(data []byte, table *TableMapEvent, bitmap []byte) return pos, nil } +func (e *RowsEvent) parseFracTime(t interface{}) interface{} { + v, ok := t.(fracTime) + if !ok { + return t + } + + if !e.parseTime { + // Don't parse time, return string directly + return v.String() + } + + // return Golang time directly + return v.Time +} + // see mysql sql/log_event.cc log_event_print_value func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{}, n int, err error) { var length int = 0 @@ -378,7 +407,7 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ case MYSQL_TYPE_NEWDECIMAL: prec := uint8(meta >> 8) scale := uint8(meta & 0xFF) - v, n, err = decodeDecimal(data, int(prec), int(scale)) + v, n, err = decodeDecimal(data, int(prec), int(scale), e.useDecimal) case MYSQL_TYPE_FLOAT: n = 4 v = ParseBinaryFloat32(data) @@ -394,28 +423,43 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ case MYSQL_TYPE_TIMESTAMP: n = 4 t := binary.LittleEndian.Uint32(data) - v = time.Unix(int64(t), 0) + if t == 0 { + v = formatZeroTime(0, 0) + } else { + v = e.parseFracTime(fracTime{ + Time: time.Unix(int64(t), 0), + Dec: 0, + timestampStringLocation: e.timestampStringLocation, + }) + } case MYSQL_TYPE_TIMESTAMP2: - v, n, err = decodeTimestamp2(data, meta) + v, n, err = decodeTimestamp2(data, meta, e.timestampStringLocation) + v = e.parseFracTime(v) case MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) - - if i64 == 0 { // commented by Shlomi Noach. Yes I know about `git blame` - return "0000-00-00 00:00:00", n, nil + if i64 == 0 { + v = formatZeroTime(0, 0) + } else { + d := i64 / 1000000 + t := i64 % 1000000 + v = e.parseFracTime(fracTime{ + Time: time.Date( + int(d/10000), + time.Month((d%10000)/100), + int(d%100), + int(t/10000), + int((t%10000)/100), + int(t%100), + 0, + time.UTC, + ), + Dec: 0, + }) } - d := i64 / 1000000 - t := i64 % 1000000 - v = time.Date(int(d/10000), - time.Month((d%10000)/100), - int(d%100), - int(t/10000), - int((t%10000)/100), - int(t%100), - 0, - time.UTC).Format(TimeFormat) case MYSQL_TYPE_DATETIME2: v, n, err = decodeDatetime2(data, meta) + v = e.parseFracTime(v) case MYSQL_TYPE_TIME: n = 3 i32 := uint32(FixedLengthInt(data[0:3])) @@ -449,7 +493,7 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ v = int64(data[0]) n = 1 case 2: - v = int64(binary.BigEndian.Uint16(data)) + v = int64(binary.LittleEndian.Uint16(data)) n = 2 default: err = fmt.Errorf("Unknown ENUM packlen=%d", l) @@ -458,7 +502,7 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ n = int(meta & 0xFF) nbits := n * 8 - v, err = decodeBit(data, nbits, n) + v, err = littleDecodeBit(data, nbits, n) case MYSQL_TYPE_BLOB: v, n, err = decodeBlob(data, meta) case MYSQL_TYPE_VARCHAR, @@ -468,10 +512,10 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ case MYSQL_TYPE_STRING: v, n = decodeString(data, length) case MYSQL_TYPE_JSON: - // Refer https://github.com/shyiko/mysql-binlog-connector-java/blob/8f9132ee773317e00313204beeae8ddcaa43c1b4/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java#L344 - length = int(binary.LittleEndian.Uint32(data[0:])) + // Refer: https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java#L404 + length = int(FixedLengthInt(data[0:meta])) n = length + int(meta) - v, err = decodeJsonBinary(data[meta:n]) + v, err = e.decodeJsonBinary(data[meta:n]) case MYSQL_TYPE_GEOMETRY: // MySQL saves Geometry as Blob in binlog // Seem that the binary format is SRID (4 bytes) + WKB, outer can use @@ -515,7 +559,7 @@ func decodeDecimalDecompressValue(compIndx int, data []byte, mask uint8) (size i return } -func decodeDecimal(data []byte, precision int, decimals int) (float64, int, error) { +func decodeDecimal(data []byte, precision int, decimals int, useDecimal bool) (interface{}, int, error) { //see python mysql replication and https://github.com/jeremycole/mysql_binlog integral := (precision - decimals) uncompIntegral := int(integral / digitsPerInteger) @@ -568,6 +612,11 @@ func decodeDecimal(data []byte, precision int, decimals int) (float64, int, erro pos += size } + if useDecimal { + f, err := decimal.NewFromString(hack.String(res.Bytes())) + return f, pos, err + } + f, err := strconv.ParseFloat(hack.String(res.Bytes()), 64) return f, pos, err } @@ -604,7 +653,39 @@ func decodeBit(data []byte, nbits int, length int) (value int64, err error) { return } -func decodeTimestamp2(data []byte, dec uint16) (interface{}, int, error) { +func littleDecodeBit(data []byte, nbits int, length int) (value int64, err error) { + if nbits > 1 { + switch length { + case 1: + value = int64(data[0]) + case 2: + value = int64(binary.LittleEndian.Uint16(data)) + case 3: + value = int64(FixedLengthInt(data[0:3])) + case 4: + value = int64(binary.LittleEndian.Uint32(data)) + case 5: + value = int64(FixedLengthInt(data[0:5])) + case 6: + value = int64(FixedLengthInt(data[0:6])) + case 7: + value = int64(FixedLengthInt(data[0:7])) + case 8: + value = int64(binary.LittleEndian.Uint64(data)) + default: + err = fmt.Errorf("invalid bit length %d", length) + } + } else { + if length != 1 { + err = fmt.Errorf("invalid bit length %d", length) + } else { + value = int64(data[0]) + } + } + return +} + +func decodeTimestamp2(data []byte, dec uint16, timestampStringLocation *time.Location) (interface{}, int, error) { //get timestamp binary length n := int(4 + (dec+1)/2) sec := int64(binary.BigEndian.Uint32(data[0:4])) @@ -619,11 +700,14 @@ func decodeTimestamp2(data []byte, dec uint16) (interface{}, int, error) { } if sec == 0 { - return "0000-00-00 00:00:00", n, nil + return formatZeroTime(int(usec), int(dec)), n, nil } - t := time.Unix(sec, usec*1000) - return t, n, nil + return fracTime{ + Time: time.Unix(sec, usec*1000), + Dec: int(dec), + timestampStringLocation: timestampStringLocation, + }, n, nil } const DATETIMEF_INT_OFS int64 = 0x8000000000 @@ -645,7 +729,7 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { } if intPart == 0 { - return "0000-00-00 00:00:00", n, nil + return formatZeroTime(int(frac), int(dec)), n, nil } tmp := intPart<<24 + frac @@ -654,7 +738,7 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { tmp = -tmp } - var secPart int64 = tmp % (1 << 24) + // var secPart int64 = tmp % (1 << 24) ymdhms := tmp >> 24 ymd := ymdhms >> 17 @@ -669,10 +753,10 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { minute := int((hms >> 6) % (1 << 6)) hour := int((hms >> 12)) - if secPart != 0 { - return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second, secPart), n, nil // commented by Shlomi Noach. Yes I know about `git blame` - } - return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second), n, nil // commented by Shlomi Noach. Yes I know about `git blame` + return fracTime{ + Time: time.Date(year, time.Month(month), day, hour, minute, second, int(frac*1000), time.UTC), + Dec: int(dec), + }, n, nil } const TIMEF_OFS int64 = 0x800000000000 diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go b/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go index b1144c3..533a876 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event_test.go @@ -2,8 +2,10 @@ package replication import ( "fmt" + "strconv" . "github.com/pingcap/check" + "github.com/shopspring/decimal" ) type testDecodeSuite struct{} @@ -17,10 +19,10 @@ type decodeDecimalChecker struct { func (_ *decodeDecimalChecker) Check(params []interface{}, names []string) (bool, string) { var test int val := struct { - Value float64 + Value decimal.Decimal Pos int Err error - EValue float64 + EValue decimal.Decimal EPos int EErr error }{} @@ -28,13 +30,13 @@ func (_ *decodeDecimalChecker) Check(params []interface{}, names []string) (bool for i, name := range names { switch name { case "obtainedValue": - val.Value, _ = params[i].(float64) + val.Value, _ = params[i].(decimal.Decimal) case "obtainedPos": val.Pos, _ = params[i].(int) case "obtainedErr": val.Err, _ = params[i].(error) case "expectedValue": - val.EValue, _ = params[i].(float64) + val.EValue, _ = params[i].(decimal.Decimal) case "expectedPos": val.EPos, _ = params[i].(int) case "expectedErr": @@ -50,7 +52,7 @@ func (_ *decodeDecimalChecker) Check(params []interface{}, names []string) (bool if val.Pos != val.EPos { return false, fmt.Sprintf(errorMsgFmt, "position", val.EPos, val.Pos) } - if val.Value != val.EValue { + if !val.Value.Equal(val.EValue) { return false, fmt.Sprintf(errorMsgFmt, "value", val.EValue, val.Value) } return true, "" @@ -66,7 +68,7 @@ func (_ *testDecodeSuite) TestDecodeDecimal(c *C) { Data []byte Precision int Decimals int - Expected float64 + Expected string ExpectedPos int ExpectedErr error }{ @@ -133,197 +135,202 @@ func (_ *testDecodeSuite) TestDecodeDecimal(c *C) { | 17 | -99.99 | -1948 | -1948.140 | -1948.14 | -1948.140 | -1948.14 | -9.99999999999999 | -1948.1400000000 | -1948.14000 | -1948.14000000000000000000 | -1948.1400000000000000000000000 | 13 | 2 | +----+--------+-------+-----------+-------------+-------------+----------------+-------------------+-----------------------+---------------------+---------------------------------+---------------------------------+------+-------+ */ - {[]byte{117, 200, 127, 255}, 4, 2, float64(-10.55), 2, nil}, - {[]byte{127, 255, 244, 127, 245}, 5, 0, float64(-11), 3, nil}, - {[]byte{127, 245, 253, 217, 127, 255}, 7, 3, float64(-10.550), 4, nil}, - {[]byte{127, 255, 255, 245, 200, 127, 255}, 10, 2, float64(-10.55), 5, nil}, - {[]byte{127, 255, 255, 245, 253, 217, 127, 255}, 10, 3, float64(-10.550), 6, nil}, - {[]byte{127, 255, 255, 255, 245, 200, 118, 196}, 13, 2, float64(-10.55), 6, nil}, - {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, float64(-9.99999999999999), 8, nil}, - {[]byte{127, 255, 255, 255, 245, 223, 55, 170, 127, 255, 127, 255}, 20, 10, float64(-10.5500000000), 10, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 245, 255, 41, 39, 127, 255}, 30, 5, float64(-10.55000), 15, nil}, - {[]byte{127, 255, 255, 255, 245, 223, 55, 170, 127, 255, 255, 255, 255, 255, 127, 255}, 30, 20, float64(-10.55000000000000000000), 14, nil}, - {[]byte{127, 255, 245, 223, 55, 170, 127, 255, 255, 255, 255, 255, 255, 255, 255, 4, 0}, 30, 25, float64(-10.5500000000000000000000000), 15, nil}, - {[]byte{128, 1, 128, 0}, 4, 2, float64(0.01), 2, nil}, - {[]byte{128, 0, 0, 128, 0}, 5, 0, float64(0), 3, nil}, - {[]byte{128, 0, 0, 12, 128, 0}, 7, 3, float64(0.012), 4, nil}, - {[]byte{128, 0, 0, 0, 1, 128, 0}, 10, 2, float64(0.01), 5, nil}, - {[]byte{128, 0, 0, 0, 0, 12, 128, 0}, 10, 3, float64(0.012), 6, nil}, - {[]byte{128, 0, 0, 0, 0, 1, 128, 0}, 13, 2, float64(0.01), 6, nil}, - {[]byte{128, 0, 188, 97, 78, 1, 96, 11, 128, 0}, 15, 14, float64(0.01234567890123), 8, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 188, 97, 78, 9, 128, 0}, 20, 10, float64(0.0123456789), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 211, 128, 0}, 30, 5, float64(0.01235), 15, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 188, 97, 78, 53, 183, 191, 135, 89, 128, 0}, 30, 20, float64(0.01234567890123456789), 14, nil}, - {[]byte{128, 0, 0, 0, 188, 97, 78, 53, 183, 191, 135, 0, 135, 253, 217, 30, 0}, 30, 25, float64(0.0123456789012345678912345), 15, nil}, - {[]byte{227, 99, 128, 48}, 4, 2, float64(99.99), 2, nil}, - {[]byte{128, 48, 57, 167, 15}, 5, 0, float64(12345), 3, nil}, - {[]byte{167, 15, 3, 231, 128, 0}, 7, 3, float64(9999.999), 4, nil}, - {[]byte{128, 0, 48, 57, 0, 128, 0}, 10, 2, float64(12345.00), 5, nil}, - {[]byte{128, 0, 48, 57, 0, 0, 128, 0}, 10, 3, float64(12345.000), 6, nil}, - {[]byte{128, 0, 0, 48, 57, 0, 137, 59}, 13, 2, float64(12345.00), 6, nil}, - {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, float64(9.99999999999999), 8, nil}, - {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 128, 0}, 20, 10, float64(12345.0000000000), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 57, 0, 0, 0, 128, 0}, 30, 5, float64(12345.00000), 15, nil}, - {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 48}, 30, 20, float64(12345.00000000000000000000), 14, nil}, - {[]byte{128, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0}, 30, 25, float64(12345.0000000000000000000000000), 15, nil}, - {[]byte{227, 99, 128, 48}, 4, 2, float64(99.99), 2, nil}, - {[]byte{128, 48, 57, 167, 15}, 5, 0, float64(12345), 3, nil}, - {[]byte{167, 15, 3, 231, 128, 0}, 7, 3, float64(9999.999), 4, nil}, - {[]byte{128, 0, 48, 57, 0, 128, 0}, 10, 2, float64(12345.00), 5, nil}, - {[]byte{128, 0, 48, 57, 0, 0, 128, 0}, 10, 3, float64(12345.000), 6, nil}, - {[]byte{128, 0, 0, 48, 57, 0, 137, 59}, 13, 2, float64(12345.00), 6, nil}, - {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, float64(9.99999999999999), 8, nil}, - {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 128, 0}, 20, 10, float64(12345.0000000000), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 57, 0, 0, 0, 128, 0}, 30, 5, float64(12345.00000), 15, nil}, - {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 48}, 30, 20, float64(12345.00000000000000000000), 14, nil}, - {[]byte{128, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0}, 30, 25, float64(12345.0000000000000000000000000), 15, nil}, - {[]byte{227, 99, 128, 0}, 4, 2, float64(99.99), 2, nil}, - {[]byte{128, 0, 123, 128, 123}, 5, 0, float64(123), 3, nil}, - {[]byte{128, 123, 1, 194, 128, 0}, 7, 3, float64(123.450), 4, nil}, - {[]byte{128, 0, 0, 123, 45, 128, 0}, 10, 2, float64(123.45), 5, nil}, - {[]byte{128, 0, 0, 123, 1, 194, 128, 0}, 10, 3, float64(123.450), 6, nil}, - {[]byte{128, 0, 0, 0, 123, 45, 137, 59}, 13, 2, float64(123.45), 6, nil}, - {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, float64(9.99999999999999), 8, nil}, - {[]byte{128, 0, 0, 0, 123, 26, 210, 116, 128, 0, 128, 0}, 20, 10, float64(123.4500000000), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 175, 200, 128, 0}, 30, 5, float64(123.45000), 15, nil}, - {[]byte{128, 0, 0, 0, 123, 26, 210, 116, 128, 0, 0, 0, 0, 0, 128, 0}, 30, 20, float64(123.45000000000000000000), 14, nil}, - {[]byte{128, 0, 123, 26, 210, 116, 128, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0}, 30, 25, float64(123.4500000000000000000000000), 15, nil}, - {[]byte{28, 156, 127, 255}, 4, 2, float64(-99.99), 2, nil}, - {[]byte{127, 255, 132, 127, 132}, 5, 0, float64(-123), 3, nil}, - {[]byte{127, 132, 254, 61, 127, 255}, 7, 3, float64(-123.450), 4, nil}, - {[]byte{127, 255, 255, 132, 210, 127, 255}, 10, 2, float64(-123.45), 5, nil}, - {[]byte{127, 255, 255, 132, 254, 61, 127, 255}, 10, 3, float64(-123.450), 6, nil}, - {[]byte{127, 255, 255, 255, 132, 210, 118, 196}, 13, 2, float64(-123.45), 6, nil}, - {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, float64(-9.99999999999999), 8, nil}, - {[]byte{127, 255, 255, 255, 132, 229, 45, 139, 127, 255, 127, 255}, 20, 10, float64(-123.4500000000), 10, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 132, 255, 80, 55, 127, 255}, 30, 5, float64(-123.45000), 15, nil}, - {[]byte{127, 255, 255, 255, 132, 229, 45, 139, 127, 255, 255, 255, 255, 255, 127, 255}, 30, 20, float64(-123.45000000000000000000), 14, nil}, - {[]byte{127, 255, 132, 229, 45, 139, 127, 255, 255, 255, 255, 255, 255, 255, 255, 20, 0}, 30, 25, float64(-123.4500000000000000000000000), 15, nil}, - {[]byte{128, 0, 128, 0}, 4, 2, float64(0.00), 2, nil}, - {[]byte{128, 0, 0, 128, 0}, 5, 0, float64(0), 3, nil}, - {[]byte{128, 0, 0, 0, 128, 0}, 7, 3, float64(0.000), 4, nil}, - {[]byte{128, 0, 0, 0, 0, 128, 0}, 10, 2, float64(0.00), 5, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 10, 3, float64(0.000), 6, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 13, 2, float64(0.00), 6, nil}, - {[]byte{128, 0, 1, 226, 58, 0, 0, 99, 128, 0}, 15, 14, float64(0.00012345000099), 8, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 128, 0}, 20, 10, float64(0.0001234500), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 128, 0}, 30, 5, float64(0.00012), 15, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 128, 0}, 30, 20, float64(0.00012345000098765000), 14, nil}, - {[]byte{128, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 0, 0, 0, 15, 0}, 30, 25, float64(0.0001234500009876500000000), 15, nil}, - {[]byte{128, 0, 128, 0}, 4, 2, float64(0.00), 2, nil}, - {[]byte{128, 0, 0, 128, 0}, 5, 0, float64(0), 3, nil}, - {[]byte{128, 0, 0, 0, 128, 0}, 7, 3, float64(0.000), 4, nil}, - {[]byte{128, 0, 0, 0, 0, 128, 0}, 10, 2, float64(0.00), 5, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 10, 3, float64(0.000), 6, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 13, 2, float64(0.00), 6, nil}, - {[]byte{128, 0, 1, 226, 58, 0, 0, 99, 128, 0}, 15, 14, float64(0.00012345000099), 8, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 128, 0}, 20, 10, float64(0.0001234500), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 128, 0}, 30, 5, float64(0.00012), 15, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 128, 0}, 30, 20, float64(0.00012345000098765000), 14, nil}, - {[]byte{128, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 0, 0, 0, 22, 0}, 30, 25, float64(0.0001234500009876500000000), 15, nil}, - {[]byte{128, 12, 128, 0}, 4, 2, float64(0.12), 2, nil}, - {[]byte{128, 0, 0, 128, 0}, 5, 0, float64(0), 3, nil}, - {[]byte{128, 0, 0, 123, 128, 0}, 7, 3, float64(0.123), 4, nil}, - {[]byte{128, 0, 0, 0, 12, 128, 0}, 10, 2, float64(0.12), 5, nil}, - {[]byte{128, 0, 0, 0, 0, 123, 128, 0}, 10, 3, float64(0.123), 6, nil}, - {[]byte{128, 0, 0, 0, 0, 12, 128, 7}, 13, 2, float64(0.12), 6, nil}, - {[]byte{128, 7, 91, 178, 144, 1, 129, 205, 128, 0}, 15, 14, float64(0.12345000098765), 8, nil}, - {[]byte{128, 0, 0, 0, 0, 7, 91, 178, 145, 0, 128, 0}, 20, 10, float64(0.1234500010), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 57, 128, 0}, 30, 5, float64(0.12345), 15, nil}, - {[]byte{128, 0, 0, 0, 0, 7, 91, 178, 144, 58, 222, 87, 208, 0, 128, 0}, 30, 20, float64(0.12345000098765000000), 14, nil}, - {[]byte{128, 0, 0, 7, 91, 178, 144, 58, 222, 87, 208, 0, 0, 0, 0, 30, 0}, 30, 25, float64(0.1234500009876500000000000), 15, nil}, - {[]byte{128, 0, 128, 0}, 4, 2, float64(0.00), 2, nil}, - {[]byte{128, 0, 0, 128, 0}, 5, 0, float64(0), 3, nil}, - {[]byte{128, 0, 0, 0, 128, 0}, 7, 3, float64(0.000), 4, nil}, - {[]byte{128, 0, 0, 0, 0, 128, 0}, 10, 2, float64(0.00), 5, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 10, 3, float64(0.000), 6, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 127, 255}, 13, 2, float64(0.00), 6, nil}, - {[]byte{127, 255, 255, 255, 243, 255, 121, 59, 127, 255}, 15, 14, float64(-0.00000001234500), 8, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 243, 252, 128, 0}, 20, 10, float64(-0.0000000123), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 255}, 30, 5, float64(0.00000), 15, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 243, 235, 111, 183, 93, 178, 127, 255}, 30, 20, float64(-0.00000001234500009877), 14, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 243, 235, 111, 183, 93, 255, 139, 69, 47, 30, 0}, 30, 25, float64(-0.0000000123450000987650000), 15, nil}, - {[]byte{227, 99, 129, 134}, 4, 2, float64(99.99), 2, nil}, - {[]byte{129, 134, 159, 167, 15}, 5, 0, float64(99999), 3, nil}, - {[]byte{167, 15, 3, 231, 133, 245}, 7, 3, float64(9999.999), 4, nil}, - {[]byte{133, 245, 224, 255, 99, 128, 152}, 10, 2, float64(99999999.99), 5, nil}, - {[]byte{128, 152, 150, 127, 3, 231, 227, 59}, 10, 3, float64(9999999.999), 6, nil}, - {[]byte{227, 59, 154, 201, 255, 99, 137, 59}, 13, 2, float64(99999999999.99), 6, nil}, - {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 137, 59}, 15, 14, float64(9.99999999999999), 8, nil}, - {[]byte{137, 59, 154, 201, 255, 59, 154, 201, 255, 9, 128, 0}, 20, 10, float64(9999999999.9999999999), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 4, 210, 29, 205, 139, 148, 0, 195, 80, 137, 59}, 30, 5, float64(1234500009876.50000), 15, nil}, - {[]byte{137, 59, 154, 201, 255, 59, 154, 201, 255, 59, 154, 201, 255, 99, 129, 134}, 30, 20, float64(9999999999.99999999999999999999), 14, nil}, - {[]byte{129, 134, 159, 59, 154, 201, 255, 59, 154, 201, 255, 0, 152, 150, 127, 30, 0}, 30, 25, float64(99999.9999999999999999999999999), 15, nil}, - {[]byte{227, 99, 129, 134}, 4, 2, float64(99.99), 2, nil}, - {[]byte{129, 134, 159, 167, 15}, 5, 0, float64(99999), 3, nil}, - {[]byte{167, 15, 3, 231, 133, 245}, 7, 3, float64(9999.999), 4, nil}, - {[]byte{133, 245, 224, 255, 99, 128, 152}, 10, 2, float64(99999999.99), 5, nil}, - {[]byte{128, 152, 150, 127, 3, 231, 128, 6}, 10, 3, float64(9999999.999), 6, nil}, - {[]byte{128, 6, 159, 107, 199, 11, 137, 59}, 13, 2, float64(111111111.11), 6, nil}, - {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 6}, 15, 14, float64(9.99999999999999), 8, nil}, - {[]byte{128, 6, 159, 107, 199, 6, 142, 119, 128, 0, 128, 0}, 20, 10, float64(111111111.1100000000), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 6, 159, 107, 199, 0, 42, 248, 128, 6}, 30, 5, float64(111111111.11000), 15, nil}, - {[]byte{128, 6, 159, 107, 199, 6, 142, 119, 128, 0, 0, 0, 0, 0, 129, 134}, 30, 20, float64(111111111.11000000000000000000), 14, nil}, - {[]byte{129, 134, 159, 59, 154, 201, 255, 59, 154, 201, 255, 0, 152, 150, 127, 10, 0}, 30, 25, float64(99999.9999999999999999999999999), 15, nil}, - {[]byte{128, 1, 128, 0}, 4, 2, float64(0.01), 2, nil}, - {[]byte{128, 0, 0, 128, 0}, 5, 0, float64(0), 3, nil}, - {[]byte{128, 0, 0, 10, 128, 0}, 7, 3, float64(0.010), 4, nil}, - {[]byte{128, 0, 0, 0, 1, 128, 0}, 10, 2, float64(0.01), 5, nil}, - {[]byte{128, 0, 0, 0, 0, 10, 128, 0}, 10, 3, float64(0.010), 6, nil}, - {[]byte{128, 0, 0, 0, 0, 1, 128, 0}, 13, 2, float64(0.01), 6, nil}, - {[]byte{128, 0, 152, 150, 128, 0, 0, 0, 128, 0}, 15, 14, float64(0.01000000000000), 8, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 152, 150, 128, 0, 128, 0}, 20, 10, float64(0.0100000000), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 232, 128, 0}, 30, 5, float64(0.01000), 15, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 152, 150, 128, 0, 0, 0, 0, 0, 128, 0}, 30, 20, float64(0.01000000000000000000), 14, nil}, - {[]byte{128, 0, 0, 0, 152, 150, 128, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0}, 30, 25, float64(0.0100000000000000000000000), 15, nil}, - {[]byte{227, 99, 128, 0}, 4, 2, float64(99.99), 2, nil}, - {[]byte{128, 0, 123, 128, 123}, 5, 0, float64(123), 3, nil}, - {[]byte{128, 123, 1, 144, 128, 0}, 7, 3, float64(123.400), 4, nil}, - {[]byte{128, 0, 0, 123, 40, 128, 0}, 10, 2, float64(123.40), 5, nil}, - {[]byte{128, 0, 0, 123, 1, 144, 128, 0}, 10, 3, float64(123.400), 6, nil}, - {[]byte{128, 0, 0, 0, 123, 40, 137, 59}, 13, 2, float64(123.40), 6, nil}, - {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, float64(9.99999999999999), 8, nil}, - {[]byte{128, 0, 0, 0, 123, 23, 215, 132, 0, 0, 128, 0}, 20, 10, float64(123.4000000000), 10, nil}, - {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 156, 64, 128, 0}, 30, 5, float64(123.40000), 15, nil}, - {[]byte{128, 0, 0, 0, 123, 23, 215, 132, 0, 0, 0, 0, 0, 0, 128, 0}, 30, 20, float64(123.40000000000000000000), 14, nil}, - {[]byte{128, 0, 123, 23, 215, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0}, 30, 25, float64(123.4000000000000000000000000), 15, nil}, - {[]byte{28, 156, 127, 253}, 4, 2, float64(-99.99), 2, nil}, - {[]byte{127, 253, 204, 125, 205}, 5, 0, float64(-563), 3, nil}, - {[]byte{125, 205, 253, 187, 127, 255}, 7, 3, float64(-562.580), 4, nil}, - {[]byte{127, 255, 253, 205, 197, 127, 255}, 10, 2, float64(-562.58), 5, nil}, - {[]byte{127, 255, 253, 205, 253, 187, 127, 255}, 10, 3, float64(-562.580), 6, nil}, - {[]byte{127, 255, 255, 253, 205, 197, 118, 196}, 13, 2, float64(-562.58), 6, nil}, - {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, float64(-9.99999999999999), 8, nil}, - {[]byte{127, 255, 255, 253, 205, 221, 109, 230, 255, 255, 127, 255}, 20, 10, float64(-562.5800000000), 10, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 253, 205, 255, 29, 111, 127, 255}, 30, 5, float64(-562.58000), 15, nil}, - {[]byte{127, 255, 255, 253, 205, 221, 109, 230, 255, 255, 255, 255, 255, 255, 127, 253}, 30, 20, float64(-562.58000000000000000000), 14, nil}, - {[]byte{127, 253, 205, 221, 109, 230, 255, 255, 255, 255, 255, 255, 255, 255, 255, 13, 0}, 30, 25, float64(-562.5800000000000000000000000), 15, nil}, - {[]byte{28, 156, 127, 241}, 4, 2, float64(-99.99), 2, nil}, - {[]byte{127, 241, 140, 113, 140}, 5, 0, float64(-3699), 3, nil}, - {[]byte{113, 140, 255, 245, 127, 255}, 7, 3, float64(-3699.010), 4, nil}, - {[]byte{127, 255, 241, 140, 254, 127, 255}, 10, 2, float64(-3699.01), 5, nil}, - {[]byte{127, 255, 241, 140, 255, 245, 127, 255}, 10, 3, float64(-3699.010), 6, nil}, - {[]byte{127, 255, 255, 241, 140, 254, 118, 196}, 13, 2, float64(-3699.01), 6, nil}, - {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, float64(-9.99999999999999), 8, nil}, - {[]byte{127, 255, 255, 241, 140, 255, 103, 105, 127, 255, 127, 255}, 20, 10, float64(-3699.0100000000), 10, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 241, 140, 255, 252, 23, 127, 255}, 30, 5, float64(-3699.01000), 15, nil}, - {[]byte{127, 255, 255, 241, 140, 255, 103, 105, 127, 255, 255, 255, 255, 255, 127, 241}, 30, 20, float64(-3699.01000000000000000000), 14, nil}, - {[]byte{127, 241, 140, 255, 103, 105, 127, 255, 255, 255, 255, 255, 255, 255, 255, 13, 0}, 30, 25, float64(-3699.0100000000000000000000000), 15, nil}, - {[]byte{28, 156, 127, 248}, 4, 2, float64(-99.99), 2, nil}, - {[]byte{127, 248, 99, 120, 99}, 5, 0, float64(-1948), 3, nil}, - {[]byte{120, 99, 255, 115, 127, 255}, 7, 3, float64(-1948.140), 4, nil}, - {[]byte{127, 255, 248, 99, 241, 127, 255}, 10, 2, float64(-1948.14), 5, nil}, - {[]byte{127, 255, 248, 99, 255, 115, 127, 255}, 10, 3, float64(-1948.140), 6, nil}, - {[]byte{127, 255, 255, 248, 99, 241, 118, 196}, 13, 2, float64(-1948.14), 6, nil}, - {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, float64(-9.99999999999999), 8, nil}, - {[]byte{127, 255, 255, 248, 99, 247, 167, 196, 255, 255, 127, 255}, 20, 10, float64(-1948.1400000000), 10, nil}, - {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 248, 99, 255, 201, 79, 127, 255}, 30, 5, float64(-1948.14000), 15, nil}, - {[]byte{127, 255, 255, 248, 99, 247, 167, 196, 255, 255, 255, 255, 255, 255, 127, 248}, 30, 20, float64(-1948.14000000000000000000), 14, nil}, - {[]byte{127, 248, 99, 247, 167, 196, 255, 255, 255, 255, 255, 255, 255, 255, 255, 13, 0}, 30, 25, float64(-1948.1400000000000000000000000), 15, nil}, + {[]byte{117, 200, 127, 255}, 4, 2, "-10.55", 2, nil}, + {[]byte{127, 255, 244, 127, 245}, 5, 0, "-11", 3, nil}, + {[]byte{127, 245, 253, 217, 127, 255}, 7, 3, "-10.550", 4, nil}, + {[]byte{127, 255, 255, 245, 200, 127, 255}, 10, 2, "-10.55", 5, nil}, + {[]byte{127, 255, 255, 245, 253, 217, 127, 255}, 10, 3, "-10.550", 6, nil}, + {[]byte{127, 255, 255, 255, 245, 200, 118, 196}, 13, 2, "-10.55", 6, nil}, + {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, "-9.99999999999999", 8, nil}, + {[]byte{127, 255, 255, 255, 245, 223, 55, 170, 127, 255, 127, 255}, 20, 10, "-10.5500000000", 10, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 245, 255, 41, 39, 127, 255}, 30, 5, "-10.55000", 15, nil}, + {[]byte{127, 255, 255, 255, 245, 223, 55, 170, 127, 255, 255, 255, 255, 255, 127, 255}, 30, 20, "-10.55000000000000000000", 14, nil}, + {[]byte{127, 255, 245, 223, 55, 170, 127, 255, 255, 255, 255, 255, 255, 255, 255, 4, 0}, 30, 25, "-10.5500000000000000000000000", 15, nil}, + {[]byte{128, 1, 128, 0}, 4, 2, "0.01", 2, nil}, + {[]byte{128, 0, 0, 128, 0}, 5, 0, "0", 3, nil}, + {[]byte{128, 0, 0, 12, 128, 0}, 7, 3, "0.012", 4, nil}, + {[]byte{128, 0, 0, 0, 1, 128, 0}, 10, 2, "0.01", 5, nil}, + {[]byte{128, 0, 0, 0, 0, 12, 128, 0}, 10, 3, "0.012", 6, nil}, + {[]byte{128, 0, 0, 0, 0, 1, 128, 0}, 13, 2, "0.01", 6, nil}, + {[]byte{128, 0, 188, 97, 78, 1, 96, 11, 128, 0}, 15, 14, "0.01234567890123", 8, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 188, 97, 78, 9, 128, 0}, 20, 10, "0.0123456789", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 211, 128, 0}, 30, 5, "0.01235", 15, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 188, 97, 78, 53, 183, 191, 135, 89, 128, 0}, 30, 20, "0.01234567890123456789", 14, nil}, + {[]byte{128, 0, 0, 0, 188, 97, 78, 53, 183, 191, 135, 0, 135, 253, 217, 30, 0}, 30, 25, "0.0123456789012345678912345", 15, nil}, + {[]byte{227, 99, 128, 48}, 4, 2, "99.99", 2, nil}, + {[]byte{128, 48, 57, 167, 15}, 5, 0, "12345", 3, nil}, + {[]byte{167, 15, 3, 231, 128, 0}, 7, 3, "9999.999", 4, nil}, + {[]byte{128, 0, 48, 57, 0, 128, 0}, 10, 2, "12345.00", 5, nil}, + {[]byte{128, 0, 48, 57, 0, 0, 128, 0}, 10, 3, "12345.000", 6, nil}, + {[]byte{128, 0, 0, 48, 57, 0, 137, 59}, 13, 2, "12345.00", 6, nil}, + {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, "9.99999999999999", 8, nil}, + {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 128, 0}, 20, 10, "12345.0000000000", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 57, 0, 0, 0, 128, 0}, 30, 5, "12345.00000", 15, nil}, + {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 48}, 30, 20, "12345.00000000000000000000", 14, nil}, + {[]byte{128, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0}, 30, 25, "12345.0000000000000000000000000", 15, nil}, + {[]byte{227, 99, 128, 48}, 4, 2, "99.99", 2, nil}, + {[]byte{128, 48, 57, 167, 15}, 5, 0, "12345", 3, nil}, + {[]byte{167, 15, 3, 231, 128, 0}, 7, 3, "9999.999", 4, nil}, + {[]byte{128, 0, 48, 57, 0, 128, 0}, 10, 2, "12345.00", 5, nil}, + {[]byte{128, 0, 48, 57, 0, 0, 128, 0}, 10, 3, "12345.000", 6, nil}, + {[]byte{128, 0, 0, 48, 57, 0, 137, 59}, 13, 2, "12345.00", 6, nil}, + {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, "9.99999999999999", 8, nil}, + {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 128, 0}, 20, 10, "12345.0000000000", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 57, 0, 0, 0, 128, 0}, 30, 5, "12345.00000", 15, nil}, + {[]byte{128, 0, 0, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 48}, 30, 20, "12345.00000000000000000000", 14, nil}, + {[]byte{128, 48, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0}, 30, 25, "12345.0000000000000000000000000", 15, nil}, + {[]byte{227, 99, 128, 0}, 4, 2, "99.99", 2, nil}, + {[]byte{128, 0, 123, 128, 123}, 5, 0, "123", 3, nil}, + {[]byte{128, 123, 1, 194, 128, 0}, 7, 3, "123.450", 4, nil}, + {[]byte{128, 0, 0, 123, 45, 128, 0}, 10, 2, "123.45", 5, nil}, + {[]byte{128, 0, 0, 123, 1, 194, 128, 0}, 10, 3, "123.450", 6, nil}, + {[]byte{128, 0, 0, 0, 123, 45, 137, 59}, 13, 2, "123.45", 6, nil}, + {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, "9.99999999999999", 8, nil}, + {[]byte{128, 0, 0, 0, 123, 26, 210, 116, 128, 0, 128, 0}, 20, 10, "123.4500000000", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 175, 200, 128, 0}, 30, 5, "123.45000", 15, nil}, + {[]byte{128, 0, 0, 0, 123, 26, 210, 116, 128, 0, 0, 0, 0, 0, 128, 0}, 30, 20, "123.45000000000000000000", 14, nil}, + {[]byte{128, 0, 123, 26, 210, 116, 128, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0}, 30, 25, "123.4500000000000000000000000", 15, nil}, + {[]byte{28, 156, 127, 255}, 4, 2, "-99.99", 2, nil}, + {[]byte{127, 255, 132, 127, 132}, 5, 0, "-123", 3, nil}, + {[]byte{127, 132, 254, 61, 127, 255}, 7, 3, "-123.450", 4, nil}, + {[]byte{127, 255, 255, 132, 210, 127, 255}, 10, 2, "-123.45", 5, nil}, + {[]byte{127, 255, 255, 132, 254, 61, 127, 255}, 10, 3, "-123.450", 6, nil}, + {[]byte{127, 255, 255, 255, 132, 210, 118, 196}, 13, 2, "-123.45", 6, nil}, + {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, "-9.99999999999999", 8, nil}, + {[]byte{127, 255, 255, 255, 132, 229, 45, 139, 127, 255, 127, 255}, 20, 10, "-123.4500000000", 10, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 132, 255, 80, 55, 127, 255}, 30, 5, "-123.45000", 15, nil}, + {[]byte{127, 255, 255, 255, 132, 229, 45, 139, 127, 255, 255, 255, 255, 255, 127, 255}, 30, 20, "-123.45000000000000000000", 14, nil}, + {[]byte{127, 255, 132, 229, 45, 139, 127, 255, 255, 255, 255, 255, 255, 255, 255, 20, 0}, 30, 25, "-123.4500000000000000000000000", 15, nil}, + {[]byte{128, 0, 128, 0}, 4, 2, "0.00", 2, nil}, + {[]byte{128, 0, 0, 128, 0}, 5, 0, "0", 3, nil}, + {[]byte{128, 0, 0, 0, 128, 0}, 7, 3, "0.000", 4, nil}, + {[]byte{128, 0, 0, 0, 0, 128, 0}, 10, 2, "0.00", 5, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 10, 3, "0.000", 6, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 13, 2, "0.00", 6, nil}, + {[]byte{128, 0, 1, 226, 58, 0, 0, 99, 128, 0}, 15, 14, "0.00012345000099", 8, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 128, 0}, 20, 10, "0.0001234500", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 128, 0}, 30, 5, "0.00012", 15, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 128, 0}, 30, 20, "0.00012345000098765000", 14, nil}, + {[]byte{128, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 0, 0, 0, 15, 0}, 30, 25, "0.0001234500009876500000000", 15, nil}, + {[]byte{128, 0, 128, 0}, 4, 2, "0.00", 2, nil}, + {[]byte{128, 0, 0, 128, 0}, 5, 0, "0", 3, nil}, + {[]byte{128, 0, 0, 0, 128, 0}, 7, 3, "0.000", 4, nil}, + {[]byte{128, 0, 0, 0, 0, 128, 0}, 10, 2, "0.00", 5, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 10, 3, "0.000", 6, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 13, 2, "0.00", 6, nil}, + {[]byte{128, 0, 1, 226, 58, 0, 0, 99, 128, 0}, 15, 14, "0.00012345000099", 8, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 128, 0}, 20, 10, "0.0001234500", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 128, 0}, 30, 5, "0.00012", 15, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 128, 0}, 30, 20, "0.00012345000098765000", 14, nil}, + {[]byte{128, 0, 0, 0, 1, 226, 58, 0, 15, 18, 2, 0, 0, 0, 0, 22, 0}, 30, 25, "0.0001234500009876500000000", 15, nil}, + {[]byte{128, 12, 128, 0}, 4, 2, "0.12", 2, nil}, + {[]byte{128, 0, 0, 128, 0}, 5, 0, "0", 3, nil}, + {[]byte{128, 0, 0, 123, 128, 0}, 7, 3, "0.123", 4, nil}, + {[]byte{128, 0, 0, 0, 12, 128, 0}, 10, 2, "0.12", 5, nil}, + {[]byte{128, 0, 0, 0, 0, 123, 128, 0}, 10, 3, "0.123", 6, nil}, + {[]byte{128, 0, 0, 0, 0, 12, 128, 7}, 13, 2, "0.12", 6, nil}, + {[]byte{128, 7, 91, 178, 144, 1, 129, 205, 128, 0}, 15, 14, "0.12345000098765", 8, nil}, + {[]byte{128, 0, 0, 0, 0, 7, 91, 178, 145, 0, 128, 0}, 20, 10, "0.1234500010", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 57, 128, 0}, 30, 5, "0.12345", 15, nil}, + {[]byte{128, 0, 0, 0, 0, 7, 91, 178, 144, 58, 222, 87, 208, 0, 128, 0}, 30, 20, "0.12345000098765000000", 14, nil}, + {[]byte{128, 0, 0, 7, 91, 178, 144, 58, 222, 87, 208, 0, 0, 0, 0, 30, 0}, 30, 25, "0.1234500009876500000000000", 15, nil}, + {[]byte{128, 0, 128, 0}, 4, 2, "0.00", 2, nil}, + {[]byte{128, 0, 0, 128, 0}, 5, 0, "0", 3, nil}, + {[]byte{128, 0, 0, 0, 128, 0}, 7, 3, "0.000", 4, nil}, + {[]byte{128, 0, 0, 0, 0, 128, 0}, 10, 2, "0.00", 5, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 128, 0}, 10, 3, "0.000", 6, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 127, 255}, 13, 2, "0.00", 6, nil}, + {[]byte{127, 255, 255, 255, 243, 255, 121, 59, 127, 255}, 15, 14, "-0.00000001234500", 8, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 243, 252, 128, 0}, 20, 10, "-0.0000000123", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 255}, 30, 5, "0.00000", 15, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 243, 235, 111, 183, 93, 178, 127, 255}, 30, 20, "-0.00000001234500009877", 14, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 243, 235, 111, 183, 93, 255, 139, 69, 47, 30, 0}, 30, 25, "-0.0000000123450000987650000", 15, nil}, + {[]byte{227, 99, 129, 134}, 4, 2, "99.99", 2, nil}, + {[]byte{129, 134, 159, 167, 15}, 5, 0, "99999", 3, nil}, + {[]byte{167, 15, 3, 231, 133, 245}, 7, 3, "9999.999", 4, nil}, + {[]byte{133, 245, 224, 255, 99, 128, 152}, 10, 2, "99999999.99", 5, nil}, + {[]byte{128, 152, 150, 127, 3, 231, 227, 59}, 10, 3, "9999999.999", 6, nil}, + {[]byte{227, 59, 154, 201, 255, 99, 137, 59}, 13, 2, "99999999999.99", 6, nil}, + {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 137, 59}, 15, 14, "9.99999999999999", 8, nil}, + {[]byte{137, 59, 154, 201, 255, 59, 154, 201, 255, 9, 128, 0}, 20, 10, "9999999999.9999999999", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 4, 210, 29, 205, 139, 148, 0, 195, 80, 137, 59}, 30, 5, "1234500009876.50000", 15, nil}, + {[]byte{137, 59, 154, 201, 255, 59, 154, 201, 255, 59, 154, 201, 255, 99, 129, 134}, 30, 20, "9999999999.99999999999999999999", 14, nil}, + {[]byte{129, 134, 159, 59, 154, 201, 255, 59, 154, 201, 255, 0, 152, 150, 127, 30, 0}, 30, 25, "99999.9999999999999999999999999", 15, nil}, + {[]byte{227, 99, 129, 134}, 4, 2, "99.99", 2, nil}, + {[]byte{129, 134, 159, 167, 15}, 5, 0, "99999", 3, nil}, + {[]byte{167, 15, 3, 231, 133, 245}, 7, 3, "9999.999", 4, nil}, + {[]byte{133, 245, 224, 255, 99, 128, 152}, 10, 2, "99999999.99", 5, nil}, + {[]byte{128, 152, 150, 127, 3, 231, 128, 6}, 10, 3, "9999999.999", 6, nil}, + {[]byte{128, 6, 159, 107, 199, 11, 137, 59}, 13, 2, "111111111.11", 6, nil}, + {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 6}, 15, 14, "9.99999999999999", 8, nil}, + {[]byte{128, 6, 159, 107, 199, 6, 142, 119, 128, 0, 128, 0}, 20, 10, "111111111.1100000000", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 6, 159, 107, 199, 0, 42, 248, 128, 6}, 30, 5, "111111111.11000", 15, nil}, + {[]byte{128, 6, 159, 107, 199, 6, 142, 119, 128, 0, 0, 0, 0, 0, 129, 134}, 30, 20, "111111111.11000000000000000000", 14, nil}, + {[]byte{129, 134, 159, 59, 154, 201, 255, 59, 154, 201, 255, 0, 152, 150, 127, 10, 0}, 30, 25, "99999.9999999999999999999999999", 15, nil}, + {[]byte{128, 1, 128, 0}, 4, 2, "0.01", 2, nil}, + {[]byte{128, 0, 0, 128, 0}, 5, 0, "0", 3, nil}, + {[]byte{128, 0, 0, 10, 128, 0}, 7, 3, "0.010", 4, nil}, + {[]byte{128, 0, 0, 0, 1, 128, 0}, 10, 2, "0.01", 5, nil}, + {[]byte{128, 0, 0, 0, 0, 10, 128, 0}, 10, 3, "0.010", 6, nil}, + {[]byte{128, 0, 0, 0, 0, 1, 128, 0}, 13, 2, "0.01", 6, nil}, + {[]byte{128, 0, 152, 150, 128, 0, 0, 0, 128, 0}, 15, 14, "0.01000000000000", 8, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 152, 150, 128, 0, 128, 0}, 20, 10, "0.0100000000", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 232, 128, 0}, 30, 5, "0.01000", 15, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 152, 150, 128, 0, 0, 0, 0, 0, 128, 0}, 30, 20, "0.01000000000000000000", 14, nil}, + {[]byte{128, 0, 0, 0, 152, 150, 128, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0}, 30, 25, "0.0100000000000000000000000", 15, nil}, + {[]byte{227, 99, 128, 0}, 4, 2, "99.99", 2, nil}, + {[]byte{128, 0, 123, 128, 123}, 5, 0, "123", 3, nil}, + {[]byte{128, 123, 1, 144, 128, 0}, 7, 3, "123.400", 4, nil}, + {[]byte{128, 0, 0, 123, 40, 128, 0}, 10, 2, "123.40", 5, nil}, + {[]byte{128, 0, 0, 123, 1, 144, 128, 0}, 10, 3, "123.400", 6, nil}, + {[]byte{128, 0, 0, 0, 123, 40, 137, 59}, 13, 2, "123.40", 6, nil}, + {[]byte{137, 59, 154, 201, 255, 1, 134, 159, 128, 0}, 15, 14, "9.99999999999999", 8, nil}, + {[]byte{128, 0, 0, 0, 123, 23, 215, 132, 0, 0, 128, 0}, 20, 10, "123.4000000000", 10, nil}, + {[]byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 156, 64, 128, 0}, 30, 5, "123.40000", 15, nil}, + {[]byte{128, 0, 0, 0, 123, 23, 215, 132, 0, 0, 0, 0, 0, 0, 128, 0}, 30, 20, "123.40000000000000000000", 14, nil}, + {[]byte{128, 0, 123, 23, 215, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0}, 30, 25, "123.4000000000000000000000000", 15, nil}, + {[]byte{28, 156, 127, 253}, 4, 2, "-99.99", 2, nil}, + {[]byte{127, 253, 204, 125, 205}, 5, 0, "-563", 3, nil}, + {[]byte{125, 205, 253, 187, 127, 255}, 7, 3, "-562.580", 4, nil}, + {[]byte{127, 255, 253, 205, 197, 127, 255}, 10, 2, "-562.58", 5, nil}, + {[]byte{127, 255, 253, 205, 253, 187, 127, 255}, 10, 3, "-562.580", 6, nil}, + {[]byte{127, 255, 255, 253, 205, 197, 118, 196}, 13, 2, "-562.58", 6, nil}, + {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, "-9.99999999999999", 8, nil}, + {[]byte{127, 255, 255, 253, 205, 221, 109, 230, 255, 255, 127, 255}, 20, 10, "-562.5800000000", 10, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 253, 205, 255, 29, 111, 127, 255}, 30, 5, "-562.58000", 15, nil}, + {[]byte{127, 255, 255, 253, 205, 221, 109, 230, 255, 255, 255, 255, 255, 255, 127, 253}, 30, 20, "-562.58000000000000000000", 14, nil}, + {[]byte{127, 253, 205, 221, 109, 230, 255, 255, 255, 255, 255, 255, 255, 255, 255, 13, 0}, 30, 25, "-562.5800000000000000000000000", 15, nil}, + {[]byte{28, 156, 127, 241}, 4, 2, "-99.99", 2, nil}, + {[]byte{127, 241, 140, 113, 140}, 5, 0, "-3699", 3, nil}, + {[]byte{113, 140, 255, 245, 127, 255}, 7, 3, "-3699.010", 4, nil}, + {[]byte{127, 255, 241, 140, 254, 127, 255}, 10, 2, "-3699.01", 5, nil}, + {[]byte{127, 255, 241, 140, 255, 245, 127, 255}, 10, 3, "-3699.010", 6, nil}, + {[]byte{127, 255, 255, 241, 140, 254, 118, 196}, 13, 2, "-3699.01", 6, nil}, + {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, "-9.99999999999999", 8, nil}, + {[]byte{127, 255, 255, 241, 140, 255, 103, 105, 127, 255, 127, 255}, 20, 10, "-3699.0100000000", 10, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 241, 140, 255, 252, 23, 127, 255}, 30, 5, "-3699.01000", 15, nil}, + {[]byte{127, 255, 255, 241, 140, 255, 103, 105, 127, 255, 255, 255, 255, 255, 127, 241}, 30, 20, "-3699.01000000000000000000", 14, nil}, + {[]byte{127, 241, 140, 255, 103, 105, 127, 255, 255, 255, 255, 255, 255, 255, 255, 13, 0}, 30, 25, "-3699.0100000000000000000000000", 15, nil}, + {[]byte{28, 156, 127, 248}, 4, 2, "-99.99", 2, nil}, + {[]byte{127, 248, 99, 120, 99}, 5, 0, "-1948", 3, nil}, + {[]byte{120, 99, 255, 115, 127, 255}, 7, 3, "-1948.140", 4, nil}, + {[]byte{127, 255, 248, 99, 241, 127, 255}, 10, 2, "-1948.14", 5, nil}, + {[]byte{127, 255, 248, 99, 255, 115, 127, 255}, 10, 3, "-1948.140", 6, nil}, + {[]byte{127, 255, 255, 248, 99, 241, 118, 196}, 13, 2, "-1948.14", 6, nil}, + {[]byte{118, 196, 101, 54, 0, 254, 121, 96, 127, 255}, 15, 14, "-9.99999999999999", 8, nil}, + {[]byte{127, 255, 255, 248, 99, 247, 167, 196, 255, 255, 127, 255}, 20, 10, "-1948.1400000000", 10, nil}, + {[]byte{127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 248, 99, 255, 201, 79, 127, 255}, 30, 5, "-1948.14000", 15, nil}, + {[]byte{127, 255, 255, 248, 99, 247, 167, 196, 255, 255, 255, 255, 255, 255, 127, 248}, 30, 20, "-1948.14000000000000000000", 14, nil}, + {[]byte{127, 248, 99, 247, 167, 196, 255, 255, 255, 255, 255, 255, 255, 255, 255, 13, 0}, 30, 25, "-1948.1400000000000000000000000", 15, nil}, } for i, tc := range testcases { - value, pos, err := decodeDecimal(tc.Data, tc.Precision, tc.Decimals) - c.Assert(value, DecodeDecimalsEquals, pos, err, tc.Expected, tc.ExpectedPos, tc.ExpectedErr, i) + value, pos, err := decodeDecimal(tc.Data, tc.Precision, tc.Decimals, false) + expectedFloat, _ := strconv.ParseFloat(tc.Expected, 64) + c.Assert(value.(float64), DecodeDecimalsEquals, pos, err, expectedFloat, tc.ExpectedPos, tc.ExpectedErr, i) + + value, pos, err = decodeDecimal(tc.Data, tc.Precision, tc.Decimals, true) + expectedDecimal, _ := decimal.NewFromString(tc.Expected) + c.Assert(value.(decimal.Decimal), DecodeDecimalsEquals, pos, err, expectedDecimal, tc.ExpectedPos, tc.ExpectedErr, i) } } @@ -386,6 +393,25 @@ func (_ *testDecodeSuite) TestParseRowPanic(c *C) { c.Assert(rows.Rows[0][0], Equals, int32(16270)) } +type simpleDecimalEqualsChecker struct { + *CheckerInfo +} + +var SimpleDecimalEqualsChecker Checker = &simpleDecimalEqualsChecker{ + &CheckerInfo{Name: "Equals", Params: []string{"obtained", "expected"}}, +} + +func (checker *simpleDecimalEqualsChecker) Check(params []interface{}, names []string) (result bool, error string) { + defer func() { + if v := recover(); v != nil { + result = false + error = fmt.Sprint(v) + } + }() + + return params[0].(decimal.Decimal).Equal(params[1].(decimal.Decimal)), "" +} + func (_ *testDecodeSuite) TestParseJson(c *C) { // Table format: // mysql> desc t10; @@ -405,7 +431,6 @@ func (_ *testDecodeSuite) TestParseJson(c *C) { // INSERT INTO `t10` (`c1`, `c2`) VALUES ('{"key1": "value1", "key2": "value2"}', 1); // test json deserialization // INSERT INTO `t10`(`c1`,`c2`) VALUES ('{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}',101); - tableMapEventData := []byte("m\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x03t10\x00\x02\xf5\xf6\x03\x04\n\x00\x03") tableMapEvent := new(TableMapEvent) @@ -442,3 +467,196 @@ func (_ *testDecodeSuite) TestParseJson(c *C) { c.Assert(rows.Rows[0][1], Equals, float64(101)) } } +func (_ *testDecodeSuite) TestParseJsonDecimal(c *C) { + // Table format: + // mysql> desc t10; + // +-------+---------------+------+-----+---------+-------+ + // | Field | Type | Null | Key | Default | Extra | + // +-------+---------------+------+-----+---------+-------+ + // | c1 | json | YES | | NULL | | + // | c2 | decimal(10,0) | YES | | NULL | | + // +-------+---------------+------+-----+---------+-------+ + + // CREATE TABLE `t10` ( + // `c1` json DEFAULT NULL, + // `c2` decimal(10,0) + // ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + // INSERT INTO `t10` (`c2`) VALUES (1); + // INSERT INTO `t10` (`c1`, `c2`) VALUES ('{"key1": "value1", "key2": "value2"}', 1); + // test json deserialization + // INSERT INTO `t10`(`c1`,`c2`) VALUES ('{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}',101); + tableMapEventData := []byte("m\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x03t10\x00\x02\xf5\xf6\x03\x04\n\x00\x03") + + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tableMapEventData) + c.Assert(err, IsNil) + + rows := RowsEvent{useDecimal: true} + rows.tableIDSize = 6 + rows.tables = make(map[uint64]*TableMapEvent) + rows.tables[tableMapEvent.TableID] = tableMapEvent + rows.Version = 2 + + tbls := [][]byte{ + []byte("m\x00\x00\x00\x00\x00\x01\x00\x02\x00\x02\xff\xfd\x80\x00\x00\x00\x01"), + []byte("m\x00\x00\x00\x00\x00\x01\x00\x02\x00\x02\xff\xfc)\x00\x00\x00\x00\x02\x00(\x00\x12\x00\x04\x00\x16\x00\x04\x00\f\x1a\x00\f!\x00key1key2\x06value1\x06value2\x80\x00\x00\x00\x01"), + } + + for _, tbl := range tbls { + rows.Rows = nil + err = rows.Decode(tbl) + c.Assert(err, IsNil) + c.Assert(rows.Rows[0][1], SimpleDecimalEqualsChecker, decimal.NewFromFloat(1)) + } + + longTbls := [][]byte{ + []byte("m\x00\x00\x00\x00\x00\x01\x00\x02\x00\x02\xff\xfc\xd0\n\x00\x00\x00\x01\x00\xcf\n\v\x00\x04\x00\f\x0f\x00text\xbe\x15Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac\x80\x00\x00\x00e"), + } + + for _, ltbl := range longTbls { + rows.Rows = nil + err = rows.Decode(ltbl) + c.Assert(err, IsNil) + c.Assert(rows.Rows[0][1], SimpleDecimalEqualsChecker, decimal.NewFromFloat(101)) + } +} + +func (_ *testDecodeSuite) TestEnum(c *C) { + // mysql> desc aenum; + // +-------+-------------------------------------------+------+-----+---------+-------+ + // | Field | Type | Null | Key | Default | Extra | + // +-------+-------------------------------------------+------+-----+---------+-------+ + // | id | int(11) | YES | | NULL | | + // | aset | enum('0','1','2','3','4','5','6','7','8') | YES | | NULL | | + // +-------+-------------------------------------------+------+-----+---------+-------+ + // 2 rows in set (0.00 sec) + // + // insert into aenum(id, aset) values(1, '0'); + tableMapEventData := []byte("\x42\x0f\x00\x00\x00\x00\x01\x00\x05\x74\x74\x65\x73\x74\x00\x05") + tableMapEventData = append(tableMapEventData, []byte("\x61\x65\x6e\x75\x6d\x00\x02\x03\xfe\x02\xf7\x01\x03")...) + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tableMapEventData) + c.Assert(err, IsNil) + + rows := new(RowsEvent) + rows.tableIDSize = 6 + rows.tables = make(map[uint64]*TableMapEvent) + rows.tables[tableMapEvent.TableID] = tableMapEvent + rows.Version = 2 + + data := []byte("\x42\x0f\x00\x00\x00\x00\x01\x00\x02\x00\x02\xff\xfc\x01\x00\x00\x00\x01") + + rows.Rows = nil + err = rows.Decode(data) + c.Assert(err, IsNil) + c.Assert(rows.Rows[0][1], Equals, int64(1)) +} + +func (_ *testDecodeSuite) TestMultiBytesEnum(c *C) { + // CREATE TABLE numbers ( + // id int auto_increment, + // num ENUM( '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36', '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48', '49', '50', '51', '52', '53', '54', '55', '56', '57', '58', '59', '60', '61', '62', '63', '64', '65', '66', '67', '68', '69', '70', '71', '72', '73', '74', '75', '76', '77', '78', '79', '80', '81', '82', '83', '84', '85', '86', '87', '88', '89', '90', '91', '92', '93', '94', '95', '96', '97', '98', '99', '100', '101', '102', '103', '104', '105', '106', '107', '108', '109', '110', '111', '112', '113', '114', '115', '116', '117', '118', '119', '120', '121', '122', '123', '124', '125', '126', '127', '128', '129', '130', '131', '132', '133', '134', '135', '136', '137', '138', '139', '140', '141', '142', '143', '144', '145', '146', '147', '148', '149', '150', '151', '152', '153', '154', '155', '156', '157', '158', '159', '160', '161', '162', '163', '164', '165', '166', '167', '168', '169', '170', '171', '172', '173', '174', '175', '176', '177', '178', '179', '180', '181', '182', '183', '184', '185', '186', '187', '188', '189', '190', '191', '192', '193', '194', '195', '196', '197', '198', '199', '200', '201', '202', '203', '204', '205', '206', '207', '208', '209', '210', '211', '212', '213', '214', '215', '216', '217', '218', '219', '220', '221', '222', '223', '224', '225', '226', '227', '228', '229', '230', '231', '232', '233', '234', '235', '236', '237', '238', '239', '240', '241', '242', '243', '244', '245', '246', '247', '248', '249', '250', '251', '252', '253', '254', '255','256','257' + + // ), + // primary key(id) + // ); + + // + // insert into numbers(num) values ('0'), ('256'); + tableMapEventData := []byte("\x84\x0f\x00\x00\x00\x00\x01\x00\x05\x74\x74\x65\x73\x74\x00\x07") + tableMapEventData = append(tableMapEventData, []byte("\x6e\x75\x6d\x62\x65\x72\x73\x00\x02\x03\xfe\x02\xf7\x02\x02")...) + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tableMapEventData) + c.Assert(err, IsNil) + + rows := new(RowsEvent) + rows.tableIDSize = 6 + rows.tables = make(map[uint64]*TableMapEvent) + rows.tables[tableMapEvent.TableID] = tableMapEvent + rows.Version = 2 + + data := []byte("\x84\x0f\x00\x00\x00\x00\x01\x00\x02\x00\x02\xff\xfc\x01\x00\x00\x00\x01\x00\xfc\x02\x00\x00\x00\x01\x01") + + rows.Rows = nil + err = rows.Decode(data) + c.Assert(err, IsNil) + c.Assert(rows.Rows[0][1], Equals, int64(1)) + c.Assert(rows.Rows[1][1], Equals, int64(257)) +} + +func (_ *testDecodeSuite) TestSet(c *C) { + // mysql> desc aset; + // +--------+---------------------------------------------------------------------------------------+------+-----+---------+-------+ + // | Field | Type | Null | Key | Default | Extra | + // +--------+---------------------------------------------------------------------------------------+------+-----+---------+-------+ + // | id | int(11) | YES | | NULL | | + // | region | set('1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18') | YES | | NULL | | + // +--------+---------------------------------------------------------------------------------------+------+-----+---------+-------+ + // 2 rows in set (0.00 sec) + // + // insert into aset(id, region) values(1, '1,3'); + + tableMapEventData := []byte("\xe7\x0e\x00\x00\x00\x00\x01\x00\x05\x74\x74\x65\x73\x74\x00\x04") + tableMapEventData = append(tableMapEventData, []byte("\x61\x73\x65\x74\x00\x02\x03\xfe\x02\xf8\x03\x03")...) + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tableMapEventData) + c.Assert(err, IsNil) + + rows := new(RowsEvent) + rows.tableIDSize = 6 + rows.tables = make(map[uint64]*TableMapEvent) + rows.tables[tableMapEvent.TableID] = tableMapEvent + rows.Version = 2 + + data := []byte("\xe7\x0e\x00\x00\x00\x00\x01\x00\x02\x00\x02\xff\xfc\x01\x00\x00\x00\x05\x00\x00") + + rows.Rows = nil + err = rows.Decode(data) + c.Assert(err, IsNil) + c.Assert(rows.Rows[0][1], Equals, int64(5)) +} + +func (_ *testDecodeSuite) TestJsonNull(c *C) { + // Table: + // desc hj_order_preview + // +------------------+------------+------+-----+-------------------+----------------+ + // | Field | Type | Null | Key | Default | Extra | + // +------------------+------------+------+-----+-------------------+----------------+ + // | id | int(13) | NO | PRI | | auto_increment | + // | buyer_id | bigint(13) | NO | | | | + // | order_sn | bigint(13) | NO | | | | + // | order_detail | json | NO | | | | + // | is_del | tinyint(1) | NO | | 0 | | + // | add_time | int(13) | NO | | | | + // | last_update_time | timestamp | NO | | CURRENT_TIMESTAMP | | + // +------------------+------------+------+-----+-------------------+----------------+ + // insert into hj_order_preview + // (id, buyer_id, order_sn, is_del, add_time, last_update_time) + // values (1, 95891865464386, 13376222192996417, 0, 1479983995, 1479983995) + + tableMapEventData := []byte("r\x00\x00\x00\x00\x00\x01\x00\x04test\x00\x10hj_order_preview\x00\a\x03\b\b\xf5\x01\x03\x11\x02\x04\x00\x00") + + tableMapEvent := new(TableMapEvent) + tableMapEvent.tableIDSize = 6 + err := tableMapEvent.Decode(tableMapEventData) + c.Assert(err, IsNil) + + rows := new(RowsEvent) + rows.tableIDSize = 6 + rows.tables = make(map[uint64]*TableMapEvent) + rows.tables[tableMapEvent.TableID] = tableMapEvent + rows.Version = 2 + + data := + []byte("r\x00\x00\x00\x00\x00\x01\x00\x02\x00\a\xff\x80\x01\x00\x00\x00B\ue4d06W\x00\x00A\x10@l\x9a\x85/\x00\x00\x00\x00\x00\x00{\xc36X\x00\x00\x00\x00") + + rows.Rows = nil + err = rows.Decode(data) + c.Assert(err, IsNil) + c.Assert(rows.Rows[0][3], HasLen, 0) +} diff --git a/vendor/github.com/siddontang/go-mysql/schema/schema.go b/vendor/github.com/siddontang/go-mysql/schema/schema.go index 86d2128..c98b9ac 100644 --- a/vendor/github.com/siddontang/go-mysql/schema/schema.go +++ b/vendor/github.com/siddontang/go-mysql/schema/schema.go @@ -5,6 +5,7 @@ package schema import ( + "database/sql" "fmt" "strings" @@ -12,6 +13,11 @@ import ( "github.com/siddontang/go-mysql/mysql" ) +var ErrTableNotExist = errors.New("table is not exist") +var ErrMissingTableMeta = errors.New("missing table meta") +var HAHealthCheckSchema = "mysql.ha_health_check" + +// Different column type const ( TYPE_NUMBER = iota + 1 // tinyint, smallint, mediumint, int, bigint, year TYPE_FLOAT // float, double @@ -24,12 +30,16 @@ const ( TYPE_TIME // time TYPE_BIT // bit TYPE_JSON // json + TYPE_DECIMAL // decimal ) type TableColumn struct { Name string Type int + Collation string + RawType string IsAuto bool + IsUnsigned bool EnumValues []string SetValues []string } @@ -47,22 +57,24 @@ type Table struct { Columns []TableColumn Indexes []*Index PKColumns []int + + UnsignedColumns []int } func (ta *Table) String() string { return fmt.Sprintf("%s.%s", ta.Schema, ta.Name) } -func (ta *Table) AddColumn(name string, columnType string, extra string) { +func (ta *Table) AddColumn(name string, columnType string, collation string, extra string) { index := len(ta.Columns) - ta.Columns = append(ta.Columns, TableColumn{Name: name}) + ta.Columns = append(ta.Columns, TableColumn{Name: name, Collation: collation}) + ta.Columns[index].RawType = columnType - if strings.Contains(columnType, "int") || strings.HasPrefix(columnType, "year") { - ta.Columns[index].Type = TYPE_NUMBER - } else if strings.HasPrefix(columnType, "float") || - strings.HasPrefix(columnType, "double") || - strings.HasPrefix(columnType, "decimal") { + if strings.HasPrefix(columnType, "float") || + strings.HasPrefix(columnType, "double") { ta.Columns[index].Type = TYPE_FLOAT + } else if strings.HasPrefix(columnType, "decimal") { + ta.Columns[index].Type = TYPE_DECIMAL } else if strings.HasPrefix(columnType, "enum") { ta.Columns[index].Type = TYPE_ENUM ta.Columns[index].EnumValues = strings.Split(strings.Replace( @@ -93,10 +105,17 @@ func (ta *Table) AddColumn(name string, columnType string, extra string) { ta.Columns[index].Type = TYPE_BIT } else if strings.HasPrefix(columnType, "json") { ta.Columns[index].Type = TYPE_JSON + } else if strings.Contains(columnType, "int") || strings.HasPrefix(columnType, "year") { + ta.Columns[index].Type = TYPE_NUMBER } else { ta.Columns[index].Type = TYPE_STRING } + if strings.Contains(columnType, "unsigned") || strings.Contains(columnType, "zerofill") { + ta.Columns[index].IsUnsigned = true + ta.UnsignedColumns = append(ta.UnsignedColumns, index) + } + if extra == "auto_increment" { ta.Columns[index].IsAuto = true } @@ -142,6 +161,35 @@ func (idx *Index) FindColumn(name string) int { return -1 } +func IsTableExist(conn mysql.Executer, schema string, name string) (bool, error) { + query := fmt.Sprintf("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' and TABLE_NAME = '%s' LIMIT 1", schema, name) + r, err := conn.Execute(query) + if err != nil { + return false, errors.Trace(err) + } + + return r.RowNumber() == 1, nil +} + +func NewTableFromSqlDB(conn *sql.DB, schema string, name string) (*Table, error) { + ta := &Table{ + Schema: schema, + Name: name, + Columns: make([]TableColumn, 0, 16), + Indexes: make([]*Index, 0, 8), + } + + if err := ta.fetchColumnsViaSqlDB(conn); err != nil { + return nil, errors.Trace(err) + } + + if err := ta.fetchIndexesViaSqlDB(conn); err != nil { + return nil, errors.Trace(err) + } + + return ta, nil +} + func NewTable(conn mysql.Executer, schema string, name string) (*Table, error) { ta := &Table{ Schema: schema, @@ -151,18 +199,18 @@ func NewTable(conn mysql.Executer, schema string, name string) (*Table, error) { } if err := ta.fetchColumns(conn); err != nil { - return nil, err + return nil, errors.Trace(err) } if err := ta.fetchIndexes(conn); err != nil { - return nil, err + return nil, errors.Trace(err) } return ta, nil } func (ta *Table) fetchColumns(conn mysql.Executer) error { - r, err := conn.Execute(fmt.Sprintf("describe `%s`.`%s`", ta.Schema, ta.Name)) + r, err := conn.Execute(fmt.Sprintf("show full columns from `%s`.`%s`", ta.Schema, ta.Name)) if err != nil { return errors.Trace(err) } @@ -170,14 +218,39 @@ func (ta *Table) fetchColumns(conn mysql.Executer) error { for i := 0; i < r.RowNumber(); i++ { name, _ := r.GetString(i, 0) colType, _ := r.GetString(i, 1) - extra, _ := r.GetString(i, 5) + collation, _ := r.GetString(i, 2) + extra, _ := r.GetString(i, 6) - ta.AddColumn(name, colType, extra) + ta.AddColumn(name, colType, collation, extra) } return nil } +func (ta *Table) fetchColumnsViaSqlDB(conn *sql.DB) error { + r, err := conn.Query(fmt.Sprintf("show full columns from `%s`.`%s`", ta.Schema, ta.Name)) + if err != nil { + return errors.Trace(err) + } + + defer r.Close() + + var unusedVal interface{} + unused := &unusedVal + + for r.Next() { + var name, colType, extra string + var collation sql.NullString + err := r.Scan(&name, &colType, &collation, &unused, &unused, &unused, &extra, &unused, &unused) + if err != nil { + return errors.Trace(err) + } + ta.AddColumn(name, colType, collation.String, extra) + } + + return r.Err() +} + func (ta *Table) fetchIndexes(conn mysql.Executer) error { r, err := conn.Execute(fmt.Sprintf("show index from `%s`.`%s`", ta.Schema, ta.Name)) if err != nil { @@ -197,6 +270,87 @@ func (ta *Table) fetchIndexes(conn mysql.Executer) error { currentIndex.AddColumn(colName, cardinality) } + return ta.fetchPrimaryKeyColumns() + +} + +func (ta *Table) fetchIndexesViaSqlDB(conn *sql.DB) error { + r, err := conn.Query(fmt.Sprintf("show index from `%s`.`%s`", ta.Schema, ta.Name)) + if err != nil { + return errors.Trace(err) + } + + defer r.Close() + + var currentIndex *Index + currentName := "" + + var unusedVal interface{} + unused := &unusedVal + + for r.Next() { + var indexName, colName string + var cardinality interface{} + + err := r.Scan( + &unused, + &unused, + &indexName, + &unused, + &colName, + &unused, + &cardinality, + &unused, + &unused, + &unused, + &unused, + &unused, + &unused, + ) + if err != nil { + return errors.Trace(err) + } + + if currentName != indexName { + currentIndex = ta.AddIndex(indexName) + currentName = indexName + } + + c := toUint64(cardinality) + currentIndex.AddColumn(colName, c) + } + + return ta.fetchPrimaryKeyColumns() +} + +func toUint64(i interface{}) uint64 { + switch i := i.(type) { + case int: + return uint64(i) + case int8: + return uint64(i) + case int16: + return uint64(i) + case int32: + return uint64(i) + case int64: + return uint64(i) + case uint: + return uint64(i) + case uint8: + return uint64(i) + case uint16: + return uint64(i) + case uint32: + return uint64(i) + case uint64: + return uint64(i) + } + + return 0 +} + +func (ta *Table) fetchPrimaryKeyColumns() error { if len(ta.Indexes) == 0 { return nil } @@ -213,3 +367,32 @@ func (ta *Table) fetchIndexes(conn mysql.Executer) error { return nil } + +// Get primary keys in one row for a table, a table may use multi fields as the PK +func (ta *Table) GetPKValues(row []interface{}) ([]interface{}, error) { + indexes := ta.PKColumns + if len(indexes) == 0 { + return nil, errors.Errorf("table %s has no PK", ta) + } else if len(ta.Columns) != len(row) { + return nil, errors.Errorf("table %s has %d columns, but row data %v len is %d", ta, + len(ta.Columns), row, len(row)) + } + + values := make([]interface{}, 0, len(indexes)) + + for _, index := range indexes { + values = append(values, row[index]) + } + + return values, nil +} + +// Get term column's value +func (ta *Table) GetColumnValue(column string, row []interface{}) (interface{}, error) { + index := ta.FindColumn(column) + if index == -1 { + return nil, errors.Errorf("table %s has no column name %s", ta, column) + } + + return row[index], nil +} diff --git a/vendor/github.com/siddontang/go-mysql/schema/schema_test.go b/vendor/github.com/siddontang/go-mysql/schema/schema_test.go index 327c622..c5bafe1 100644 --- a/vendor/github.com/siddontang/go-mysql/schema/schema_test.go +++ b/vendor/github.com/siddontang/go-mysql/schema/schema_test.go @@ -1,12 +1,14 @@ package schema import ( + "database/sql" "flag" "fmt" "testing" . "github.com/pingcap/check" "github.com/siddontang/go-mysql/client" + _ "github.com/siddontang/go-mysql/driver" ) // use docker mysql for test @@ -17,7 +19,8 @@ func Test(t *testing.T) { } type schemaTestSuite struct { - conn *client.Conn + conn *client.Conn + sqlDB *sql.DB } var _ = Suite(&schemaTestSuite{}) @@ -26,12 +29,19 @@ func (s *schemaTestSuite) SetUpSuite(c *C) { var err error s.conn, err = client.Connect(fmt.Sprintf("%s:%d", *host, 3306), "root", "", "test") c.Assert(err, IsNil) + + s.sqlDB, err = sql.Open("mysql", fmt.Sprintf("root:@%s:3306", *host)) + c.Assert(err, IsNil) } func (s *schemaTestSuite) TearDownSuite(c *C) { if s.conn != nil { s.conn.Close() } + + if s.sqlDB != nil { + s.sqlDB.Close() + } } func (s *schemaTestSuite) TestSchema(c *C) { @@ -44,10 +54,14 @@ func (s *schemaTestSuite) TestSchema(c *C) { id1 INT, id2 INT, name VARCHAR(256), - e ENUM("a", "b", "c"), + status ENUM('appointing','serving','abnormal','stop','noaftermarket','finish','financial_audit'), se SET('a', 'b', 'c'), f FLOAT, d DECIMAL(2, 1), + uint INT UNSIGNED, + zfint INT ZEROFILL, + name_ucs VARCHAR(256) CHARACTER SET ucs2, + name_utf8 VARCHAR(256) CHARACTER SET utf8, PRIMARY KEY(id2, id), UNIQUE (id1), INDEX name_idx (name) @@ -60,15 +74,25 @@ func (s *schemaTestSuite) TestSchema(c *C) { ta, err := NewTable(s.conn, "test", "schema_test") c.Assert(err, IsNil) - c.Assert(ta.Columns, HasLen, 8) + c.Assert(ta.Columns, HasLen, 12) c.Assert(ta.Indexes, HasLen, 3) c.Assert(ta.PKColumns, DeepEquals, []int{2, 0}) c.Assert(ta.Indexes[0].Columns, HasLen, 2) c.Assert(ta.Indexes[0].Name, Equals, "PRIMARY") c.Assert(ta.Indexes[2].Name, Equals, "name_idx") - c.Assert(ta.Columns[4].EnumValues, DeepEquals, []string{"a", "b", "c"}) + c.Assert(ta.Columns[4].EnumValues, DeepEquals, []string{"appointing", "serving", "abnormal", "stop", "noaftermarket", "finish", "financial_audit"}) c.Assert(ta.Columns[5].SetValues, DeepEquals, []string{"a", "b", "c"}) - c.Assert(ta.Columns[7].Type, Equals, TYPE_FLOAT) + c.Assert(ta.Columns[7].Type, Equals, TYPE_DECIMAL) + c.Assert(ta.Columns[0].IsUnsigned, IsFalse) + c.Assert(ta.Columns[8].IsUnsigned, IsTrue) + c.Assert(ta.Columns[9].IsUnsigned, IsTrue) + c.Assert(ta.Columns[10].Collation, Matches, "^ucs2.*") + c.Assert(ta.Columns[11].Collation, Matches, "^utf8.*") + + taSqlDb, err := NewTableFromSqlDB(s.sqlDB, "test", "schema_test") + c.Assert(err, IsNil) + + c.Assert(taSqlDb, DeepEquals, ta) } func (s *schemaTestSuite) TestQuoteSchema(c *C) { diff --git a/vendor/github.com/siddontang/go-mysql/server/auth.go b/vendor/github.com/siddontang/go-mysql/server/auth.go index b66ea4e..0eb54a6 100644 --- a/vendor/github.com/siddontang/go-mysql/server/auth.go +++ b/vendor/github.com/siddontang/go-mysql/server/auth.go @@ -2,118 +2,173 @@ package server import ( "bytes" - "encoding/binary" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/tls" + "fmt" + "github.com/juju/errors" . "github.com/siddontang/go-mysql/mysql" ) -func (c *Conn) writeInitialHandshake() error { - capability := CLIENT_LONG_PASSWORD | CLIENT_LONG_FLAG | - CLIENT_CONNECT_WITH_DB | CLIENT_PROTOCOL_41 | - CLIENT_TRANSACTIONS | CLIENT_SECURE_CONNECTION +var ErrAccessDenied = errors.New("access denied") - data := make([]byte, 4, 128) +func (c *Conn) compareAuthData(authPluginName string, clientAuthData []byte) error { + switch authPluginName { + case AUTH_NATIVE_PASSWORD: + if err := c.acquirePassword(); err != nil { + return err + } + return c.compareNativePasswordAuthData(clientAuthData, c.password) - //min version 10 - data = append(data, 10) + case AUTH_CACHING_SHA2_PASSWORD: + if err := c.compareCacheSha2PasswordAuthData(clientAuthData); err != nil { + return err + } + if c.cachingSha2FullAuth { + return c.handleAuthSwitchResponse() + } + return nil - //server version[00] - data = append(data, ServerVersion...) - data = append(data, 0) + case AUTH_SHA256_PASSWORD: + if err := c.acquirePassword(); err != nil { + return err + } + cont, err := c.handlePublicKeyRetrieval(clientAuthData) + if err != nil { + return err + } + if !cont { + return nil + } + return c.compareSha256PasswordAuthData(clientAuthData, c.password) - //connection id - data = append(data, byte(c.connectionID), byte(c.connectionID>>8), byte(c.connectionID>>16), byte(c.connectionID>>24)) - - //auth-plugin-data-part-1 - data = append(data, c.salt[0:8]...) - - //filter [00] - data = append(data, 0) - - //capability flag lower 2 bytes, using default capability here - data = append(data, byte(capability), byte(capability>>8)) - - //charset, utf-8 default - data = append(data, uint8(DEFAULT_COLLATION_ID)) - - //status - data = append(data, byte(c.status), byte(c.status>>8)) - - //below 13 byte may not be used - //capability flag upper 2 bytes, using default capability here - data = append(data, byte(capability>>16), byte(capability>>24)) - - //filter [0x15], for wireshark dump, value is 0x15 - data = append(data, 0x15) - - //reserved 10 [00] - data = append(data, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - - //auth-plugin-data-part-2 - data = append(data, c.salt[8:]...) - - //filter [00] - data = append(data, 0) - - return c.WritePacket(data) + default: + return errors.Errorf("unknown authentication plugin name '%s'", authPluginName) + } } -func (c *Conn) readHandshakeResponse(password string) error { - data, err := c.ReadPacket() - +func (c *Conn) acquirePassword() error { + password, found, err := c.credentialProvider.GetCredential(c.user) if err != nil { return err } - - pos := 0 - - //capability - c.capability = binary.LittleEndian.Uint32(data[:4]) - pos += 4 - - //skip max packet size - pos += 4 - - //charset, skip, if you want to use another charset, use set names - //c.collation = CollationId(data[pos]) - pos++ - - //skip reserved 23[00] - pos += 23 - - //user name - user := string(data[pos : pos+bytes.IndexByte(data[pos:], 0)]) - pos += len(user) + 1 - - if c.user != user { - return NewDefaultError(ER_NO_SUCH_USER, user, c.RemoteAddr().String()) + if !found { + return NewDefaultError(ER_NO_SUCH_USER, c.user, c.RemoteAddr().String()) } - - //auth length and auth - authLen := int(data[pos]) - pos++ - auth := data[pos : pos+authLen] - - checkAuth := CalcPassword(c.salt, []byte(password)) - - if !bytes.Equal(auth, checkAuth) { - return NewDefaultError(ER_ACCESS_DENIED_ERROR, c.RemoteAddr().String(), c.user, "Yes") - } - - pos += authLen - - if c.capability|CLIENT_CONNECT_WITH_DB > 0 { - if len(data[pos:]) == 0 { - return nil - } - - db := string(data[pos : pos+bytes.IndexByte(data[pos:], 0)]) - pos += len(db) + 1 - - if err = c.h.UseDB(db); err != nil { - return err - } - } - + c.password = password + return nil +} + +func scrambleValidation(cached, nonce, scramble []byte) bool { + // SHA256(SHA256(SHA256(STORED_PASSWORD)), NONCE) + crypt := sha256.New() + crypt.Write(cached) + crypt.Write(nonce) + message2 := crypt.Sum(nil) + // SHA256(PASSWORD) + if len(message2) != len(scramble) { + return false + } + for i := range message2 { + message2[i] ^= scramble[i] + } + // SHA256(SHA256(PASSWORD) + crypt.Reset() + crypt.Write(message2) + m := crypt.Sum(nil) + return bytes.Equal(m, cached) +} + +func (c *Conn) compareNativePasswordAuthData(clientAuthData []byte, password string) error { + if bytes.Equal(CalcPassword(c.salt, []byte(c.password)), clientAuthData) { + return nil + } + return ErrAccessDenied +} + +func (c *Conn) compareSha256PasswordAuthData(clientAuthData []byte, password string) error { + // Empty passwords are not hashed, but sent as empty string + if len(clientAuthData) == 0 { + if password == "" { + return nil + } + return ErrAccessDenied + } + if tlsConn, ok := c.Conn.Conn.(*tls.Conn); ok { + if !tlsConn.ConnectionState().HandshakeComplete { + return errors.New("incomplete TSL handshake") + } + // connection is SSL/TLS, client should send plain password + // deal with the trailing \NUL added for plain text password received + if l := len(clientAuthData); l != 0 && clientAuthData[l-1] == 0x00 { + clientAuthData = clientAuthData[:l-1] + } + if bytes.Equal(clientAuthData, []byte(password)) { + return nil + } + return ErrAccessDenied + } else { + // client should send encrypted password + // decrypt + dbytes, err := rsa.DecryptOAEP(sha1.New(), rand.Reader, (c.serverConf.tlsConfig.Certificates[0].PrivateKey).(*rsa.PrivateKey), clientAuthData, nil) + if err != nil { + return err + } + plain := make([]byte, len(password)+1) + copy(plain, password) + for i := range plain { + j := i % len(c.salt) + plain[i] ^= c.salt[j] + } + if bytes.Equal(plain, dbytes) { + return nil + } + return ErrAccessDenied + } +} + +func (c *Conn) compareCacheSha2PasswordAuthData(clientAuthData []byte) error { + // Empty passwords are not hashed, but sent as empty string + if len(clientAuthData) == 0 { + if err := c.acquirePassword(); err != nil { + return err + } + if c.password == "" { + return nil + } + return ErrAccessDenied + } + // the caching of 'caching_sha2_password' in MySQL, see: https://dev.mysql.com/worklog/task/?id=9591 + if _, ok := c.credentialProvider.(*InMemoryProvider); ok { + // since we have already kept the password in memory and calculate the scramble is not that high of cost, we eliminate + // the caching part. So our server will never ask the client to do a full authentication via RSA key exchange and it appears + // like the auth will always hit the cache. + if err := c.acquirePassword(); err != nil { + return err + } + if bytes.Equal(CalcCachingSha2Password(c.salt, c.password), clientAuthData) { + // 'fast' auth: write "More data" packet (first byte == 0x01) with the second byte = 0x03 + return c.writeAuthMoreDataFastAuth() + } + return ErrAccessDenied + } + // other type of credential provider, we use the cache + cached, ok := c.serverConf.cacheShaPassword.Load(fmt.Sprintf("%s@%s", c.user, c.Conn.LocalAddr())) + if ok { + // Scramble validation + if scrambleValidation(cached.([]byte), c.salt, clientAuthData) { + // 'fast' auth: write "More data" packet (first byte == 0x01) with the second byte = 0x03 + return c.writeAuthMoreDataFastAuth() + } + return ErrAccessDenied + } + // cache miss, do full auth + if err := c.writeAuthMoreDataFullAuth(); err != nil { + return err + } + c.cachingSha2FullAuth = true return nil } diff --git a/vendor/github.com/siddontang/go-mysql/server/command.go b/vendor/github.com/siddontang/go-mysql/server/command.go index fb7dcdf..6c8d13a 100644 --- a/vendor/github.com/siddontang/go-mysql/server/command.go +++ b/vendor/github.com/siddontang/go-mysql/server/command.go @@ -11,8 +11,8 @@ import ( type Handler interface { //handle COM_INIT_DB command, you can check whether the dbName is valid, or other. UseDB(dbName string) error - //handle COM_QUERY comamnd, like SELECT, INSERT, UPDATE, etc... - //If Result has a Resultset (SELECT, SHOW, etc...), we will send this as the repsonse, otherwise, we will send Result + //handle COM_QUERY command, like SELECT, INSERT, UPDATE, etc... + //If Result has a Resultset (SELECT, SHOW, etc...), we will send this as the response, otherwise, we will send Result HandleQuery(query string) (*Result, error) //handle COM_FILED_LIST command HandleFieldList(table string, fieldWildcard string) ([]*Field, error) @@ -25,6 +25,9 @@ type Handler interface { //handle COM_STMT_CLOSE, context is the previous one set in prepare //this handler has no response HandleStmtClose(context interface{}) error + //handle any other command that is not currently handled by the library, + //default implementation for this method will return an ER_UNKNOWN_ERROR + HandleOtherCommand(cmd byte, data []byte) error } func (c *Conn) HandleCommand() error { @@ -119,8 +122,7 @@ func (c *Conn) dispatch(data []byte) interface{} { return r } default: - msg := fmt.Sprintf("command %d is not supported now", cmd) - return NewError(ER_UNKNOWN_ERROR, msg) + return c.h.HandleOtherCommand(cmd, data) } return fmt.Errorf("command %d is not handled correctly", cmd) @@ -149,3 +151,10 @@ func (h EmptyHandler) HandleStmtExecute(context interface{}, query string, args func (h EmptyHandler) HandleStmtClose(context interface{}) error { return nil } + +func (h EmptyHandler) HandleOtherCommand(cmd byte, data []byte) error { + return NewError( + ER_UNKNOWN_ERROR, + fmt.Sprintf("command %d is not supported now", cmd), + ) +} diff --git a/vendor/github.com/siddontang/go-mysql/server/conn.go b/vendor/github.com/siddontang/go-mysql/server/conn.go index d6ea846..a279b93 100644 --- a/vendor/github.com/siddontang/go-mysql/server/conn.go +++ b/vendor/github.com/siddontang/go-mysql/server/conn.go @@ -15,15 +15,17 @@ import ( type Conn struct { *packet.Conn - capability uint32 + serverConf *Server + capability uint32 + authPluginName string + connectionID uint32 + status uint16 + salt []byte // should be 8 + 12 for auth-plugin-data-part-1 and auth-plugin-data-part-2 - connectionID uint32 - - status uint16 - - user string - - salt []byte + credentialProvider CredentialProvider + user string + password string + cachingSha2FullAuth bool h Handler @@ -35,23 +37,23 @@ type Conn struct { var baseConnID uint32 = 10000 +// create connection with default server settings func NewConn(conn net.Conn, user string, password string, h Handler) (*Conn, error) { - c := new(Conn) - - c.h = h - - c.user = user - c.Conn = packet.NewConn(conn) - - c.connectionID = atomic.AddUint32(&baseConnID, 1) - - c.stmts = make(map[uint32]*Stmt) - - c.salt, _ = RandomBuf(20) - + p := NewInMemoryProvider() + p.AddUser(user, password) + salt, _ := RandomBuf(20) + c := &Conn{ + Conn: packet.NewConn(conn), + serverConf: defaultServer, + credentialProvider: p, + h: h, + connectionID: atomic.AddUint32(&baseConnID, 1), + stmts: make(map[uint32]*Stmt), + salt: salt, + } c.closed.Set(false) - if err := c.handshake(password); err != nil { + if err := c.handshake(); err != nil { c.Close() return nil, err } @@ -59,14 +61,38 @@ func NewConn(conn net.Conn, user string, password string, h Handler) (*Conn, err return c, nil } -func (c *Conn) handshake(password string) error { +// create connection with customized server settings +func NewCustomizedConn(conn net.Conn, serverConf *Server, p CredentialProvider, h Handler) (*Conn, error) { + salt, _ := RandomBuf(20) + c := &Conn{ + Conn: packet.NewConn(conn), + serverConf: serverConf, + credentialProvider: p, + h: h, + connectionID: atomic.AddUint32(&baseConnID, 1), + stmts: make(map[uint32]*Stmt), + salt: salt, + } + c.closed.Set(false) + + if err := c.handshake(); err != nil { + c.Close() + return nil, err + } + + return c, nil +} + +func (c *Conn) handshake() error { if err := c.writeInitialHandshake(); err != nil { return err } - if err := c.readHandshakeResponse(password); err != nil { + if err := c.readHandshakeResponse(); err != nil { + if err == ErrAccessDenied { + err = NewDefaultError(ER_ACCESS_DENIED_ERROR, c.user, c.LocalAddr().String(), "Yes") + } c.writeError(err) - return err } diff --git a/vendor/github.com/siddontang/go-mysql/server/resp.go b/vendor/github.com/siddontang/go-mysql/server/resp.go index 1123032..db86323 100644 --- a/vendor/github.com/siddontang/go-mysql/server/resp.go +++ b/vendor/github.com/siddontang/go-mysql/server/resp.go @@ -62,6 +62,59 @@ func (c *Conn) writeEOF() error { return c.WritePacket(data) } +// see: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_auth_switch_request.html +func (c *Conn) writeAuthSwitchRequest(newAuthPluginName string) error { + data := make([]byte, 4) + data = append(data, EOF_HEADER) + data = append(data, []byte(newAuthPluginName)...) + data = append(data, 0x00) + rnd, err := RandomBuf(20) + if err != nil { + return err + } + // new auth data + c.salt = rnd + data = append(data, c.salt...) + // the online doc states it's a string.EOF, however, the actual MySQL server add a \NUL to the end, without it, the + // official MySQL client will fail. + data = append(data, 0x00) + return c.WritePacket(data) +} + +// see: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_auth_switch_response.html +func (c *Conn) readAuthSwitchRequestResponse() ([]byte, error) { + data, err := c.ReadPacket() + if err != nil { + return nil, err + } + if len(data) == 1 && data[0] == 0x00 { + // \NUL + return make([]byte, 0), nil + } + return data, nil +} + +func (c *Conn) writeAuthMoreDataPubkey() error { + data := make([]byte, 4) + data = append(data, MORE_DATE_HEADER) + data = append(data, c.serverConf.pubKey...) + return c.WritePacket(data) +} + +func (c *Conn) writeAuthMoreDataFullAuth() error { + data := make([]byte, 4) + data = append(data, MORE_DATE_HEADER) + data = append(data, CACHE_SHA2_FULL_AUTH) + return c.WritePacket(data) +} + +func (c *Conn) writeAuthMoreDataFastAuth() error { + data := make([]byte, 4) + data = append(data, MORE_DATE_HEADER) + data = append(data, CACHE_SHA2_FAST_AUTH) + return c.WritePacket(data) +} + func (c *Conn) writeResultset(r *Resultset) error { columnLen := PutLengthEncodedInt(uint64(len(r.Fields))) diff --git a/vendor/github.com/siddontang/go-mysql/server/server_test.go b/vendor/github.com/siddontang/go-mysql/server/server_test.go index 54bcf05..1f427fd 100644 --- a/vendor/github.com/siddontang/go-mysql/server/server_test.go +++ b/vendor/github.com/siddontang/go-mysql/server/server_test.go @@ -1,6 +1,7 @@ package server import ( + "crypto/tls" "database/sql" "flag" "fmt" @@ -12,110 +13,90 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/juju/errors" . "github.com/pingcap/check" - mysql "github.com/siddontang/go-mysql/mysql" + "github.com/siddontang/go-log/log" + "github.com/siddontang/go-mysql/mysql" + "github.com/siddontang/go-mysql/test_util/test_keys" ) var testAddr = flag.String("addr", "127.0.0.1:4000", "MySQL proxy server address") var testUser = flag.String("user", "root", "MySQL user") -var testPassword = flag.String("pass", "", "MySQL password") +var testPassword = flag.String("pass", "123456", "MySQL password") var testDB = flag.String("db", "test", "MySQL test database") +var tlsConf = NewServerTLSConfig(test_keys.CaPem, test_keys.CertPem, test_keys.KeyPem, tls.VerifyClientCertIfGiven) + +func prepareServerConf() []*Server { + // add default server without TLS + var servers = []*Server{ + // with default TLS + NewDefaultServer(), + // for key exchange, CLIENT_SSL must be enabled for the server and if the connection is not secured with TLS + // server permits MYSQL_NATIVE_PASSWORD only + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_NATIVE_PASSWORD, test_keys.PubPem, tlsConf), + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_NATIVE_PASSWORD, test_keys.PubPem, tlsConf), + // server permits SHA256_PASSWORD only + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_SHA256_PASSWORD, test_keys.PubPem, tlsConf), + // server permits CACHING_SHA2_PASSWORD only + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_CACHING_SHA2_PASSWORD, test_keys.PubPem, tlsConf), + + // test auth switch: server permits SHA256_PASSWORD only but sent different method MYSQL_NATIVE_PASSWORD in handshake response + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_NATIVE_PASSWORD, test_keys.PubPem, tlsConf), + // test auth switch: server permits CACHING_SHA2_PASSWORD only but sent different method MYSQL_NATIVE_PASSWORD in handshake response + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_NATIVE_PASSWORD, test_keys.PubPem, tlsConf), + // test auth switch: server permits CACHING_SHA2_PASSWORD only but sent different method SHA256_PASSWORD in handshake response + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_SHA256_PASSWORD, test_keys.PubPem, tlsConf), + // test auth switch: server permits MYSQL_NATIVE_PASSWORD only but sent different method SHA256_PASSWORD in handshake response + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_SHA256_PASSWORD, test_keys.PubPem, tlsConf), + // test auth switch: server permits SHA256_PASSWORD only but sent different method CACHING_SHA2_PASSWORD in handshake response + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_CACHING_SHA2_PASSWORD, test_keys.PubPem, tlsConf), + // test auth switch: server permits MYSQL_NATIVE_PASSWORD only but sent different method CACHING_SHA2_PASSWORD in handshake response + NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_CACHING_SHA2_PASSWORD, test_keys.PubPem, tlsConf), + } + return servers +} + func Test(t *testing.T) { + log.SetLevel(log.LevelDebug) + + // general tests + inMemProvider := NewInMemoryProvider() + inMemProvider.AddUser(*testUser, *testPassword) + + servers := prepareServerConf() + //no TLS + for _, svr := range servers { + Suite(&serverTestSuite{ + server: svr, + credProvider: inMemProvider, + tlsPara: "false", + }) + } + + // TLS if server supports + for _, svr := range servers { + if svr.tlsConfig != nil { + Suite(&serverTestSuite{ + server: svr, + credProvider: inMemProvider, + tlsPara: "skip-verify", + }) + } + } + TestingT(t) } type serverTestSuite struct { + server *Server + credProvider CredentialProvider + + tlsPara string + db *sql.DB l net.Listener } -var _ = Suite(&serverTestSuite{}) - -type testHandler struct { - s *serverTestSuite -} - -func (h *testHandler) UseDB(dbName string) error { - return nil -} - -func (h *testHandler) handleQuery(query string, binary bool) (*mysql.Result, error) { - ss := strings.Split(query, " ") - switch strings.ToLower(ss[0]) { - case "select": - var r *mysql.Resultset - var err error - //for handle go mysql driver select @@max_allowed_packet - if strings.Contains(strings.ToLower(query), "max_allowed_packet") { - r, err = mysql.BuildSimpleResultset([]string{"@@max_allowed_packet"}, [][]interface{}{ - []interface{}{mysql.MaxPayloadLen}, - }, binary) - } else { - r, err = mysql.BuildSimpleResultset([]string{"a", "b"}, [][]interface{}{ - []interface{}{1, "hello world"}, - }, binary) - } - - if err != nil { - return nil, errors.Trace(err) - } else { - return &mysql.Result{0, 0, 0, r}, nil - } - case "insert": - return &mysql.Result{0, 1, 0, nil}, nil - case "delete": - return &mysql.Result{0, 0, 1, nil}, nil - case "update": - return &mysql.Result{0, 0, 1, nil}, nil - case "replace": - return &mysql.Result{0, 0, 1, nil}, nil - default: - return nil, fmt.Errorf("invalid query %s", query) - } - - return nil, nil -} - -func (h *testHandler) HandleQuery(query string) (*mysql.Result, error) { - return h.handleQuery(query, false) -} - -func (h *testHandler) HandleFieldList(table string, fieldWildcard string) ([]*mysql.Field, error) { - return nil, nil -} -func (h *testHandler) HandleStmtPrepare(sql string) (params int, columns int, ctx interface{}, err error) { - ss := strings.Split(sql, " ") - switch strings.ToLower(ss[0]) { - case "select": - params = 1 - columns = 2 - case "insert": - params = 2 - columns = 0 - case "replace": - params = 2 - columns = 0 - case "update": - params = 1 - columns = 0 - case "delete": - params = 1 - columns = 0 - default: - err = fmt.Errorf("invalid prepare %s", sql) - } - return params, columns, nil, err -} - -func (h *testHandler) HandleStmtClose(context interface{}) error { - return nil -} - -func (h *testHandler) HandleStmtExecute(ctx interface{}, query string, args []interface{}) (*mysql.Result, error) { - return h.handleQuery(query, true) -} - func (s *serverTestSuite) SetUpSuite(c *C) { var err error @@ -124,9 +105,9 @@ func (s *serverTestSuite) SetUpSuite(c *C) { go s.onAccept(c) - time.Sleep(500 * time.Millisecond) + time.Sleep(20 * time.Millisecond) - s.db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s", *testUser, *testPassword, *testAddr, *testDB)) + s.db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?tls=%s", *testUser, *testPassword, *testAddr, *testDB, s.tlsPara)) c.Assert(err, IsNil) s.db.SetMaxIdleConns(4) @@ -154,9 +135,10 @@ func (s *serverTestSuite) onAccept(c *C) { } func (s *serverTestSuite) onConn(conn net.Conn, c *C) { - co, err := NewConn(conn, *testUser, *testPassword, &testHandler{s}) + //co, err := NewConn(conn, *testUser, *testPassword, &testHandler{s}) + co, err := NewCustomizedConn(conn, s.server, s.credProvider, &testHandler{s}) c.Assert(err, IsNil) - + // set SSL if defined for { err = co.HandleCommand() if err != nil { @@ -228,3 +210,91 @@ func (s *serverTestSuite) TestStmtExec(c *C) { i, _ = r.RowsAffected() c.Assert(i, Equals, int64(1)) } + +type testHandler struct { + s *serverTestSuite +} + +func (h *testHandler) UseDB(dbName string) error { + return nil +} + +func (h *testHandler) handleQuery(query string, binary bool) (*mysql.Result, error) { + ss := strings.Split(query, " ") + switch strings.ToLower(ss[0]) { + case "select": + var r *mysql.Resultset + var err error + //for handle go mysql driver select @@max_allowed_packet + if strings.Contains(strings.ToLower(query), "max_allowed_packet") { + r, err = mysql.BuildSimpleResultset([]string{"@@max_allowed_packet"}, [][]interface{}{ + {mysql.MaxPayloadLen}, + }, binary) + } else { + r, err = mysql.BuildSimpleResultset([]string{"a", "b"}, [][]interface{}{ + {1, "hello world"}, + }, binary) + } + + if err != nil { + return nil, errors.Trace(err) + } else { + return &mysql.Result{0, 0, 0, r}, nil + } + case "insert": + return &mysql.Result{0, 1, 0, nil}, nil + case "delete": + return &mysql.Result{0, 0, 1, nil}, nil + case "update": + return &mysql.Result{0, 0, 1, nil}, nil + case "replace": + return &mysql.Result{0, 0, 1, nil}, nil + default: + return nil, fmt.Errorf("invalid query %s", query) + } + + return nil, nil +} + +func (h *testHandler) HandleQuery(query string) (*mysql.Result, error) { + return h.handleQuery(query, false) +} + +func (h *testHandler) HandleFieldList(table string, fieldWildcard string) ([]*mysql.Field, error) { + return nil, nil +} +func (h *testHandler) HandleStmtPrepare(sql string) (params int, columns int, ctx interface{}, err error) { + ss := strings.Split(sql, " ") + switch strings.ToLower(ss[0]) { + case "select": + params = 1 + columns = 2 + case "insert": + params = 2 + columns = 0 + case "replace": + params = 2 + columns = 0 + case "update": + params = 1 + columns = 0 + case "delete": + params = 1 + columns = 0 + default: + err = fmt.Errorf("invalid prepare %s", sql) + } + return params, columns, nil, err +} + +func (h *testHandler) HandleStmtClose(context interface{}) error { + return nil +} + +func (h *testHandler) HandleStmtExecute(ctx interface{}, query string, args []interface{}) (*mysql.Result, error) { + return h.handleQuery(query, true) +} + +func (h *testHandler) HandleOtherCommand(cmd byte, data []byte) error { + return mysql.NewError(mysql.ER_UNKNOWN_ERROR, fmt.Sprintf("command %d is not supported now", cmd)) +} \ No newline at end of file diff --git a/vendor/github.com/siddontang/go-mysql/server/stmt.go b/vendor/github.com/siddontang/go-mysql/server/stmt.go index 7a325d7..9bef23e 100644 --- a/vendor/github.com/siddontang/go-mysql/server/stmt.go +++ b/vendor/github.com/siddontang/go-mysql/server/stmt.go @@ -144,7 +144,7 @@ func (c *Conn) handleStmtExecute(data []byte) (*Result, error) { } paramTypes = data[pos : pos+(paramNum<<1)] - pos += (paramNum << 1) + pos += paramNum << 1 paramValues = data[pos:] } @@ -211,7 +211,7 @@ func (c *Conn) bindStmtArgs(s *Stmt, nullBitmap, paramTypes, paramValues []byte) if isUnsigned { args[i] = uint16(binary.LittleEndian.Uint16(paramValues[pos : pos+2])) } else { - args[i] = int16((binary.LittleEndian.Uint16(paramValues[pos : pos+2]))) + args[i] = int16(binary.LittleEndian.Uint16(paramValues[pos : pos+2])) } pos += 2 continue @@ -270,7 +270,7 @@ func (c *Conn) bindStmtArgs(s *Stmt, nullBitmap, paramTypes, paramValues []byte) return ErrMalformPacket } - v, isNull, n, err = LengthEnodedString(paramValues[pos:]) + v, isNull, n, err = LengthEncodedString(paramValues[pos:]) pos += n if err != nil { return errors.Trace(err) @@ -290,7 +290,7 @@ func (c *Conn) bindStmtArgs(s *Stmt, nullBitmap, paramTypes, paramValues []byte) return nil } -// stmt send long data command has no repsonse +// stmt send long data command has no response func (c *Conn) handleStmtSendLongData(data []byte) error { if len(data) < 6 { return nil @@ -340,7 +340,7 @@ func (c *Conn) handleStmtReset(data []byte) (*Result, error) { return &Result{}, nil } -// stmt close command has no repsonse +// stmt close command has no response func (c *Conn) handleStmtClose(data []byte) error { if len(data) < 4 { return nil From 14e396ed730987069a250b2cce0d9eb6b5a0a147 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 1 Jan 2019 10:58:12 +0200 Subject: [PATCH 083/176] updated go-mysql --- .../github.com/siddontang/go-mysql/Gopkg.lock | 78 + .../github.com/siddontang/go-mysql/Gopkg.toml | 56 + .../siddontang/go-mysql/clear_vendor.sh | 6 + .../siddontang/go-mysql/client/tls.go | 28 + .../go-mysql/docker/docker-compose.yaml | 80 + .../go-mysql/docker/resources/ca.key | 27 + .../go-mysql/docker/resources/ca.pem | 22 + .../go-mysql/docker/resources/client-cert.pem | 19 + .../go-mysql/docker/resources/client-key.pem | 27 + .../go-mysql/docker/resources/server-cert.pem | 19 + .../go-mysql/docker/resources/server-key.pem | 27 + .../go-mysql/mysql/mariadb_gtid_test.go | 234 +++ .../go-mysql/replication/backup_test.go | 50 + .../siddontang/go-mysql/replication/time.go | 49 + .../go-mysql/replication/time_test.go | 70 + .../go-mysql/server/auth_switch_response.go | 133 ++ .../server/caching_sha2_cache_test.go | 233 +++ .../go-mysql/server/command_test.go | 4 + .../go-mysql/server/credential_provider.go | 45 + .../go-mysql/server/example/server_example.go | 51 + .../go-mysql/server/handshake_resp.go | 190 +++ .../go-mysql/server/initial_handshake.go | 57 + .../siddontang/go-mysql/server/server_conf.go | 103 ++ .../siddontang/go-mysql/server/ssl.go | 133 ++ .../go-mysql/test_util/test_keys/keys.go | 85 + .../vendor/github.com/BurntSushi/toml/COPYING | 14 + .../toml/cmd/toml-test-decoder/COPYING | 14 + .../toml/cmd/toml-test-encoder/COPYING | 14 + .../BurntSushi/toml/cmd/tomlv/COPYING | 14 + .../github.com/BurntSushi/toml/decode.go | 509 ++++++ .../github.com/BurntSushi/toml/decode_meta.go | 121 ++ .../vendor/github.com/BurntSushi/toml/doc.go | 27 + .../github.com/BurntSushi/toml/encode.go | 568 +++++++ .../BurntSushi/toml/encoding_types.go | 19 + .../BurntSushi/toml/encoding_types_1.1.go | 18 + .../vendor/github.com/BurntSushi/toml/lex.go | 953 +++++++++++ .../github.com/BurntSushi/toml/parse.go | 592 +++++++ .../github.com/BurntSushi/toml/type_check.go | 91 ++ .../github.com/BurntSushi/toml/type_fields.go | 242 +++ .../github.com/go-sql-driver/mysql/AUTHORS | 90 ++ .../github.com/go-sql-driver/mysql/LICENSE | 373 +++++ .../go-sql-driver/mysql/appengine.go | 19 + .../github.com/go-sql-driver/mysql/auth.go | 420 +++++ .../github.com/go-sql-driver/mysql/buffer.go | 147 ++ .../go-sql-driver/mysql/collations.go | 251 +++ .../go-sql-driver/mysql/connection.go | 654 ++++++++ .../github.com/go-sql-driver/mysql/const.go | 174 ++ .../github.com/go-sql-driver/mysql/driver.go | 165 ++ .../github.com/go-sql-driver/mysql/dsn.go | 611 +++++++ .../github.com/go-sql-driver/mysql/errors.go | 65 + .../github.com/go-sql-driver/mysql/fields.go | 194 +++ .../github.com/go-sql-driver/mysql/infile.go | 182 +++ .../github.com/go-sql-driver/mysql/packets.go | 1298 +++++++++++++++ .../github.com/go-sql-driver/mysql/result.go | 22 + .../github.com/go-sql-driver/mysql/rows.go | 216 +++ .../go-sql-driver/mysql/statement.go | 211 +++ .../go-sql-driver/mysql/transaction.go | 31 + .../github.com/go-sql-driver/mysql/utils.go | 755 +++++++++ .../vendor/github.com/jmoiron/sqlx/LICENSE | 23 + .../vendor/github.com/jmoiron/sqlx/bind.go | 208 +++ .../vendor/github.com/jmoiron/sqlx/doc.go | 12 + .../vendor/github.com/jmoiron/sqlx/named.go | 346 ++++ .../github.com/jmoiron/sqlx/named_context.go | 132 ++ .../jmoiron/sqlx/reflectx/reflect.go | 441 +++++ .../vendor/github.com/jmoiron/sqlx/sqlx.go | 1047 ++++++++++++ .../github.com/jmoiron/sqlx/sqlx_context.go | 348 ++++ .../vendor/github.com/juju/errors/LICENSE | 191 +++ .../vendor/github.com/juju/errors/doc.go | 81 + .../vendor/github.com/juju/errors/error.go | 172 ++ .../github.com/juju/errors/errortypes.go | 309 ++++ .../github.com/juju/errors/functions.go | 330 ++++ .../vendor/github.com/juju/errors/path.go | 38 + .../github.com/pingcap/check/benchmark.go | 187 +++ .../vendor/github.com/pingcap/check/check.go | 980 +++++++++++ .../github.com/pingcap/check/checkers.go | 458 ++++++ .../github.com/pingcap/check/checkers2.go | 131 ++ .../github.com/pingcap/check/compare.go | 161 ++ .../github.com/pingcap/check/helpers.go | 236 +++ .../github.com/pingcap/check/printer.go | 168 ++ .../vendor/github.com/pingcap/check/run.go | 179 ++ .../vendor/github.com/satori/go.uuid/LICENSE | 20 + .../vendor/github.com/satori/go.uuid/codec.go | 206 +++ .../github.com/satori/go.uuid/generator.go | 239 +++ .../vendor/github.com/satori/go.uuid/sql.go | 78 + .../vendor/github.com/satori/go.uuid/uuid.go | 161 ++ .../github.com/shopspring/decimal/LICENSE | 45 + .../shopspring/decimal/decimal-go.go | 414 +++++ .../github.com/shopspring/decimal/decimal.go | 1434 +++++++++++++++++ .../github.com/shopspring/decimal/rounding.go | 118 ++ .../github.com/siddontang/go-log/LICENSE | 21 + .../github.com/siddontang/go-log/log/doc.go | 20 + .../siddontang/go-log/log/filehandler.go | 230 +++ .../siddontang/go-log/log/handler.go | 54 + .../github.com/siddontang/go-log/log/log.go | 137 ++ .../siddontang/go-log/log/logger.go | 340 ++++ .../siddontang/go-log/loggers/loggers.go | 68 + .../vendor/github.com/siddontang/go/LICENSE | 20 + .../github.com/siddontang/go/bson/LICENSE | 25 + .../github.com/siddontang/go/filelock/LICENSE | 27 + .../github.com/siddontang/go/hack/hack.go | 27 + .../github.com/siddontang/go/snappy/LICENSE | 27 + .../github.com/siddontang/go/sync2/atomic.go | 146 ++ .../siddontang/go/sync2/semaphore.go | 65 + .../google.golang.org/appengine/LICENSE | 202 +++ .../appengine/cloudsql/cloudsql.go | 62 + .../appengine/cloudsql/cloudsql_classic.go | 17 + .../appengine/cloudsql/cloudsql_vm.go | 16 + 107 files changed, 21297 insertions(+) create mode 100644 vendor/github.com/siddontang/go-mysql/Gopkg.lock create mode 100644 vendor/github.com/siddontang/go-mysql/Gopkg.toml create mode 100755 vendor/github.com/siddontang/go-mysql/clear_vendor.sh create mode 100644 vendor/github.com/siddontang/go-mysql/client/tls.go create mode 100644 vendor/github.com/siddontang/go-mysql/docker/docker-compose.yaml create mode 100644 vendor/github.com/siddontang/go-mysql/docker/resources/ca.key create mode 100644 vendor/github.com/siddontang/go-mysql/docker/resources/ca.pem create mode 100644 vendor/github.com/siddontang/go-mysql/docker/resources/client-cert.pem create mode 100644 vendor/github.com/siddontang/go-mysql/docker/resources/client-key.pem create mode 100644 vendor/github.com/siddontang/go-mysql/docker/resources/server-cert.pem create mode 100644 vendor/github.com/siddontang/go-mysql/docker/resources/server-key.pem create mode 100644 vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/replication/backup_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/replication/time.go create mode 100644 vendor/github.com/siddontang/go-mysql/replication/time_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/auth_switch_response.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/caching_sha2_cache_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/command_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/credential_provider.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/example/server_example.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/handshake_resp.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/initial_handshake.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/server_conf.go create mode 100644 vendor/github.com/siddontang/go-mysql/server/ssl.go create mode 100644 vendor/github.com/siddontang/go-mysql/test_util/test_keys/keys.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/COPYING create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode_meta.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/doc.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encode.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/lex.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/parse.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_check.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_fields.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/const.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/fields.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/infile.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/result.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/transaction.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/bind.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/doc.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named_context.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx_context.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/doc.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/error.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/errortypes.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/functions.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/path.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/benchmark.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/check.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers2.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/compare.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/helpers.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/printer.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/run.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/codec.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/generator.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/sql.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/uuid.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal-go.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/rounding.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/doc.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/filehandler.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/handler.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/log.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/logger.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/loggers/loggers.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/bson/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/filelock/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/hack/hack.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/snappy/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/atomic.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/semaphore.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/LICENSE create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_classic.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_vm.go diff --git a/vendor/github.com/siddontang/go-mysql/Gopkg.lock b/vendor/github.com/siddontang/go-mysql/Gopkg.lock new file mode 100644 index 0000000..ae65b1d --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/Gopkg.lock @@ -0,0 +1,78 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/BurntSushi/toml" + packages = ["."] + revision = "b26d9c308763d68093482582cea63d69be07a0f0" + version = "v0.3.0" + +[[projects]] + branch = "master" + name = "github.com/go-sql-driver/mysql" + packages = ["."] + revision = "99ff426eb706cffe92ff3d058e168b278cabf7c7" + +[[projects]] + branch = "master" + name = "github.com/jmoiron/sqlx" + packages = [ + ".", + "reflectx" + ] + revision = "2aeb6a910c2b94f2d5eb53d9895d80e27264ec41" + +[[projects]] + branch = "master" + name = "github.com/juju/errors" + packages = ["."] + revision = "c7d06af17c68cd34c835053720b21f6549d9b0ee" + +[[projects]] + branch = "master" + name = "github.com/pingcap/check" + packages = ["."] + revision = "1c287c953996ab3a0bf535dba9d53d809d3dc0b6" + +[[projects]] + name = "github.com/satori/go.uuid" + packages = ["."] + revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" + version = "v1.2.0" + +[[projects]] + name = "github.com/shopspring/decimal" + packages = ["."] + revision = "cd690d0c9e2447b1ef2a129a6b7b49077da89b8e" + version = "1.1.0" + +[[projects]] + branch = "master" + name = "github.com/siddontang/go" + packages = [ + "hack", + "sync2" + ] + revision = "2b7082d296ba89ae7ead0f977816bddefb65df9d" + +[[projects]] + branch = "master" + name = "github.com/siddontang/go-log" + packages = [ + "log", + "loggers" + ] + revision = "a4d157e46fa3e08b7e7ff329af341fa3ff86c02c" + +[[projects]] + name = "google.golang.org/appengine" + packages = ["cloudsql"] + revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" + version = "v1.1.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "a1f9939938a58551bbb3f19411c9d1386995d36296de6f6fb5d858f5923db85e" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/vendor/github.com/siddontang/go-mysql/Gopkg.toml b/vendor/github.com/siddontang/go-mysql/Gopkg.toml new file mode 100644 index 0000000..71df4b3 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/Gopkg.toml @@ -0,0 +1,56 @@ +# Gopkg.toml example +# +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/BurntSushi/toml" + version = "v0.3.0" + +[[constraint]] + name = "github.com/go-sql-driver/mysql" + branch = "master" + +[[constraint]] + branch = "master" + name = "github.com/juju/errors" + +[[constraint]] + name = "github.com/satori/go.uuid" + version = "v1.2.0" + +[[constraint]] + name = "github.com/shopspring/decimal" + version = "v1.1.0" + +[[constraint]] + branch = "master" + name = "github.com/siddontang/go" + +[prune] + go-tests = true + unused-packages = true + non-go = true + diff --git a/vendor/github.com/siddontang/go-mysql/clear_vendor.sh b/vendor/github.com/siddontang/go-mysql/clear_vendor.sh new file mode 100755 index 0000000..81ba6b1 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/clear_vendor.sh @@ -0,0 +1,6 @@ +find vendor \( -type f -or -type l \) -not -name "*.go" -not -name "LICENSE" -not -name "*.s" -not -name "PATENTS" -not -name "*.h" -not -name "*.c" | xargs -I {} rm {} +# delete all test files +find vendor -type f -name "*_generated.go" | xargs -I {} rm {} +find vendor -type f -name "*_test.go" | xargs -I {} rm {} +find vendor -type d -name "_vendor" | xargs -I {} rm -rf {} +find vendor -type d -empty | xargs -I {} rm -rf {} \ No newline at end of file diff --git a/vendor/github.com/siddontang/go-mysql/client/tls.go b/vendor/github.com/siddontang/go-mysql/client/tls.go new file mode 100644 index 0000000..3772a50 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/client/tls.go @@ -0,0 +1,28 @@ +package client + +import ( + "crypto/tls" + "crypto/x509" +) + +// generate TLS config for client side +// if insecureSkipVerify is set to true, serverName will not be validated +func NewClientTLSConfig(caPem, certPem, keyPem []byte, insecureSkipVerify bool, serverName string) *tls.Config { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPem) { + panic("failed to add ca PEM") + } + + cert, err := tls.X509KeyPair(certPem, keyPem) + if err != nil { + panic(err) + } + + config := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: pool, + InsecureSkipVerify: insecureSkipVerify, + ServerName: serverName, + } + return config +} diff --git a/vendor/github.com/siddontang/go-mysql/docker/docker-compose.yaml b/vendor/github.com/siddontang/go-mysql/docker/docker-compose.yaml new file mode 100644 index 0000000..151786e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/docker/docker-compose.yaml @@ -0,0 +1,80 @@ +version: '3' +services: + + mysql-5.5.61: + image: "mysql:5.5.61" + container_name: "mysql-server-5.5.61" + ports: + - "5561:3306" + command: --ssl=TRUE --ssl-ca=/usr/local/mysql/ca.pem --ssl-cert=/usr/local/mysql/server-cert.pem --ssl-key=/usr/local/mysql/server-key.pem + volumes: + - ./resources/ca.pem:/usr/local/mysql/ca.pem + - ./resources/server-cert.pem:/usr/local/mysql/server-cert.pem + - ./resources/server-key.pem:/usr/local/mysql/server-key.pem + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - bind-address=0.0.0.0 + + mysql-5.6.41: + image: "mysql:5.6.41" + container_name: "mysql-server-5.6.41" + ports: + - "5641:3306" + command: --ssl=TRUE --ssl-ca=/usr/local/mysql/ca.pem --ssl-cert=/usr/local/mysql/server-cert.pem --ssl-key=/usr/local/mysql/server-key.pem + volumes: + - ./resources/ca.pem:/usr/local/mysql/ca.pem + - ./resources/server-cert.pem:/usr/local/mysql/server-cert.pem + - ./resources/server-key.pem:/usr/local/mysql/server-key.pem + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - bind-address=0.0.0.0 + + mysql-default: + image: "mysql:5.7.22" + container_name: "mysql-server-default" + ports: + - "3306:3306" + command: ["mysqld", "--log-bin=mysql-bin", "--server-id=1"] + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - bind-address=0.0.0.0 + + mysql-5.7.22: + image: "mysql:5.7.22" + container_name: "mysql-server-5.7.22" + ports: + - "5722:3306" + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - bind-address=0.0.0.0 + + mysql-8.0.3: + image: "mysql:8.0.3" + container_name: "mysql-server-8.0.3" + ports: + - "8003:3306" + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - bind-address=0.0.0.0 + + mysql-8.0.12: + image: "mysql:8.0.12" + container_name: "mysql-server-8.0.12" + ports: + - "8012:3306" + environment: + #- MYSQL_ROOT_PASSWORD=abc123 + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - bind-address=0.0.0.0 + + mysql-8.0.12-sha256: + image: "mysql:8.0.12" + container_name: "mysql-server-8.0.12-sha256" + ports: + - "8013:3306" + entrypoint: ['/entrypoint.sh', '--default-authentication-plugin=sha256_password'] + environment: + #- MYSQL_ROOT_PASSWORD=abc123 + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - bind-address=0.0.0.0 + diff --git a/vendor/github.com/siddontang/go-mysql/docker/resources/ca.key b/vendor/github.com/siddontang/go-mysql/docker/resources/ca.key new file mode 100644 index 0000000..8344ed2 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/docker/resources/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAsV6xlhFxMn14Pn7XBRGLt8/HXmhVVu20IKFgIOyX7gAZr0QL +suT1fGf5zH9HrlgOMkfdhV847U03KPfUnBsi9lS6/xOxnH/OzTYM0WW0eNMGF7eo +xrS64GSbPVX4pLi5+uwrrZT5HmDgZi49ANmuX6UYmH/eRRvSIoYUTV6t0aYsLyKv +lpEAtRAe4AlKB236j5ggmJ36QUhTFTbeNbeOOgloTEdPK8Y/kgpnhiqzMdPqqIc7 +IeXUc456yX8MJUgniTM2qCNTFdEw+C2Ok0RbU6TI2SuEgVF4jtCcVEKxZ8kYbioO +NaePQKFR/EhdXO+/ag1IEdXElH9knLOfB+zCgwIDAQABAoIBAC2U0jponRiGmgIl +gohw6+D+6pNeaKAAUkwYbKXJZ3noWLFr4T3GDTg9WDqvcvJg+rT9NvZxdCW3tDc5 +CVBcwO1g9PVcUEaRqcme3EhrxKdQQ76QmjUGeQf1ktd+YnmiZ1kOnGLtZ9/gsYpQ +06iGSIOX3+xA4BQOhEAPCOShMjYv+pWvWGhZCSmeulKulNVPBbG2H1I9EoT5Wd+Q +8LUfgZOuUXrtcsuvEf2XeacCo0pUbjx8ErhDHP6aPasFAXq15Bm8DnsUOrrsjcLy +sPy/mHwpd6kTw+O3EzjTdaYSFRoDSpfpIS5Bk+yicdxOmTwp1pzDu6HyYnuOnc9Q +JQ8HvlECgYEA2z+1HKVz5k7NYyRihW4l30vAcAGcgG1RObB6DmLbGu4MPvMymgLO +1QhYjlCcKfRHhVS2864op3Oba2fIgCc2am0DIQQ6kZ23ick78aj9G2ZXYpdpIPLu +Kl1AZHj6XDrOPVqidwcE6iYHLLWp9x4Atgw5d44XmhQ0kwrqAfccOX8CgYEAzxnl +7Uu+v5WI3hBVJpxS6eoS1TdztVIJaumyE43pBoHEuJrp4MRf0Lu2DiDpH8R3/RoE +o+ykn6xzphYwUopYaCWzYTKoXvxCvmLkDjHcpdzLtwWbKG+MJih2nTADEDI7sK4e +a3IU8miK6FeqkQHfs/5dlQa8q31yxiukw0qQEP0CgYAtLg6jTZD5l6mJUZkfx9f0 +EMciDaLzcBN54Nz2E/b0sLNDUZhO1l9K1QJyqTfVCWqnlhJxWqU0BIW1d1iA2BPF +kJtBdX6gPTDyKs64eMtXlxpQzcSzLnxXrIm1apyk3tVbHU83WfHwUk/OLc1NiBg7 +a394HIbOkHVZC7m3F/Xv/wKBgQDHrM2du8D+kJs0l4SxxFjAxPlBb8R01tLTrNwP +tGwu5OEZp+rE1jEXXFRMTPjXsyKI+hPtRJT4ilm6kXwnqNFSIL9RgHkLk6Z6T3hY +I0T8+ePD43jURLBYffzW0tqxO+2HDGmx6H0/twHuv89pHehkb2Qk8ijoIvyNCrlB +vVsntQKBgCK04nbb+G45D6TKCcZ6XKT/+qneJQE5cfvHl5EqrfjSmlnEUpJjJfyc +6Q1PtXtWOtOScU93f1JKL7+JBbWDn9uBlboM8BSkAVVd/2vyg88RuEtIru1syxcW +d1rMxqaMRJuhuqaS33CoPUpn30b4zVrPhQJ2+TwDAol4qIGHaie8 +-----END RSA PRIVATE KEY----- diff --git a/vendor/github.com/siddontang/go-mysql/docker/resources/ca.pem b/vendor/github.com/siddontang/go-mysql/docker/resources/ca.pem new file mode 100644 index 0000000..e251bd6 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/docker/resources/ca.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDtTCCAp2gAwIBAgIJANeS1FOzWXlZMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTgwODE2MTUxNDE5WhcNMjEwNjA1MTUxNDE5WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAsV6xlhFxMn14Pn7XBRGLt8/HXmhVVu20IKFgIOyX7gAZr0QLsuT1fGf5 +zH9HrlgOMkfdhV847U03KPfUnBsi9lS6/xOxnH/OzTYM0WW0eNMGF7eoxrS64GSb +PVX4pLi5+uwrrZT5HmDgZi49ANmuX6UYmH/eRRvSIoYUTV6t0aYsLyKvlpEAtRAe +4AlKB236j5ggmJ36QUhTFTbeNbeOOgloTEdPK8Y/kgpnhiqzMdPqqIc7IeXUc456 +yX8MJUgniTM2qCNTFdEw+C2Ok0RbU6TI2SuEgVF4jtCcVEKxZ8kYbioONaePQKFR +/EhdXO+/ag1IEdXElH9knLOfB+zCgwIDAQABo4GnMIGkMB0GA1UdDgQWBBQgHiwD +00upIbCOunlK4HRw89DhjjB1BgNVHSMEbjBsgBQgHiwD00upIbCOunlK4HRw89Dh +jqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNV +BAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJANeS1FOzWXlZMAwGA1UdEwQF +MAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAFMZFQTFKU5tWIpWh8BbVZeVZcng0Kiq +qwbhVwaTkqtfmbqw8/w+faOWylmLncQEMmgvnUltGMQlQKBwQM2byzPkz9phal3g +uI0JWJYqtcMyIQUB9QbbhrDNC9kdt/ji/x6rrIqzaMRuiBXqH5LQ9h856yXzArqd +cAQGzzYpbUCIv7ciSB93cKkU73fQLZVy5ZBy1+oAa1V9U4cb4G/20/PDmT+G3Gxz +pEjeDKtz8XINoWgA2cSdfAhNZt5vqJaCIZ8qN0z6C7SUKwUBderERUMLUXdhUldC +KTVHyEPvd0aULd5S5vEpKCnHcQmFcLdoN8t9k9pR9ZgwqXbyJHlxWFo= +-----END CERTIFICATE----- diff --git a/vendor/github.com/siddontang/go-mysql/docker/resources/client-cert.pem b/vendor/github.com/siddontang/go-mysql/docker/resources/client-cert.pem new file mode 100644 index 0000000..e478e78 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/docker/resources/client-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBjCCAe4CCQDg06wCf7hcuTANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 +cyBQdHkgTHRkMB4XDTE4MDgxOTA4NDY0N1oXDTI4MDgxNjA4NDY0N1owRTELMAkG +A1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0 +IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AMmivNyk3Rc1ZvLPhb3WPNkf9f2G4g9nMc0+eMrR1IKJ1U1A98ojeIBT+pfk1bSq +Ol0UDm66Vd3YQ+4HpyYHaYV6mwoTEulL9Quk8RLa7TRwQu3PLi3o567RhVIrx8Z3 +umuWb9UUzJfSFH04Uy9+By4CJCqIQXU4BocLIKHhIkNjmAQ9fWO1hZ8zmPHSEfvu +Wqa/DYKGvF0MJr4Lnkm/sKUd+O94p9suvwM6OGIDibACiKRF2H+JbgQLbA58zkLv +DHtXOqsCL7HxiONX8VDrQjN/66Nh9omk/Bx2Ec8IqappHvWf768HSH79x/znaial +VEV+6K0gP+voJHfnA10laWMCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAPD+Fn1qj +HN62GD3eIgx6wJxYuemhdbgmEwrZZf4V70lS6e9Iloif0nBiISDxJUpXVWNRCN3Z +3QVC++F7deDmWL/3dSpXRvWsapzbCUhVQ2iBcnZ7QCOdvAqYR1ecZx70zvXCwBcd +6XKmRtdeNV6B211KRFmTYtVyPq4rcWrkTPGwPBncJI1eQQmyFv2T9SwVVp96Nbrq +sf7zrJGmuVCdXGPRi/ALVHtJCz6oPoft3I707eMe+ijnFqwGbmMD4fMD6Ync/hEz +PyR5FMZkXSXHS0gkA5pfwW7wJ2WSWDhI6JMS1gbatY7QzgHbKoQpxBPUXlnzzj2h +7O9cgFTh/XOZXQ== +-----END CERTIFICATE----- diff --git a/vendor/github.com/siddontang/go-mysql/docker/resources/client-key.pem b/vendor/github.com/siddontang/go-mysql/docker/resources/client-key.pem new file mode 100644 index 0000000..996a97b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/docker/resources/client-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAyaK83KTdFzVm8s+FvdY82R/1/YbiD2cxzT54ytHUgonVTUD3 +yiN4gFP6l+TVtKo6XRQObrpV3dhD7genJgdphXqbChMS6Uv1C6TxEtrtNHBC7c8u +LejnrtGFUivHxne6a5Zv1RTMl9IUfThTL34HLgIkKohBdTgGhwsgoeEiQ2OYBD19 +Y7WFnzOY8dIR++5apr8Ngoa8XQwmvgueSb+wpR3473in2y6/Azo4YgOJsAKIpEXY +f4luBAtsDnzOQu8Me1c6qwIvsfGI41fxUOtCM3/ro2H2iaT8HHYRzwipqmke9Z/v +rwdIfv3H/OdqJqVURX7orSA/6+gkd+cDXSVpYwIDAQABAoIBAAGLY5L1GFRzLkSx +3j5kA7dODV5RyC2CBtmhnt8+2DffwmiDFOLRfrzM5+B9+j0WCLhpzOqANuQqIesS +1+7so5xIIiPjnYN393qNWuNgFe0O5xRXP+1OGWg3ZqQIfdFBXYYxcs3ZCPAoxctn +wQteFcP+dDR3MrkpIrOqHCfhR5foieOMP+9k5kCjk+aZqhEmFyko+X+xVO/32xs+ ++3qXhUrHt3Op5on30QMOFguniQlYwLJkd9qVjGuGMIrVPxoUz0rya4SKrGKgkAr8 +mvQe2+sZo7cc6zC2ceaGMJU7z1RalTrCObbg5mynlu+Vf0E/YiES0abkQhSbcSB9 +mAkJC7ECgYEA/H1NDEiO164yYK9ji4HM/8CmHegWS4qsgrzAs8lU0yAcgdg9e19A +mNi8yssfIBCw62RRE4UGWS5F82myhmvq/mXbf8eCJ2CMgdCHQh1rT7WFD/Uc5Pe/ +8Lv2jNMQ61POguPyq6D0qcf8iigKIMHa1MIgAOmrgWrxulfbSUhm370CgYEAzHBu +J9p4dAqW32+Hrtv2XE0KUjH72TXr13WErosgeGTfsIW2exXByvLasxOJSY4Wb8oS +OLZ7bgp/EBchAc7my+nF8n5uOJxipWQUB5BoeB9aUJZ9AnWF4RDl94Jlm5PYBG/J +lRXrMtSTTIgmSw3Ft2A1vRMOQaHX89lNwOZL758CgYAXOT84/gOFexRPKFKzpkDA +1WtyHMLQN/UeIVZoMwCGWtHEb6tYCa7bYDQdQwmd3Wsoe5WpgfbPhR4SAYrWKl72 +/09tNWCXVp4V4qRORH52Wm/ew+Dgfpk8/0zyLwfDXXYFPAo6Fxfp9ecYng4wbSQ/ +pYtkChooUTniteoJl4s+0QKBgHbFEpoAqF3yEPi52L/TdmrlLwvVkhT86IkB8xVc +Kn8HS5VH+V3EpBN9x2SmAupCq/JCGRftnAOwAWWdqkVcqGTq6V8Z6HrnD8A6RhCm +6qpuvI94/iNBl4fLw25pyRH7cFITh68fTsb3DKQ3rNeJpsYEFPRFb9Ddb5JxOmTI +5nDNAoGBAM+SyOhUGU+0Uw2WJaGWzmEutjeMRr5Z+cZ8keC/ZJNdji/faaQoeOQR +OXI8O6RBTBwVNQMyDyttT8J8BkISwfAhSdPkjgPw9GZ1pGREl53uCFDIlX2nvtQM +ioNzG5WHB7Gd7eUUTA91kRF9MZJTHPqNiNGR0Udj/trGyGqJebni +-----END RSA PRIVATE KEY----- diff --git a/vendor/github.com/siddontang/go-mysql/docker/resources/server-cert.pem b/vendor/github.com/siddontang/go-mysql/docker/resources/server-cert.pem new file mode 100644 index 0000000..3cb3b9c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/docker/resources/server-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDBjCCAe4CCQDg06wCf7hcuDANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 +cyBQdHkgTHRkMB4XDTE4MDgxOTA4NDUyNVoXDTI4MDgxNjA4NDUyNVowRTELMAkG +A1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0 +IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALK2gqK4uvTlxJANO2JKdibvmh899z6oCo9Km0mz5unj4dpnq9hljsQuKtcHUcM4 +HXcE06knaJ4TOF7lcsjaqoDO7r/SaFgjjXCqNvHD0Su4B+7qe52BZZTRV1AANP10 +PvebarXSEzaZUCyHHhSF8+Qb4vX04XKX/TOqinTVGtlnduKzP5+qsaFBtpLAw1V0 +At9EQB5BgnTYtdIsmvD4/2WhBvOjVxab75yx0R4oof4F3u528tbEegcWhBtmy2Xd +HI3S+TLljj3kOOdB+pgrVUl+KaDavWK3T+F1vTNDe56HEVNKeWlLy1scul61E0j9 +IkZAu6aRDxtKdl7bKu0BkzMCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAma3yFqR7 +xkeaZBg4/1I3jSlaNe5+2JB4iybAkMOu77fG5zytLomTbzdhewsuBwpTVMJdga8T +IdPeIFCin1U+5SkbjSMlpKf+krE+5CyrNJ5jAgO9ATIqx66oCTYXfGlNapGRLfSE +sa0iMqCe/dr4GPU+flW2DZFWiyJVDSF1JjReQnfrWY+SD2SpP/lmlgltnY8MJngd +xBLG5nsZCpUXGB713Q8ZyIm2ThVAMiskcxBleIZDDghLuhGvY/9eFJhZpvOkjWa6 +XGEi4E1G/SA+zVKFl41nHKCdqXdmIOnpcLlFBUVloQok5a95Kqc1TYw3f+WbdFff +99dAgk3gWwWZQA== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/vendor/github.com/siddontang/go-mysql/docker/resources/server-key.pem b/vendor/github.com/siddontang/go-mysql/docker/resources/server-key.pem new file mode 100644 index 0000000..babaaae --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/docker/resources/server-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAsraCori69OXEkA07Ykp2Ju+aHz33PqgKj0qbSbPm6ePh2mer +2GWOxC4q1wdRwzgddwTTqSdonhM4XuVyyNqqgM7uv9JoWCONcKo28cPRK7gH7up7 +nYFllNFXUAA0/XQ+95tqtdITNplQLIceFIXz5Bvi9fThcpf9M6qKdNUa2Wd24rM/ +n6qxoUG2ksDDVXQC30RAHkGCdNi10iya8Pj/ZaEG86NXFpvvnLHRHiih/gXe7nby +1sR6BxaEG2bLZd0cjdL5MuWOPeQ450H6mCtVSX4poNq9YrdP4XW9M0N7nocRU0p5 +aUvLWxy6XrUTSP0iRkC7ppEPG0p2Xtsq7QGTMwIDAQABAoIBAGh1m8hHWCg7gXh9 +838RbRx3IswuKS27hWiaQEiFWmzOIb7KqDy1qAxtu+ayRY1paHegH6QY/+Kd824s +ibpzbgQacJ04/HrAVTVMmQ8Z2VLHoAN7lcPL1bd14aZGaLLZVtDeTDJ413grhxxv +4ho27gcgcbo4Z+rWgk7H2WRPCAGYqWYAycm3yF5vy9QaO6edU+T588YsEQOos5iy +5pVFSGDGZkcUp1ukL3BJYR+jvygn6WPCobQ/LScUdi+ucitaI9i+UdlLokZARVRG +M/msqcTM73thR8yVRcexU6NUDxRBfZ/f7moSAEbBmGDXuxDcIyH9KGMQ2rMtN1X3 +lK8UNwkCgYEA2STJq/IUQHjdqd3Dqh/Q7Zm8/pMWFqLJSkqpnFtFsXPyUOx9zDOy +KqkIfGeyKwvsj9X9BcZ0FUKj9zoct1/WpPY+h7i7+z0MIujBh4AMjAcDrt4o76yK +UHuVmG2xKTdJoAbqOdToQeX6E82Ioal5pbB2W7AbCQScNBPZ52jxgtcCgYEA0rE7 +2dFiRm0YmuszFYxft2+GP6NgP3R2TQNEooi1uCXG2xgwObie1YCHzpZ5CfSqJIxP +XB7DXpIWi7PxJoeai2F83LnmdFz6F1BPRobwDoSFNdaSKLg4Yf856zpgYNKhL1fE +OoOXj4VBWBZh1XDfZV44fgwlMIf7edOF1XOagwUCgYAw953O+7FbdKYwF0V3iOM5 +oZDAK/UwN5eC/GFRVDfcM5RycVJRCVtlSWcTfuLr2C2Jpiz/72fgH34QU3eEVsV1 +v94MBznFB1hESw7ReqvZq/9FoO3EVrl+OtBaZmosLD6bKtQJJJ0Xtz/01UW5hxla +pveZ55XBK9v51nwuNjk4UwKBgHD8fJUllSchUCWb5cwzeAz98Kdl7LJ6uQo5q2/i +EllLYOWThiEeIYdrIuklholRPIDXAaPsF2c6vn5yo+q+o6EFSZlw0+YpCjDAb5Lp +wAh5BprFk6HkkM/0t9Guf4rMyYWC8odSlE9x7YXYkuSMYDCTI4Zs6vCoq7I8PbQn +B4AlAoGAZ6Ee5m/ph5UVp/3+cR6jCY7aHBUU/M3pbJSkVjBW+ymEBVJ6sUdz8k3P +x8BiPEQggNN7faWBqRWP7KXPnDYHh6shYUgPJwI5HX6NE/ZDnnXjeysHRyf0oCo5 +S6tHXwHNKB5HS1c/KDyyNGjP2oi/MF4o/MGWNWEcK6TJA3RGOYM= +-----END RSA PRIVATE KEY----- diff --git a/vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid_test.go b/vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid_test.go new file mode 100644 index 0000000..1455e26 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/mysql/mariadb_gtid_test.go @@ -0,0 +1,234 @@ +package mysql + +import ( + "github.com/pingcap/check" +) + +type mariaDBTestSuite struct { +} + +var _ = check.Suite(&mariaDBTestSuite{}) + +func (t *mariaDBTestSuite) SetUpSuite(c *check.C) { + +} + +func (t *mariaDBTestSuite) TearDownSuite(c *check.C) { + +} + +func (t *mariaDBTestSuite) TestParseMariaDBGTID(c *check.C) { + cases := []struct { + gtidStr string + hashError bool + }{ + {"0-1-1", false}, + {"", false}, + {"0-1-1-1", true}, + {"1", true}, + {"0-1-seq", true}, + } + + for _, cs := range cases { + gtid, err := ParseMariadbGTID(cs.gtidStr) + if cs.hashError { + c.Assert(err, check.NotNil) + } else { + c.Assert(err, check.IsNil) + c.Assert(gtid.String(), check.Equals, cs.gtidStr) + } + } +} + +func (t *mariaDBTestSuite) TestMariaDBGTIDConatin(c *check.C) { + cases := []struct { + originGTIDStr, otherGTIDStr string + contain bool + }{ + {"0-1-1", "0-1-2", false}, + {"0-1-1", "", true}, + {"2-1-1", "1-1-1", false}, + {"1-2-1", "1-1-1", true}, + {"1-2-2", "1-1-1", true}, + } + + for _, cs := range cases { + originGTID, err := ParseMariadbGTID(cs.originGTIDStr) + c.Assert(err, check.IsNil) + otherGTID, err := ParseMariadbGTID(cs.otherGTIDStr) + c.Assert(err, check.IsNil) + + c.Assert(originGTID.Contain(otherGTID), check.Equals, cs.contain) + } +} + +func (t *mariaDBTestSuite) TestMariaDBGTIDClone(c *check.C) { + gtid, err := ParseMariadbGTID("1-1-1") + c.Assert(err, check.IsNil) + + clone := gtid.Clone() + c.Assert(gtid, check.DeepEquals, clone) +} + +func (t *mariaDBTestSuite) TestMariaDBForward(c *check.C) { + cases := []struct { + currentGTIDStr, newerGTIDStr string + hashError bool + }{ + {"0-1-1", "0-1-2", false}, + {"0-1-1", "", false}, + {"2-1-1", "1-1-1", true}, + {"1-2-1", "1-1-1", false}, + {"1-2-2", "1-1-1", false}, + } + + for _, cs := range cases { + currentGTID, err := ParseMariadbGTID(cs.currentGTIDStr) + c.Assert(err, check.IsNil) + newerGTID, err := ParseMariadbGTID(cs.newerGTIDStr) + c.Assert(err, check.IsNil) + + err = currentGTID.forward(newerGTID) + if cs.hashError { + c.Assert(err, check.NotNil) + c.Assert(currentGTID.String(), check.Equals, cs.currentGTIDStr) + } else { + c.Assert(err, check.IsNil) + c.Assert(currentGTID.String(), check.Equals, cs.newerGTIDStr) + } + } +} + +func (t *mariaDBTestSuite) TestParseMariaDBGTIDSet(c *check.C) { + cases := []struct { + gtidStr string + subGTIDs map[uint32]string //domain ID => gtid string + expectedStr []string // test String() + hasError bool + }{ + {"0-1-1", map[uint32]string{0: "0-1-1"}, []string{"0-1-1"}, false}, + {"", nil, []string{""}, false}, + {"0-1-1,1-2-3", map[uint32]string{0: "0-1-1", 1: "1-2-3"}, []string{"0-1-1,1-2-3", "1-2-3,0-1-1"}, false}, + {"0-1--1", nil, nil, true}, + } + + for _, cs := range cases { + gtidSet, err := ParseMariadbGTIDSet(cs.gtidStr) + if cs.hasError { + c.Assert(err, check.NotNil) + } else { + c.Assert(err, check.IsNil) + mariadbGTIDSet, ok := gtidSet.(*MariadbGTIDSet) + c.Assert(ok, check.IsTrue) + + // check sub gtid + c.Assert(mariadbGTIDSet.Sets, check.HasLen, len(cs.subGTIDs)) + for domainID, gtid := range mariadbGTIDSet.Sets { + c.Assert(mariadbGTIDSet.Sets, check.HasKey, domainID) + c.Assert(gtid.String(), check.Equals, cs.subGTIDs[domainID]) + } + + // check String() function + inExpectedResult := false + actualStr := mariadbGTIDSet.String() + for _, str := range cs.expectedStr { + if str == actualStr { + inExpectedResult = true + break + } + } + c.Assert(inExpectedResult, check.IsTrue) + } + } +} + +func (t *mariaDBTestSuite) TestMariaDBGTIDSetUpdate(c *check.C) { + cases := []struct { + isNilGTID bool + gtidStr string + subGTIDs map[uint32]string + }{ + {true, "", map[uint32]string{1: "1-1-1", 2: "2-2-2"}}, + {false, "1-2-2", map[uint32]string{1: "1-2-2", 2: "2-2-2"}}, + {false, "1-2-1", map[uint32]string{1: "1-2-1", 2: "2-2-2"}}, + {false, "3-2-1", map[uint32]string{1: "1-1-1", 2: "2-2-2", 3: "3-2-1"}}, + } + + for _, cs := range cases { + gtidSet, err := ParseMariadbGTIDSet("1-1-1,2-2-2") + c.Assert(err, check.IsNil) + mariadbGTIDSet, ok := gtidSet.(*MariadbGTIDSet) + c.Assert(ok, check.IsTrue) + + if cs.isNilGTID { + c.Assert(mariadbGTIDSet.AddSet(nil), check.IsNil) + } else { + err := gtidSet.Update(cs.gtidStr) + c.Assert(err, check.IsNil) + } + // check sub gtid + c.Assert(mariadbGTIDSet.Sets, check.HasLen, len(cs.subGTIDs)) + for domainID, gtid := range mariadbGTIDSet.Sets { + c.Assert(mariadbGTIDSet.Sets, check.HasKey, domainID) + c.Assert(gtid.String(), check.Equals, cs.subGTIDs[domainID]) + } + } +} + +func (t *mariaDBTestSuite) TestMariaDBGTIDSetEqual(c *check.C) { + cases := []struct { + originGTIDStr, otherGTIDStr string + equals bool + }{ + {"", "", true}, + {"1-1-1", "1-1-1,2-2-2", false}, + {"1-1-1,2-2-2", "1-1-1", false}, + {"1-1-1,2-2-2", "1-1-1,2-2-2", true}, + {"1-1-1,2-2-2", "1-1-1,2-2-3", false}, + } + + for _, cs := range cases { + originGTID, err := ParseMariadbGTIDSet(cs.originGTIDStr) + c.Assert(err, check.IsNil) + + otherGTID, err := ParseMariadbGTIDSet(cs.otherGTIDStr) + c.Assert(err, check.IsNil) + + c.Assert(originGTID.Equal(otherGTID), check.Equals, cs.equals) + } +} + +func (t *mariaDBTestSuite) TestMariaDBGTIDSetContain(c *check.C) { + cases := []struct { + originGTIDStr, otherGTIDStr string + contain bool + }{ + {"", "", true}, + {"1-1-1", "1-1-1,2-2-2", false}, + {"1-1-1,2-2-2", "1-1-1", true}, + {"1-1-1,2-2-2", "1-1-1,2-2-2", true}, + {"1-1-1,2-2-2", "1-1-1,2-2-1", true}, + {"1-1-1,2-2-2", "1-1-1,2-2-3", false}, + } + + for _, cs := range cases { + originGTIDSet, err := ParseMariadbGTIDSet(cs.originGTIDStr) + c.Assert(err, check.IsNil) + + otherGTIDSet, err := ParseMariadbGTIDSet(cs.otherGTIDStr) + c.Assert(err, check.IsNil) + + c.Assert(originGTIDSet.Contain(otherGTIDSet), check.Equals, cs.contain) + } +} + +func (t *mariaDBTestSuite) TestMariaDBGTIDSetClone(c *check.C) { + cases := []string{"", "1-1-1", "1-1-1,2-2-2"} + + for _, str := range cases { + gtidSet, err := ParseMariadbGTIDSet(str) + c.Assert(err, check.IsNil) + + c.Assert(gtidSet.Clone(), check.DeepEquals, gtidSet) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/replication/backup_test.go b/vendor/github.com/siddontang/go-mysql/replication/backup_test.go new file mode 100644 index 0000000..5e39e7f --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/replication/backup_test.go @@ -0,0 +1,50 @@ +package replication + +import ( + "context" + "github.com/juju/errors" + . "github.com/pingcap/check" + "github.com/siddontang/go-mysql/mysql" + "os" + "sync" + "time" +) + +func (t *testSyncerSuite) TestStartBackupEndInGivenTime(c *C) { + t.setupTest(c, mysql.MySQLFlavor) + + t.testExecute(c, "RESET MASTER") + + var wg sync.WaitGroup + wg.Add(1) + defer wg.Wait() + + go func() { + defer wg.Done() + + t.testSync(c, nil) + + t.testExecute(c, "FLUSH LOGS") + + t.testSync(c, nil) + }() + + os.RemoveAll("./var") + timeout := 2 * time.Second + + done := make(chan bool) + + go func() { + err := t.b.StartBackup("./var", mysql.Position{Name: "", Pos: uint32(0)}, timeout) + c.Assert(err, IsNil) + done <- true + }() + failTimeout := 5 * timeout + ctx, _ := context.WithTimeout(context.Background(), failTimeout) + select { + case <-done: + return + case <-ctx.Done(): + c.Assert(errors.New("time out error"), IsNil) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/replication/time.go b/vendor/github.com/siddontang/go-mysql/replication/time.go new file mode 100644 index 0000000..bd27c4e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/replication/time.go @@ -0,0 +1,49 @@ +package replication + +import ( + "fmt" + "strings" + "time" +) + +var ( + fracTimeFormat []string +) + +// fracTime is a help structure wrapping Golang Time. +type fracTime struct { + time.Time + + // Dec must in [0, 6] + Dec int + + timestampStringLocation *time.Location +} + +func (t fracTime) String() string { + tt := t.Time + if t.timestampStringLocation != nil { + tt = tt.In(t.timestampStringLocation) + } + return tt.Format(fracTimeFormat[t.Dec]) +} + +func formatZeroTime(frac int, dec int) string { + if dec == 0 { + return "0000-00-00 00:00:00" + } + + s := fmt.Sprintf("0000-00-00 00:00:00.%06d", frac) + + // dec must < 6, if frac is 924000, but dec is 3, we must output 924 here. + return s[0 : len(s)-(6-dec)] +} + +func init() { + fracTimeFormat = make([]string, 7) + fracTimeFormat[0] = "2006-01-02 15:04:05" + + for i := 1; i <= 6; i++ { + fracTimeFormat[i] = fmt.Sprintf("2006-01-02 15:04:05.%s", strings.Repeat("0", i)) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/replication/time_test.go b/vendor/github.com/siddontang/go-mysql/replication/time_test.go new file mode 100644 index 0000000..3a06aaf --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/replication/time_test.go @@ -0,0 +1,70 @@ +package replication + +import ( + "time" + + . "github.com/pingcap/check" +) + +type testTimeSuite struct{} + +var _ = Suite(&testTimeSuite{}) + +func (s *testTimeSuite) TestTime(c *C) { + tbls := []struct { + year int + month int + day int + hour int + min int + sec int + microSec int + frac int + expected string + }{ + {2000, 1, 1, 1, 1, 1, 1, 0, "2000-01-01 01:01:01"}, + {2000, 1, 1, 1, 1, 1, 1, 1, "2000-01-01 01:01:01.0"}, + {2000, 1, 1, 1, 1, 1, 1, 6, "2000-01-01 01:01:01.000001"}, + } + + for _, t := range tbls { + t1 := fracTime{time.Date(t.year, time.Month(t.month), t.day, t.hour, t.min, t.sec, t.microSec*1000, time.UTC), t.frac, nil} + c.Assert(t1.String(), Equals, t.expected) + } + + zeroTbls := []struct { + frac int + dec int + expected string + }{ + {0, 1, "0000-00-00 00:00:00.0"}, + {1, 1, "0000-00-00 00:00:00.0"}, + {123, 3, "0000-00-00 00:00:00.000"}, + {123000, 3, "0000-00-00 00:00:00.123"}, + {123, 6, "0000-00-00 00:00:00.000123"}, + {123000, 6, "0000-00-00 00:00:00.123000"}, + } + + for _, t := range zeroTbls { + c.Assert(formatZeroTime(t.frac, t.dec), Equals, t.expected) + } +} + +func (s *testTimeSuite) TestTimeStringLocation(c *C) { + t := fracTime{ + time.Date(2018, time.Month(7), 30, 10, 0, 0, 0, time.FixedZone("EST", -5*3600)), + 0, + nil, + } + + c.Assert(t.String(), Equals, "2018-07-30 10:00:00") + + t = fracTime{ + time.Date(2018, time.Month(7), 30, 10, 0, 0, 0, time.FixedZone("EST", -5*3600)), + 0, + time.UTC, + } + c.Assert(t.String(), Equals, "2018-07-30 15:00:00") +} + +var _ = Suite(&testTimeSuite{}) diff --git a/vendor/github.com/siddontang/go-mysql/server/auth_switch_response.go b/vendor/github.com/siddontang/go-mysql/server/auth_switch_response.go new file mode 100644 index 0000000..038acff --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/auth_switch_response.go @@ -0,0 +1,133 @@ +package server + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/tls" + "fmt" + + "github.com/juju/errors" + . "github.com/siddontang/go-mysql/mysql" +) + +func (c *Conn) handleAuthSwitchResponse() error { + authData, err := c.readAuthSwitchRequestResponse() + if err != nil { + return err + } + + switch c.authPluginName { + case AUTH_NATIVE_PASSWORD: + if err := c.acquirePassword(); err != nil { + return err + } + if !bytes.Equal(CalcPassword(c.salt, []byte(c.password)), authData) { + return ErrAccessDenied + } + return nil + + case AUTH_CACHING_SHA2_PASSWORD: + if !c.cachingSha2FullAuth { + // Switched auth method but no MoreData packet send yet + if err := c.compareCacheSha2PasswordAuthData(authData); err != nil { + return err + } else { + if c.cachingSha2FullAuth { + return c.handleAuthSwitchResponse() + } + return nil + } + } + // AuthMoreData packet already sent, do full auth + if err := c.handleCachingSha2PasswordFullAuth(authData); err != nil { + return err + } + c.writeCachingSha2Cache() + return nil + + case AUTH_SHA256_PASSWORD: + cont, err := c.handlePublicKeyRetrieval(authData) + if err != nil { + return err + } + if !cont { + return nil + } + if err := c.acquirePassword(); err != nil { + return err + } + return c.compareSha256PasswordAuthData(authData, c.password) + + default: + return errors.Errorf("unknown authentication plugin name '%s'", c.authPluginName) + } +} + +func (c *Conn) handleCachingSha2PasswordFullAuth(authData []byte) error { + if err := c.acquirePassword(); err != nil { + return err + } + if tlsConn, ok := c.Conn.Conn.(*tls.Conn); ok { + if !tlsConn.ConnectionState().HandshakeComplete { + return errors.New("incomplete TSL handshake") + } + // connection is SSL/TLS, client should send plain password + // deal with the trailing \NUL added for plain text password received + if l := len(authData); l != 0 && authData[l-1] == 0x00 { + authData = authData[:l-1] + } + if bytes.Equal(authData, []byte(c.password)) { + return nil + } + return ErrAccessDenied + } else { + // client either request for the public key or send the encrypted password + if len(authData) == 1 && authData[0] == 0x02 { + // send the public key + if err := c.writeAuthMoreDataPubkey(); err != nil { + return err + } + // read the encrypted password + var err error + if authData, err = c.readAuthSwitchRequestResponse(); err != nil { + return err + } + } + // the encrypted password + // decrypt + dbytes, err := rsa.DecryptOAEP(sha1.New(), rand.Reader, (c.serverConf.tlsConfig.Certificates[0].PrivateKey).(*rsa.PrivateKey), authData, nil) + if err != nil { + return err + } + plain := make([]byte, len(c.password)+1) + copy(plain, c.password) + for i := range plain { + j := i % len(c.salt) + plain[i] ^= c.salt[j] + } + if bytes.Equal(plain, dbytes) { + return nil + } + return ErrAccessDenied + } +} + +func (c *Conn) writeCachingSha2Cache() { + // write cache + if c.password == "" { + return + } + // SHA256(PASSWORD) + crypt := sha256.New() + crypt.Write([]byte(c.password)) + m1 := crypt.Sum(nil) + // SHA256(SHA256(PASSWORD)) + crypt.Reset() + crypt.Write(m1) + m2 := crypt.Sum(nil) + // caching_sha2_password will maintain an in-memory hash of `user`@`host` => SHA256(SHA256(PASSWORD)) + c.serverConf.cacheShaPassword.Store(fmt.Sprintf("%s@%s", c.user, c.Conn.LocalAddr()), m2) +} diff --git a/vendor/github.com/siddontang/go-mysql/server/caching_sha2_cache_test.go b/vendor/github.com/siddontang/go-mysql/server/caching_sha2_cache_test.go new file mode 100644 index 0000000..a8139eb --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/caching_sha2_cache_test.go @@ -0,0 +1,233 @@ +package server + +import ( + "database/sql" + "fmt" + "net" + "strings" + "sync" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/juju/errors" + . "github.com/pingcap/check" + "github.com/siddontang/go-log/log" + "github.com/siddontang/go-mysql/mysql" + "github.com/siddontang/go-mysql/test_util/test_keys" +) + +var delay = 50 + +// test caching for 'caching_sha2_password' +// NOTE the idea here is to plugin a throttled credential provider so that the first connection (cache miss) will take longer time +// than the second connection (cache hit). Remember to set the password for MySQL user otherwise it won't cache empty password. +func TestCachingSha2Cache(t *testing.T) { + log.SetLevel(log.LevelDebug) + + remoteProvider := &RemoteThrottleProvider{NewInMemoryProvider(), delay + 50} + remoteProvider.AddUser(*testUser, *testPassword) + cacheServer := NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_CACHING_SHA2_PASSWORD, test_keys.PubPem, tlsConf) + + // no TLS + Suite(&cacheTestSuite{ + server: cacheServer, + credProvider: remoteProvider, + tlsPara: "false", + }) + + TestingT(t) +} + +func TestCachingSha2CacheTLS(t *testing.T) { + log.SetLevel(log.LevelDebug) + + remoteProvider := &RemoteThrottleProvider{NewInMemoryProvider(), delay + 50} + remoteProvider.AddUser(*testUser, *testPassword) + cacheServer := NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_CACHING_SHA2_PASSWORD, test_keys.PubPem, tlsConf) + + // TLS + Suite(&cacheTestSuite{ + server: cacheServer, + credProvider: remoteProvider, + tlsPara: "skip-verify", + }) + + TestingT(t) +} + +type RemoteThrottleProvider struct { + *InMemoryProvider + delay int // in milliseconds +} + +func (m *RemoteThrottleProvider) GetCredential(username string) (password string, found bool, err error) { + time.Sleep(time.Millisecond * time.Duration(m.delay)) + return m.InMemoryProvider.GetCredential(username) +} + +type cacheTestSuite struct { + server *Server + credProvider CredentialProvider + tlsPara string + + db *sql.DB + + l net.Listener +} + +func (s *cacheTestSuite) SetUpSuite(c *C) { + var err error + + s.l, err = net.Listen("tcp", *testAddr) + c.Assert(err, IsNil) + + go s.onAccept(c) + + time.Sleep(30 * time.Millisecond) +} + +func (s *cacheTestSuite) TearDownSuite(c *C) { + if s.l != nil { + s.l.Close() + } +} + +func (s *cacheTestSuite) onAccept(c *C) { + for { + conn, err := s.l.Accept() + if err != nil { + return + } + + go s.onConn(conn, c) + } +} + +func (s *cacheTestSuite) onConn(conn net.Conn, c *C) { + //co, err := NewConn(conn, *testUser, *testPassword, &testHandler{s}) + co, err := NewCustomizedConn(conn, s.server, s.credProvider, &testCacheHandler{s}) + c.Assert(err, IsNil) + for { + err = co.HandleCommand() + if err != nil { + return + } + } +} + +func (s *cacheTestSuite) runSelect(c *C) { + var a int64 + var b string + + err := s.db.QueryRow("SELECT a, b FROM tbl WHERE id=1").Scan(&a, &b) + c.Assert(err, IsNil) + c.Assert(a, Equals, int64(1)) + c.Assert(b, Equals, "hello world") +} + +func (s *cacheTestSuite) TestCache(c *C) { + // first connection + t1 := time.Now() + var err error + s.db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?tls=%s", *testUser, *testPassword, *testAddr, *testDB, s.tlsPara)) + c.Assert(err, IsNil) + s.db.SetMaxIdleConns(4) + s.runSelect(c) + t2 := time.Now() + + d1 := int(t2.Sub(t1).Nanoseconds() / 1e6) + //log.Debugf("first connection took %d milliseconds", d1) + + c.Assert(d1, GreaterEqual, delay) + + if s.db != nil { + s.db.Close() + } + + // second connection + t3 := time.Now() + s.db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?tls=%s", *testUser, *testPassword, *testAddr, *testDB, s.tlsPara)) + c.Assert(err, IsNil) + s.db.SetMaxIdleConns(4) + s.runSelect(c) + t4 := time.Now() + + d2 := int(t4.Sub(t3).Nanoseconds() / 1e6) + //log.Debugf("second connection took %d milliseconds", d2) + + c.Assert(d2, Less, delay) + if s.db != nil { + s.db.Close() + } + + s.server.cacheShaPassword = &sync.Map{} +} + +type testCacheHandler struct { + s *cacheTestSuite +} + +func (h *testCacheHandler) UseDB(dbName string) error { + return nil +} + +func (h *testCacheHandler) handleQuery(query string, binary bool) (*mysql.Result, error) { + ss := strings.Split(query, " ") + switch strings.ToLower(ss[0]) { + case "select": + var r *mysql.Resultset + var err error + //for handle go mysql driver select @@max_allowed_packet + if strings.Contains(strings.ToLower(query), "max_allowed_packet") { + r, err = mysql.BuildSimpleResultset([]string{"@@max_allowed_packet"}, [][]interface{}{ + {mysql.MaxPayloadLen}, + }, binary) + } else { + r, err = mysql.BuildSimpleResultset([]string{"a", "b"}, [][]interface{}{ + {1, "hello world"}, + }, binary) + } + + if err != nil { + return nil, errors.Trace(err) + } else { + return &mysql.Result{0, 0, 0, r}, nil + } + case "insert": + return &mysql.Result{0, 1, 0, nil}, nil + case "delete": + return &mysql.Result{0, 0, 1, nil}, nil + case "update": + return &mysql.Result{0, 0, 1, nil}, nil + case "replace": + return &mysql.Result{0, 0, 1, nil}, nil + default: + return nil, fmt.Errorf("invalid query %s", query) + } + + return nil, nil +} + +func (h *testCacheHandler) HandleQuery(query string) (*mysql.Result, error) { + return h.handleQuery(query, false) +} + +func (h *testCacheHandler) HandleFieldList(table string, fieldWildcard string) ([]*mysql.Field, error) { + return nil, nil +} +func (h *testCacheHandler) HandleStmtPrepare(sql string) (params int, columns int, ctx interface{}, err error) { + return 0, 0, nil, nil +} + +func (h *testCacheHandler) HandleStmtClose(context interface{}) error { + return nil +} + +func (h *testCacheHandler) HandleStmtExecute(ctx interface{}, query string, args []interface{}) (*mysql.Result, error) { + return h.handleQuery(query, true) +} + +func (h *testCacheHandler) HandleOtherCommand(cmd byte, data []byte) error { + return mysql.NewError(mysql.ER_UNKNOWN_ERROR, fmt.Sprintf("command %d is not supported now", cmd)) +} diff --git a/vendor/github.com/siddontang/go-mysql/server/command_test.go b/vendor/github.com/siddontang/go-mysql/server/command_test.go new file mode 100644 index 0000000..34b034e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/command_test.go @@ -0,0 +1,4 @@ +package server + +// Ensure EmptyHandler implements Handler interface or cause compile time error +var _ Handler = EmptyHandler{} diff --git a/vendor/github.com/siddontang/go-mysql/server/credential_provider.go b/vendor/github.com/siddontang/go-mysql/server/credential_provider.go new file mode 100644 index 0000000..3d44eb0 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/credential_provider.go @@ -0,0 +1,45 @@ +package server + +import "sync" + +// interface for user credential provider +// hint: can be extended for more functionality +// =================================IMPORTANT NOTE=============================== +// if the password in a third-party credential provider could be updated at runtime, we have to invalidate the caching +// for 'caching_sha2_password' by calling 'func (s *Server)InvalidateCache(string, string)'. +type CredentialProvider interface { + // check if the user exists + CheckUsername(username string) (bool, error) + // get user credential + GetCredential(username string) (password string, found bool, err error) +} + +func NewInMemoryProvider() *InMemoryProvider { + return &InMemoryProvider{ + userPool: sync.Map{}, + } +} + +// implements a in memory credential provider +type InMemoryProvider struct { + userPool sync.Map // username -> password +} + +func (m *InMemoryProvider) CheckUsername(username string) (found bool, err error) { + _, ok := m.userPool.Load(username) + return ok, nil +} + +func (m *InMemoryProvider) GetCredential(username string) (password string, found bool, err error) { + v, ok := m.userPool.Load(username) + if !ok { + return "", false, nil + } + return v.(string), true, nil +} + +func (m *InMemoryProvider) AddUser(username, password string) { + m.userPool.Store(username, password) +} + +type Provider InMemoryProvider diff --git a/vendor/github.com/siddontang/go-mysql/server/example/server_example.go b/vendor/github.com/siddontang/go-mysql/server/example/server_example.go new file mode 100644 index 0000000..1efa1a3 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/example/server_example.go @@ -0,0 +1,51 @@ +package main + +import ( + "net" + + "github.com/siddontang/go-log/log" + "github.com/siddontang/go-mysql/mysql" + "github.com/siddontang/go-mysql/server" + "github.com/siddontang/go-mysql/test_util/test_keys" + + "crypto/tls" + "time" +) + +type RemoteThrottleProvider struct { + *server.InMemoryProvider + delay int // in milliseconds +} + +func (m *RemoteThrottleProvider) GetCredential(username string) (password string, found bool, err error) { + time.Sleep(time.Millisecond * time.Duration(m.delay)) + return m.InMemoryProvider.GetCredential(username) +} + +func main() { + l, _ := net.Listen("tcp", "127.0.0.1:3306") + // user either the in-memory credential provider or the remote credential provider (you can implement your own) + //inMemProvider := server.NewInMemoryProvider() + //inMemProvider.AddUser("root", "123") + remoteProvider := &RemoteThrottleProvider{server.NewInMemoryProvider(), 10 + 50} + remoteProvider.AddUser("root", "123") + var tlsConf = server.NewServerTLSConfig(test_keys.CaPem, test_keys.CertPem, test_keys.KeyPem, tls.VerifyClientCertIfGiven) + for { + c, _ := l.Accept() + go func() { + // Create a connection with user root and an empty password. + // You can use your own handler to handle command here. + svr := server.NewServer("8.0.12", mysql.DEFAULT_COLLATION_ID, mysql.AUTH_CACHING_SHA2_PASSWORD, test_keys.PubPem, tlsConf) + conn, err := server.NewCustomizedConn(c, svr, remoteProvider, server.EmptyHandler{}) + + if err != nil { + log.Errorf("Connection error: %v", err) + return + } + + for { + conn.HandleCommand() + } + }() + } +} diff --git a/vendor/github.com/siddontang/go-mysql/server/handshake_resp.go b/vendor/github.com/siddontang/go-mysql/server/handshake_resp.go new file mode 100644 index 0000000..79af6f2 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/handshake_resp.go @@ -0,0 +1,190 @@ +package server + +import ( + "bytes" + "crypto/tls" + "encoding/binary" + + "github.com/juju/errors" + . "github.com/siddontang/go-mysql/mysql" +) + +func (c *Conn) readHandshakeResponse() error { + data, pos, err := c.readFirstPart() + if err != nil { + return err + } + if pos, err = c.readUserName(data, pos); err != nil { + return err + } + authData, authLen, pos, err := c.readAuthData(data, pos) + if err != nil { + return err + } + + pos += authLen + + if pos, err = c.readDb(data, pos); err != nil { + return err + } + + pos = c.readPluginName(data, pos) + + cont, err := c.handleAuthMatch(authData, pos) + if err != nil { + return err + } + if !cont { + return nil + } + + // ignore connect attrs for now, the proxy does not support passing attrs to actual MySQL server + + // try to authenticate the client + return c.compareAuthData(c.authPluginName, authData) +} + +func (c *Conn) readFirstPart() ([]byte, int, error) { + data, err := c.ReadPacket() + if err != nil { + return nil, 0, err + } + + pos := 0 + + // check CLIENT_PROTOCOL_41 + if uint32(binary.LittleEndian.Uint16(data[:2]))&CLIENT_PROTOCOL_41 == 0 { + return nil, 0, errors.New("CLIENT_PROTOCOL_41 compatible client is required") + } + + //capability + c.capability = binary.LittleEndian.Uint32(data[:4]) + if c.capability&CLIENT_SECURE_CONNECTION == 0 { + return nil, 0, errors.New("CLIENT_SECURE_CONNECTION compatible client is required") + } + pos += 4 + + //skip max packet size + pos += 4 + + //charset, skip, if you want to use another charset, use set names + //c.collation = CollationId(data[pos]) + pos++ + + //skip reserved 23[00] + pos += 23 + + // is this a SSLRequest packet? + if len(data) == (4 + 4 + 1 + 23) { + if c.serverConf.capability&CLIENT_SSL == 0 { + return nil, 0, errors.Errorf("The host '%s' does not support SSL connections", c.RemoteAddr().String()) + } + // switch to TLS + tlsConn := tls.Server(c.Conn.Conn, c.serverConf.tlsConfig) + if err := tlsConn.Handshake(); err != nil { + return nil, 0, err + } + c.Conn.Conn = tlsConn + + // mysql handshake again + return c.readFirstPart() + } + return data, pos, nil +} + +func (c *Conn) readUserName(data []byte, pos int) (int, error) { + //user name + user := string(data[pos : pos+bytes.IndexByte(data[pos:], 0x00)]) + pos += len(user) + 1 + c.user = user + return pos, nil +} + +func (c *Conn) readDb(data []byte, pos int) (int, error) { + if c.capability&CLIENT_CONNECT_WITH_DB != 0 { + if len(data[pos:]) == 0 { + return pos, nil + } + + db := string(data[pos : pos+bytes.IndexByte(data[pos:], 0x00)]) + pos += len(db) + 1 + + if err := c.h.UseDB(db); err != nil { + return 0, err + } + } + return pos, nil +} + +func (c *Conn) readPluginName(data []byte, pos int) int { + if c.capability&CLIENT_PLUGIN_AUTH != 0 { + c.authPluginName = string(data[pos : pos+bytes.IndexByte(data[pos:], 0x00)]) + pos += len(c.authPluginName) + } else { + // The method used is Native Authentication if both CLIENT_PROTOCOL_41 and CLIENT_SECURE_CONNECTION are set, + // but CLIENT_PLUGIN_AUTH is not set, so we fallback to 'mysql_native_password' + c.authPluginName = AUTH_NATIVE_PASSWORD + } + return pos +} + +func (c *Conn) readAuthData(data []byte, pos int) ([]byte, int, int, error) { + // length encoded data + var auth []byte + var authLen int + if c.capability&CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA != 0 { + authData, isNULL, readBytes, err := LengthEncodedString(data[pos:]) + if err != nil { + return nil, 0, 0, err + } + if isNULL { + // no auth length and no auth data, just \NUL, considered invalid auth data, and reject connection as MySQL does + return nil, 0, 0, NewDefaultError(ER_ACCESS_DENIED_ERROR, c.LocalAddr().String(), c.user, "Yes") + } + auth = authData + authLen = readBytes + } else { + //auth length and auth + authLen = int(data[pos]) + pos++ + auth = data[pos : pos+authLen] + if authLen == 0 { + // skip the next \NUL in case the password is empty + pos++ + } + } + return auth, authLen, pos, nil +} + +// Public Key Retrieval +// See: https://dev.mysql.com/doc/internals/en/public-key-retrieval.html +func (c *Conn) handlePublicKeyRetrieval(authData []byte) (bool, error) { + // if the client use 'sha256_password' auth method, and request for a public key + // we send back a keyfile with Protocol::AuthMoreData + if c.authPluginName == AUTH_SHA256_PASSWORD && len(authData) == 1 && authData[0] == 0x01 { + if c.serverConf.capability&CLIENT_SSL == 0 { + return false, errors.New("server does not support SSL: CLIENT_SSL not enabled") + } + if err := c.writeAuthMoreDataPubkey(); err != nil { + return false, err + } + + return false, c.handleAuthSwitchResponse() + } + return true, nil +} + +func (c *Conn) handleAuthMatch(authData []byte, pos int) (bool, error) { + // if the client responds the handshake with a different auth method, the server will send the AuthSwitchRequest packet + // to the client to ask the client to switch. + + if c.authPluginName != c.serverConf.defaultAuthMethod { + if err := c.writeAuthSwitchRequest(c.serverConf.defaultAuthMethod); err != nil { + return false, err + } + c.authPluginName = c.serverConf.defaultAuthMethod + // handle AuthSwitchResponse + return false, c.handleAuthSwitchResponse() + } + return true, nil +} diff --git a/vendor/github.com/siddontang/go-mysql/server/initial_handshake.go b/vendor/github.com/siddontang/go-mysql/server/initial_handshake.go new file mode 100644 index 0000000..312ac2b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/initial_handshake.go @@ -0,0 +1,57 @@ +package server + +// see: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_v10.html +func (c *Conn) writeInitialHandshake() error { + data := make([]byte, 4) + + //min version 10 + data = append(data, 10) + + //server version[00] + data = append(data, c.serverConf.serverVersion...) + data = append(data, 0x00) + + //connection id + data = append(data, byte(c.connectionID), byte(c.connectionID>>8), byte(c.connectionID>>16), byte(c.connectionID>>24)) + + //auth-plugin-data-part-1 + data = append(data, c.salt[0:8]...) + + //filter 0x00 byte, terminating the first part of a scramble + data = append(data, 0x00) + + defaultFlag := c.serverConf.capability + //capability flag lower 2 bytes, using default capability here + data = append(data, byte(defaultFlag), byte(defaultFlag>>8)) + + //charset + data = append(data, c.serverConf.collationId) + + //status + data = append(data, byte(c.status), byte(c.status>>8)) + + //capability flag upper 2 bytes, using default capability here + data = append(data, byte(defaultFlag>>16), byte(defaultFlag>>24)) + + // server supports CLIENT_PLUGIN_AUTH and CLIENT_SECURE_CONNECTION + data = append(data, byte(8+12+1)) + + //reserved 10 [00] + data = append(data, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + + //auth-plugin-data-part-2 + data = append(data, c.salt[8:]...) + // second part of the password cipher [mininum 13 bytes], + // where len=MAX(13, length of auth-plugin-data - 8) + // add \NUL to terminate the string + data = append(data, 0x00) + + // auth plugin name + data = append(data, c.serverConf.defaultAuthMethod...) + + // EOF if MySQL version (>= 5.5.7 and < 5.5.10) or (>= 5.6.0 and < 5.6.2) + // \NUL otherwise, so we use \NUL + data = append(data, 0) + + return c.WritePacket(data) +} diff --git a/vendor/github.com/siddontang/go-mysql/server/server_conf.go b/vendor/github.com/siddontang/go-mysql/server/server_conf.go new file mode 100644 index 0000000..353595c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/server_conf.go @@ -0,0 +1,103 @@ +package server + +import ( + "crypto/tls" + "fmt" + "sync" + + . "github.com/siddontang/go-mysql/mysql" +) + +var defaultServer = NewDefaultServer() + +// Defines a basic MySQL server with configs. +// +// We do not aim at implementing the whole MySQL connection suite to have the best compatibilities for the clients. +// The MySQL server can be configured to switch auth methods covering 'mysql_old_password', 'mysql_native_password', +// 'mysql_clear_password', 'authentication_windows_client', 'sha256_password', 'caching_sha2_password', etc. +// +// However, since some old auth methods are considered broken with security issues. MySQL major versions like 5.7 and 8.0 default to +// 'mysql_native_password' or 'caching_sha2_password', and most MySQL clients should have already supported at least one of the three auth +// methods 'mysql_native_password', 'caching_sha2_password', and 'sha256_password'. Thus here we will only support these three +// auth methods, and use 'mysql_native_password' as default for maximum compatibility with the clients and leave the other two as +// config options. +// +// The MySQL doc states that 'mysql_old_password' will be used if 'CLIENT_PROTOCOL_41' or 'CLIENT_SECURE_CONNECTION' flag is not set. +// We choose to drop the support for insecure 'mysql_old_password' auth method and require client capability 'CLIENT_PROTOCOL_41' and 'CLIENT_SECURE_CONNECTION' +// are set. Besides, if 'CLIENT_PLUGIN_AUTH' is not set, we fallback to 'mysql_native_password' auth method. +type Server struct { + serverVersion string // e.g. "8.0.12" + protocolVersion int // minimal 10 + capability uint32 // server capability flag + collationId uint8 + defaultAuthMethod string // default authentication method, 'mysql_native_password' + pubKey []byte + tlsConfig *tls.Config + cacheShaPassword *sync.Map // 'user@host' -> SHA256(SHA256(PASSWORD)) +} + +// New mysql server with default settings. +// +// NOTES: +// TLS support will be enabled by default with auto-generated CA and server certificates (however, you can still use +// non-TLS connection). By default, it will verify the client certificate if present. You can enable TLS support on +// the client side without providing a client-side certificate. So only when you need the server to verify client +// identity for maximum security, you need to set a signed certificate for the client. +func NewDefaultServer() *Server { + caPem, caKey := generateCA() + certPem, keyPem := generateAndSignRSACerts(caPem, caKey) + tlsConf := NewServerTLSConfig(caPem, certPem, keyPem, tls.VerifyClientCertIfGiven) + return &Server{ + serverVersion: "5.7.0", + protocolVersion: 10, + capability: CLIENT_LONG_PASSWORD | CLIENT_LONG_FLAG | CLIENT_CONNECT_WITH_DB | CLIENT_PROTOCOL_41 | + CLIENT_TRANSACTIONS | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_SSL | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA, + collationId: DEFAULT_COLLATION_ID, + defaultAuthMethod: AUTH_NATIVE_PASSWORD, + pubKey: getPublicKeyFromCert(certPem), + tlsConfig: tlsConf, + cacheShaPassword: new(sync.Map), + } +} + +// New mysql server with customized settings. +// +// NOTES: +// You can control the authentication methods and TLS settings here. +// For auth method, you can specify one of the supported methods 'mysql_native_password', 'caching_sha2_password', and 'sha256_password'. +// The specified auth method will be enforced by the server in the connection phase. That means, client will be asked to switch auth method +// if the supplied auth method is different from the server default. +// And for TLS support, you can specify self-signed or CA-signed certificates and decide whether the client needs to provide +// a signed or unsigned certificate to provide different level of security. +func NewServer(serverVersion string, collationId uint8, defaultAuthMethod string, pubKey []byte, tlsConfig *tls.Config) *Server { + if !isAuthMethodSupported(defaultAuthMethod) { + panic(fmt.Sprintf("server authentication method '%s' is not supported", defaultAuthMethod)) + } + + //if !isAuthMethodAllowedByServer(defaultAuthMethod, allowedAuthMethods) { + // panic(fmt.Sprintf("default auth method is not one of the allowed auth methods")) + //} + var capFlag = CLIENT_LONG_PASSWORD | CLIENT_LONG_FLAG | CLIENT_CONNECT_WITH_DB | CLIENT_PROTOCOL_41 | + CLIENT_TRANSACTIONS | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA + if tlsConfig != nil { + capFlag |= CLIENT_SSL + } + return &Server{ + serverVersion: serverVersion, + protocolVersion: 10, + capability: capFlag, + collationId: collationId, + defaultAuthMethod: defaultAuthMethod, + pubKey: pubKey, + tlsConfig: tlsConfig, + cacheShaPassword: new(sync.Map), + } +} + +func isAuthMethodSupported(authMethod string) bool { + return authMethod == AUTH_NATIVE_PASSWORD || authMethod == AUTH_CACHING_SHA2_PASSWORD || authMethod == AUTH_SHA256_PASSWORD +} + +func (s *Server) InvalidateCache(username string, host string) { + s.cacheShaPassword.Delete(fmt.Sprintf("%s@%s", username, host)) +} diff --git a/vendor/github.com/siddontang/go-mysql/server/ssl.go b/vendor/github.com/siddontang/go-mysql/server/ssl.go new file mode 100644 index 0000000..1f8a9ed --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/server/ssl.go @@ -0,0 +1,133 @@ +package server + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "time" +) + +// generate TLS config for server side +// controlling the security level by authType +func NewServerTLSConfig(caPem, certPem, keyPem []byte, authType tls.ClientAuthType) *tls.Config { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPem) { + panic("failed to add ca PEM") + } + + cert, err := tls.X509KeyPair(certPem, keyPem) + if err != nil { + panic(err) + } + + config := &tls.Config{ + ClientAuth: authType, + Certificates: []tls.Certificate{cert}, + ClientCAs: pool, + } + return config +} + +// extract RSA public key from certificate +func getPublicKeyFromCert(certPem []byte) []byte { + block, _ := pem.Decode(certPem) + crt, err := x509.ParseCertificate(block.Bytes) + if err != nil { + panic(err) + } + pubKey, err := x509.MarshalPKIXPublicKey(crt.PublicKey.(*rsa.PublicKey)) + if err != nil { + panic(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubKey}) +} + +// generate and sign RSA certificates with given CA +// see: https://fale.io/blog/2017/06/05/create-a-pki-in-golang/ +func generateAndSignRSACerts(caPem, caKey []byte) ([]byte, []byte) { + // Load CA + catls, err := tls.X509KeyPair(caPem, caKey) + if err != nil { + panic(err) + } + ca, err := x509.ParseCertificate(catls.Certificate[0]) + if err != nil { + panic(err) + } + + // use the CA to sign certificates + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + panic(err) + } + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"ORGANIZATION_NAME"}, + Country: []string{"COUNTRY_CODE"}, + Province: []string{"PROVINCE"}, + Locality: []string{"CITY"}, + StreetAddress: []string{"ADDRESS"}, + PostalCode: []string{"POSTAL_CODE"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + priv, _ := rsa.GenerateKey(rand.Reader, 2048) + + // sign the certificate + cert_b, err := x509.CreateCertificate(rand.Reader, ca, cert, &priv.PublicKey, catls.PrivateKey) + if err != nil { + panic(err) + } + certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert_b}) + keyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + + return certPem, keyPem +} + +// generate CA in PEM +// see: https://github.com/golang/go/blob/master/src/crypto/tls/generate_cert.go +func generateCA() ([]byte, []byte) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + panic(err) + } + template := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"ORGANIZATION_NAME"}, + Country: []string{"COUNTRY_CODE"}, + Province: []string{"PROVINCE"}, + Locality: []string{"CITY"}, + StreetAddress: []string{"ADDRESS"}, + PostalCode: []string{"POSTAL_CODE"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign | x509.KeyUsageKeyEncipherment, + BasicConstraintsValid: true, + } + + priv, _ := rsa.GenerateKey(rand.Reader, 2048) + derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv) + if err != nil { + panic(err) + } + + caPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + caKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + + return caPem, caKey +} diff --git a/vendor/github.com/siddontang/go-mysql/test_util/test_keys/keys.go b/vendor/github.com/siddontang/go-mysql/test_util/test_keys/keys.go new file mode 100644 index 0000000..c1049b6 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/test_util/test_keys/keys.go @@ -0,0 +1,85 @@ +package test_keys + +// here we put the testing encryption keys here +// NOTE THIS IS FOR TESTING ONLY, DO NOT USE THEM IN PRODUCTION! + +var PubPem = []byte(`-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsraCori69OXEkA07Ykp2 +Ju+aHz33PqgKj0qbSbPm6ePh2mer2GWOxC4q1wdRwzgddwTTqSdonhM4XuVyyNqq +gM7uv9JoWCONcKo28cPRK7gH7up7nYFllNFXUAA0/XQ+95tqtdITNplQLIceFIXz +5Bvi9fThcpf9M6qKdNUa2Wd24rM/n6qxoUG2ksDDVXQC30RAHkGCdNi10iya8Pj/ +ZaEG86NXFpvvnLHRHiih/gXe7nby1sR6BxaEG2bLZd0cjdL5MuWOPeQ450H6mCtV +SX4poNq9YrdP4XW9M0N7nocRU0p5aUvLWxy6XrUTSP0iRkC7ppEPG0p2Xtsq7QGT +MwIDAQAB +-----END PUBLIC KEY-----`) + +var CertPem = []byte(`-----BEGIN CERTIFICATE----- +MIIDBjCCAe4CCQDg06wCf7hcuDANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 +cyBQdHkgTHRkMB4XDTE4MDgxOTA4NDUyNVoXDTI4MDgxNjA4NDUyNVowRTELMAkG +A1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0 +IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALK2gqK4uvTlxJANO2JKdibvmh899z6oCo9Km0mz5unj4dpnq9hljsQuKtcHUcM4 +HXcE06knaJ4TOF7lcsjaqoDO7r/SaFgjjXCqNvHD0Su4B+7qe52BZZTRV1AANP10 +PvebarXSEzaZUCyHHhSF8+Qb4vX04XKX/TOqinTVGtlnduKzP5+qsaFBtpLAw1V0 +At9EQB5BgnTYtdIsmvD4/2WhBvOjVxab75yx0R4oof4F3u528tbEegcWhBtmy2Xd +HI3S+TLljj3kOOdB+pgrVUl+KaDavWK3T+F1vTNDe56HEVNKeWlLy1scul61E0j9 +IkZAu6aRDxtKdl7bKu0BkzMCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAma3yFqR7 +xkeaZBg4/1I3jSlaNe5+2JB4iybAkMOu77fG5zytLomTbzdhewsuBwpTVMJdga8T +IdPeIFCin1U+5SkbjSMlpKf+krE+5CyrNJ5jAgO9ATIqx66oCTYXfGlNapGRLfSE +sa0iMqCe/dr4GPU+flW2DZFWiyJVDSF1JjReQnfrWY+SD2SpP/lmlgltnY8MJngd +xBLG5nsZCpUXGB713Q8ZyIm2ThVAMiskcxBleIZDDghLuhGvY/9eFJhZpvOkjWa6 +XGEi4E1G/SA+zVKFl41nHKCdqXdmIOnpcLlFBUVloQok5a95Kqc1TYw3f+WbdFff +99dAgk3gWwWZQA== +-----END CERTIFICATE-----`) + +var KeyPem = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAsraCori69OXEkA07Ykp2Ju+aHz33PqgKj0qbSbPm6ePh2mer +2GWOxC4q1wdRwzgddwTTqSdonhM4XuVyyNqqgM7uv9JoWCONcKo28cPRK7gH7up7 +nYFllNFXUAA0/XQ+95tqtdITNplQLIceFIXz5Bvi9fThcpf9M6qKdNUa2Wd24rM/ +n6qxoUG2ksDDVXQC30RAHkGCdNi10iya8Pj/ZaEG86NXFpvvnLHRHiih/gXe7nby +1sR6BxaEG2bLZd0cjdL5MuWOPeQ450H6mCtVSX4poNq9YrdP4XW9M0N7nocRU0p5 +aUvLWxy6XrUTSP0iRkC7ppEPG0p2Xtsq7QGTMwIDAQABAoIBAGh1m8hHWCg7gXh9 +838RbRx3IswuKS27hWiaQEiFWmzOIb7KqDy1qAxtu+ayRY1paHegH6QY/+Kd824s +ibpzbgQacJ04/HrAVTVMmQ8Z2VLHoAN7lcPL1bd14aZGaLLZVtDeTDJ413grhxxv +4ho27gcgcbo4Z+rWgk7H2WRPCAGYqWYAycm3yF5vy9QaO6edU+T588YsEQOos5iy +5pVFSGDGZkcUp1ukL3BJYR+jvygn6WPCobQ/LScUdi+ucitaI9i+UdlLokZARVRG +M/msqcTM73thR8yVRcexU6NUDxRBfZ/f7moSAEbBmGDXuxDcIyH9KGMQ2rMtN1X3 +lK8UNwkCgYEA2STJq/IUQHjdqd3Dqh/Q7Zm8/pMWFqLJSkqpnFtFsXPyUOx9zDOy +KqkIfGeyKwvsj9X9BcZ0FUKj9zoct1/WpPY+h7i7+z0MIujBh4AMjAcDrt4o76yK +UHuVmG2xKTdJoAbqOdToQeX6E82Ioal5pbB2W7AbCQScNBPZ52jxgtcCgYEA0rE7 +2dFiRm0YmuszFYxft2+GP6NgP3R2TQNEooi1uCXG2xgwObie1YCHzpZ5CfSqJIxP +XB7DXpIWi7PxJoeai2F83LnmdFz6F1BPRobwDoSFNdaSKLg4Yf856zpgYNKhL1fE +OoOXj4VBWBZh1XDfZV44fgwlMIf7edOF1XOagwUCgYAw953O+7FbdKYwF0V3iOM5 +oZDAK/UwN5eC/GFRVDfcM5RycVJRCVtlSWcTfuLr2C2Jpiz/72fgH34QU3eEVsV1 +v94MBznFB1hESw7ReqvZq/9FoO3EVrl+OtBaZmosLD6bKtQJJJ0Xtz/01UW5hxla +pveZ55XBK9v51nwuNjk4UwKBgHD8fJUllSchUCWb5cwzeAz98Kdl7LJ6uQo5q2/i +EllLYOWThiEeIYdrIuklholRPIDXAaPsF2c6vn5yo+q+o6EFSZlw0+YpCjDAb5Lp +wAh5BprFk6HkkM/0t9Guf4rMyYWC8odSlE9x7YXYkuSMYDCTI4Zs6vCoq7I8PbQn +B4AlAoGAZ6Ee5m/ph5UVp/3+cR6jCY7aHBUU/M3pbJSkVjBW+ymEBVJ6sUdz8k3P +x8BiPEQggNN7faWBqRWP7KXPnDYHh6shYUgPJwI5HX6NE/ZDnnXjeysHRyf0oCo5 +S6tHXwHNKB5HS1c/KDyyNGjP2oi/MF4o/MGWNWEcK6TJA3RGOYM= +-----END RSA PRIVATE KEY-----`) + +var CaPem = []byte(`-----BEGIN CERTIFICATE----- +MIIDtTCCAp2gAwIBAgIJANeS1FOzWXlZMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTgwODE2MTUxNDE5WhcNMjEwNjA1MTUxNDE5WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAsV6xlhFxMn14Pn7XBRGLt8/HXmhVVu20IKFgIOyX7gAZr0QLsuT1fGf5 +zH9HrlgOMkfdhV847U03KPfUnBsi9lS6/xOxnH/OzTYM0WW0eNMGF7eoxrS64GSb +PVX4pLi5+uwrrZT5HmDgZi49ANmuX6UYmH/eRRvSIoYUTV6t0aYsLyKvlpEAtRAe +4AlKB236j5ggmJ36QUhTFTbeNbeOOgloTEdPK8Y/kgpnhiqzMdPqqIc7IeXUc456 +yX8MJUgniTM2qCNTFdEw+C2Ok0RbU6TI2SuEgVF4jtCcVEKxZ8kYbioONaePQKFR +/EhdXO+/ag1IEdXElH9knLOfB+zCgwIDAQABo4GnMIGkMB0GA1UdDgQWBBQgHiwD +00upIbCOunlK4HRw89DhjjB1BgNVHSMEbjBsgBQgHiwD00upIbCOunlK4HRw89Dh +jqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNV +BAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJANeS1FOzWXlZMAwGA1UdEwQF +MAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAFMZFQTFKU5tWIpWh8BbVZeVZcng0Kiq +qwbhVwaTkqtfmbqw8/w+faOWylmLncQEMmgvnUltGMQlQKBwQM2byzPkz9phal3g +uI0JWJYqtcMyIQUB9QbbhrDNC9kdt/ji/x6rrIqzaMRuiBXqH5LQ9h856yXzArqd +cAQGzzYpbUCIv7ciSB93cKkU73fQLZVy5ZBy1+oAa1V9U4cb4G/20/PDmT+G3Gxz +pEjeDKtz8XINoWgA2cSdfAhNZt5vqJaCIZ8qN0z6C7SUKwUBderERUMLUXdhUldC +KTVHyEPvd0aULd5S5vEpKCnHcQmFcLdoN8t9k9pR9ZgwqXbyJHlxWFo= +-----END CERTIFICATE-----`) diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/COPYING new file mode 100644 index 0000000..5a8e332 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING new file mode 100644 index 0000000..5a8e332 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING new file mode 100644 index 0000000..5a8e332 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING new file mode 100644 index 0000000..5a8e332 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode.go new file mode 100644 index 0000000..b0fd51d --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode.go @@ -0,0 +1,509 @@ +package toml + +import ( + "fmt" + "io" + "io/ioutil" + "math" + "reflect" + "strings" + "time" +) + +func e(format string, args ...interface{}) error { + return fmt.Errorf("toml: "+format, args...) +} + +// Unmarshaler is the interface implemented by objects that can unmarshal a +// TOML description of themselves. +type Unmarshaler interface { + UnmarshalTOML(interface{}) error +} + +// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`. +func Unmarshal(p []byte, v interface{}) error { + _, err := Decode(string(p), v) + return err +} + +// Primitive is a TOML value that hasn't been decoded into a Go value. +// When using the various `Decode*` functions, the type `Primitive` may +// be given to any value, and its decoding will be delayed. +// +// A `Primitive` value can be decoded using the `PrimitiveDecode` function. +// +// The underlying representation of a `Primitive` value is subject to change. +// Do not rely on it. +// +// N.B. Primitive values are still parsed, so using them will only avoid +// the overhead of reflection. They can be useful when you don't know the +// exact type of TOML data until run time. +type Primitive struct { + undecoded interface{} + context Key +} + +// DEPRECATED! +// +// Use MetaData.PrimitiveDecode instead. +func PrimitiveDecode(primValue Primitive, v interface{}) error { + md := MetaData{decoded: make(map[string]bool)} + return md.unify(primValue.undecoded, rvalue(v)) +} + +// PrimitiveDecode is just like the other `Decode*` functions, except it +// decodes a TOML value that has already been parsed. Valid primitive values +// can *only* be obtained from values filled by the decoder functions, +// including this method. (i.e., `v` may contain more `Primitive` +// values.) +// +// Meta data for primitive values is included in the meta data returned by +// the `Decode*` functions with one exception: keys returned by the Undecoded +// method will only reflect keys that were decoded. Namely, any keys hidden +// behind a Primitive will be considered undecoded. Executing this method will +// update the undecoded keys in the meta data. (See the example.) +func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { + md.context = primValue.context + defer func() { md.context = nil }() + return md.unify(primValue.undecoded, rvalue(v)) +} + +// Decode will decode the contents of `data` in TOML format into a pointer +// `v`. +// +// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be +// used interchangeably.) +// +// TOML arrays of tables correspond to either a slice of structs or a slice +// of maps. +// +// TOML datetimes correspond to Go `time.Time` values. +// +// All other TOML types (float, string, int, bool and array) correspond +// to the obvious Go types. +// +// An exception to the above rules is if a type implements the +// encoding.TextUnmarshaler interface. In this case, any primitive TOML value +// (floats, strings, integers, booleans and datetimes) will be converted to +// a byte string and given to the value's UnmarshalText method. See the +// Unmarshaler example for a demonstration with time duration strings. +// +// Key mapping +// +// TOML keys can map to either keys in a Go map or field names in a Go +// struct. The special `toml` struct tag may be used to map TOML keys to +// struct fields that don't match the key name exactly. (See the example.) +// A case insensitive match to struct names will be tried if an exact match +// can't be found. +// +// The mapping between TOML values and Go values is loose. That is, there +// may exist TOML values that cannot be placed into your representation, and +// there may be parts of your representation that do not correspond to +// TOML values. This loose mapping can be made stricter by using the IsDefined +// and/or Undecoded methods on the MetaData returned. +// +// This decoder will not handle cyclic types. If a cyclic type is passed, +// `Decode` will not terminate. +func Decode(data string, v interface{}) (MetaData, error) { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v)) + } + if rv.IsNil() { + return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v)) + } + p, err := parse(data) + if err != nil { + return MetaData{}, err + } + md := MetaData{ + p.mapping, p.types, p.ordered, + make(map[string]bool, len(p.ordered)), nil, + } + return md, md.unify(p.mapping, indirect(rv)) +} + +// DecodeFile is just like Decode, except it will automatically read the +// contents of the file at `fpath` and decode it for you. +func DecodeFile(fpath string, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadFile(fpath) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// DecodeReader is just like Decode, except it will consume all bytes +// from the reader and decode it for you. +func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadAll(r) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// unify performs a sort of type unification based on the structure of `rv`, +// which is the client representation. +// +// Any type mismatch produces an error. Finding a type that we don't know +// how to handle produces an unsupported type error. +func (md *MetaData) unify(data interface{}, rv reflect.Value) error { + + // Special case. Look for a `Primitive` value. + if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { + // Save the undecoded data and the key context into the primitive + // value. + context := make(Key, len(md.context)) + copy(context, md.context) + rv.Set(reflect.ValueOf(Primitive{ + undecoded: data, + context: context, + })) + return nil + } + + // Special case. Unmarshaler Interface support. + if rv.CanAddr() { + if v, ok := rv.Addr().Interface().(Unmarshaler); ok { + return v.UnmarshalTOML(data) + } + } + + // Special case. Handle time.Time values specifically. + // TODO: Remove this code when we decide to drop support for Go 1.1. + // This isn't necessary in Go 1.2 because time.Time satisfies the encoding + // interfaces. + if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { + return md.unifyDatetime(data, rv) + } + + // Special case. Look for a value satisfying the TextUnmarshaler interface. + if v, ok := rv.Interface().(TextUnmarshaler); ok { + return md.unifyText(data, v) + } + // BUG(burntsushi) + // The behavior here is incorrect whenever a Go type satisfies the + // encoding.TextUnmarshaler interface but also corresponds to a TOML + // hash or array. In particular, the unmarshaler should only be applied + // to primitive TOML values. But at this point, it will be applied to + // all kinds of values and produce an incorrect error whenever those values + // are hashes or arrays (including arrays of tables). + + k := rv.Kind() + + // laziness + if k >= reflect.Int && k <= reflect.Uint64 { + return md.unifyInt(data, rv) + } + switch k { + case reflect.Ptr: + elem := reflect.New(rv.Type().Elem()) + err := md.unify(data, reflect.Indirect(elem)) + if err != nil { + return err + } + rv.Set(elem) + return nil + case reflect.Struct: + return md.unifyStruct(data, rv) + case reflect.Map: + return md.unifyMap(data, rv) + case reflect.Array: + return md.unifyArray(data, rv) + case reflect.Slice: + return md.unifySlice(data, rv) + case reflect.String: + return md.unifyString(data, rv) + case reflect.Bool: + return md.unifyBool(data, rv) + case reflect.Interface: + // we only support empty interfaces. + if rv.NumMethod() > 0 { + return e("unsupported type %s", rv.Type()) + } + return md.unifyAnything(data, rv) + case reflect.Float32: + fallthrough + case reflect.Float64: + return md.unifyFloat64(data, rv) + } + return e("unsupported type %s", rv.Kind()) +} + +func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + if mapping == nil { + return nil + } + return e("type mismatch for %s: expected table but found %T", + rv.Type().String(), mapping) + } + + for key, datum := range tmap { + var f *field + fields := cachedTypeFields(rv.Type()) + for i := range fields { + ff := &fields[i] + if ff.name == key { + f = ff + break + } + if f == nil && strings.EqualFold(ff.name, key) { + f = ff + } + } + if f != nil { + subv := rv + for _, i := range f.index { + subv = indirect(subv.Field(i)) + } + if isUnifiable(subv) { + md.decoded[md.context.add(key).String()] = true + md.context = append(md.context, key) + if err := md.unify(datum, subv); err != nil { + return err + } + md.context = md.context[0 : len(md.context)-1] + } else if f.name != "" { + // Bad user! No soup for you! + return e("cannot write unexported field %s.%s", + rv.Type().String(), f.name) + } + } + } + return nil +} + +func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + if tmap == nil { + return nil + } + return badtype("map", mapping) + } + if rv.IsNil() { + rv.Set(reflect.MakeMap(rv.Type())) + } + for k, v := range tmap { + md.decoded[md.context.add(k).String()] = true + md.context = append(md.context, k) + + rvkey := indirect(reflect.New(rv.Type().Key())) + rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) + if err := md.unify(v, rvval); err != nil { + return err + } + md.context = md.context[0 : len(md.context)-1] + + rvkey.SetString(k) + rv.SetMapIndex(rvkey, rvval) + } + return nil +} + +func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { + datav := reflect.ValueOf(data) + if datav.Kind() != reflect.Slice { + if !datav.IsValid() { + return nil + } + return badtype("slice", data) + } + sliceLen := datav.Len() + if sliceLen != rv.Len() { + return e("expected array length %d; got TOML array of length %d", + rv.Len(), sliceLen) + } + return md.unifySliceArray(datav, rv) +} + +func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { + datav := reflect.ValueOf(data) + if datav.Kind() != reflect.Slice { + if !datav.IsValid() { + return nil + } + return badtype("slice", data) + } + n := datav.Len() + if rv.IsNil() || rv.Cap() < n { + rv.Set(reflect.MakeSlice(rv.Type(), n, n)) + } + rv.SetLen(n) + return md.unifySliceArray(datav, rv) +} + +func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { + sliceLen := data.Len() + for i := 0; i < sliceLen; i++ { + v := data.Index(i).Interface() + sliceval := indirect(rv.Index(i)) + if err := md.unify(v, sliceval); err != nil { + return err + } + } + return nil +} + +func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { + if _, ok := data.(time.Time); ok { + rv.Set(reflect.ValueOf(data)) + return nil + } + return badtype("time.Time", data) +} + +func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { + if s, ok := data.(string); ok { + rv.SetString(s) + return nil + } + return badtype("string", data) +} + +func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { + if num, ok := data.(float64); ok { + switch rv.Kind() { + case reflect.Float32: + fallthrough + case reflect.Float64: + rv.SetFloat(num) + default: + panic("bug") + } + return nil + } + return badtype("float", data) +} + +func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { + if num, ok := data.(int64); ok { + if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 { + switch rv.Kind() { + case reflect.Int, reflect.Int64: + // No bounds checking necessary. + case reflect.Int8: + if num < math.MinInt8 || num > math.MaxInt8 { + return e("value %d is out of range for int8", num) + } + case reflect.Int16: + if num < math.MinInt16 || num > math.MaxInt16 { + return e("value %d is out of range for int16", num) + } + case reflect.Int32: + if num < math.MinInt32 || num > math.MaxInt32 { + return e("value %d is out of range for int32", num) + } + } + rv.SetInt(num) + } else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 { + unum := uint64(num) + switch rv.Kind() { + case reflect.Uint, reflect.Uint64: + // No bounds checking necessary. + case reflect.Uint8: + if num < 0 || unum > math.MaxUint8 { + return e("value %d is out of range for uint8", num) + } + case reflect.Uint16: + if num < 0 || unum > math.MaxUint16 { + return e("value %d is out of range for uint16", num) + } + case reflect.Uint32: + if num < 0 || unum > math.MaxUint32 { + return e("value %d is out of range for uint32", num) + } + } + rv.SetUint(unum) + } else { + panic("unreachable") + } + return nil + } + return badtype("integer", data) +} + +func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { + if b, ok := data.(bool); ok { + rv.SetBool(b) + return nil + } + return badtype("boolean", data) +} + +func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { + rv.Set(reflect.ValueOf(data)) + return nil +} + +func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error { + var s string + switch sdata := data.(type) { + case TextMarshaler: + text, err := sdata.MarshalText() + if err != nil { + return err + } + s = string(text) + case fmt.Stringer: + s = sdata.String() + case string: + s = sdata + case bool: + s = fmt.Sprintf("%v", sdata) + case int64: + s = fmt.Sprintf("%d", sdata) + case float64: + s = fmt.Sprintf("%f", sdata) + default: + return badtype("primitive (string-like)", data) + } + if err := v.UnmarshalText([]byte(s)); err != nil { + return err + } + return nil +} + +// rvalue returns a reflect.Value of `v`. All pointers are resolved. +func rvalue(v interface{}) reflect.Value { + return indirect(reflect.ValueOf(v)) +} + +// indirect returns the value pointed to by a pointer. +// Pointers are followed until the value is not a pointer. +// New values are allocated for each nil pointer. +// +// An exception to this rule is if the value satisfies an interface of +// interest to us (like encoding.TextUnmarshaler). +func indirect(v reflect.Value) reflect.Value { + if v.Kind() != reflect.Ptr { + if v.CanSet() { + pv := v.Addr() + if _, ok := pv.Interface().(TextUnmarshaler); ok { + return pv + } + } + return v + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + return indirect(reflect.Indirect(v)) +} + +func isUnifiable(rv reflect.Value) bool { + if rv.CanSet() { + return true + } + if _, ok := rv.Interface().(TextUnmarshaler); ok { + return true + } + return false +} + +func badtype(expected string, data interface{}) error { + return e("cannot load TOML value of type %T into a Go %s", data, expected) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode_meta.go new file mode 100644 index 0000000..b9914a6 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/decode_meta.go @@ -0,0 +1,121 @@ +package toml + +import "strings" + +// MetaData allows access to meta information about TOML data that may not +// be inferrable via reflection. In particular, whether a key has been defined +// and the TOML type of a key. +type MetaData struct { + mapping map[string]interface{} + types map[string]tomlType + keys []Key + decoded map[string]bool + context Key // Used only during decoding. +} + +// IsDefined returns true if the key given exists in the TOML data. The key +// should be specified hierarchially. e.g., +// +// // access the TOML key 'a.b.c' +// IsDefined("a", "b", "c") +// +// IsDefined will return false if an empty key given. Keys are case sensitive. +func (md *MetaData) IsDefined(key ...string) bool { + if len(key) == 0 { + return false + } + + var hash map[string]interface{} + var ok bool + var hashOrVal interface{} = md.mapping + for _, k := range key { + if hash, ok = hashOrVal.(map[string]interface{}); !ok { + return false + } + if hashOrVal, ok = hash[k]; !ok { + return false + } + } + return true +} + +// Type returns a string representation of the type of the key specified. +// +// Type will return the empty string if given an empty key or a key that +// does not exist. Keys are case sensitive. +func (md *MetaData) Type(key ...string) string { + fullkey := strings.Join(key, ".") + if typ, ok := md.types[fullkey]; ok { + return typ.typeString() + } + return "" +} + +// Key is the type of any TOML key, including key groups. Use (MetaData).Keys +// to get values of this type. +type Key []string + +func (k Key) String() string { + return strings.Join(k, ".") +} + +func (k Key) maybeQuotedAll() string { + var ss []string + for i := range k { + ss = append(ss, k.maybeQuoted(i)) + } + return strings.Join(ss, ".") +} + +func (k Key) maybeQuoted(i int) string { + quote := false + for _, c := range k[i] { + if !isBareKeyChar(c) { + quote = true + break + } + } + if quote { + return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" + } + return k[i] +} + +func (k Key) add(piece string) Key { + newKey := make(Key, len(k)+1) + copy(newKey, k) + newKey[len(k)] = piece + return newKey +} + +// Keys returns a slice of every key in the TOML data, including key groups. +// Each key is itself a slice, where the first element is the top of the +// hierarchy and the last is the most specific. +// +// The list will have the same order as the keys appeared in the TOML data. +// +// All keys returned are non-empty. +func (md *MetaData) Keys() []Key { + return md.keys +} + +// Undecoded returns all keys that have not been decoded in the order in which +// they appear in the original TOML document. +// +// This includes keys that haven't been decoded because of a Primitive value. +// Once the Primitive value is decoded, the keys will be considered decoded. +// +// Also note that decoding into an empty interface will result in no decoding, +// and so no keys will be considered decoded. +// +// In this sense, the Undecoded keys correspond to keys in the TOML document +// that do not have a concrete type in your representation. +func (md *MetaData) Undecoded() []Key { + undecoded := make([]Key, 0, len(md.keys)) + for _, key := range md.keys { + if !md.decoded[key.String()] { + undecoded = append(undecoded, key) + } + } + return undecoded +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/doc.go new file mode 100644 index 0000000..b371f39 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/doc.go @@ -0,0 +1,27 @@ +/* +Package toml provides facilities for decoding and encoding TOML configuration +files via reflection. There is also support for delaying decoding with +the Primitive type, and querying the set of keys in a TOML document with the +MetaData type. + +The specification implemented: https://github.com/toml-lang/toml + +The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify +whether a file is a valid TOML document. It can also be used to print the +type of each key in a TOML document. + +Testing + +There are two important types of tests used for this package. The first is +contained inside '*_test.go' files and uses the standard Go unit testing +framework. These tests are primarily devoted to holistically testing the +decoder and encoder. + +The second type of testing is used to verify the implementation's adherence +to the TOML specification. These tests have been factored into their own +project: https://github.com/BurntSushi/toml-test + +The reason the tests are in a separate project is so that they can be used by +any implementation of TOML. Namely, it is language agnostic. +*/ +package toml diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encode.go new file mode 100644 index 0000000..d905c21 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encode.go @@ -0,0 +1,568 @@ +package toml + +import ( + "bufio" + "errors" + "fmt" + "io" + "reflect" + "sort" + "strconv" + "strings" + "time" +) + +type tomlEncodeError struct{ error } + +var ( + errArrayMixedElementTypes = errors.New( + "toml: cannot encode array with mixed element types") + errArrayNilElement = errors.New( + "toml: cannot encode array with nil element") + errNonString = errors.New( + "toml: cannot encode a map with non-string key type") + errAnonNonStruct = errors.New( + "toml: cannot encode an anonymous field that is not a struct") + errArrayNoTable = errors.New( + "toml: TOML array element cannot contain a table") + errNoKey = errors.New( + "toml: top-level values must be Go maps or structs") + errAnything = errors.New("") // used in testing +) + +var quotedReplacer = strings.NewReplacer( + "\t", "\\t", + "\n", "\\n", + "\r", "\\r", + "\"", "\\\"", + "\\", "\\\\", +) + +// Encoder controls the encoding of Go values to a TOML document to some +// io.Writer. +// +// The indentation level can be controlled with the Indent field. +type Encoder struct { + // A single indentation level. By default it is two spaces. + Indent string + + // hasWritten is whether we have written any output to w yet. + hasWritten bool + w *bufio.Writer +} + +// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer +// given. By default, a single indentation level is 2 spaces. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + w: bufio.NewWriter(w), + Indent: " ", + } +} + +// Encode writes a TOML representation of the Go value to the underlying +// io.Writer. If the value given cannot be encoded to a valid TOML document, +// then an error is returned. +// +// The mapping between Go values and TOML values should be precisely the same +// as for the Decode* functions. Similarly, the TextMarshaler interface is +// supported by encoding the resulting bytes as strings. (If you want to write +// arbitrary binary data then you will need to use something like base64 since +// TOML does not have any binary types.) +// +// When encoding TOML hashes (i.e., Go maps or structs), keys without any +// sub-hashes are encoded first. +// +// If a Go map is encoded, then its keys are sorted alphabetically for +// deterministic output. More control over this behavior may be provided if +// there is demand for it. +// +// Encoding Go values without a corresponding TOML representation---like map +// types with non-string keys---will cause an error to be returned. Similarly +// for mixed arrays/slices, arrays/slices with nil elements, embedded +// non-struct types and nested slices containing maps or structs. +// (e.g., [][]map[string]string is not allowed but []map[string]string is OK +// and so is []map[string][]string.) +func (enc *Encoder) Encode(v interface{}) error { + rv := eindirect(reflect.ValueOf(v)) + if err := enc.safeEncode(Key([]string{}), rv); err != nil { + return err + } + return enc.w.Flush() +} + +func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { + defer func() { + if r := recover(); r != nil { + if terr, ok := r.(tomlEncodeError); ok { + err = terr.error + return + } + panic(r) + } + }() + enc.encode(key, rv) + return nil +} + +func (enc *Encoder) encode(key Key, rv reflect.Value) { + // Special case. Time needs to be in ISO8601 format. + // Special case. If we can marshal the type to text, then we used that. + // Basically, this prevents the encoder for handling these types as + // generic structs (or whatever the underlying type of a TextMarshaler is). + switch rv.Interface().(type) { + case time.Time, TextMarshaler: + enc.keyEqElement(key, rv) + return + } + + k := rv.Kind() + switch k { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.String, reflect.Bool: + enc.keyEqElement(key, rv) + case reflect.Array, reflect.Slice: + if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) { + enc.eArrayOfTables(key, rv) + } else { + enc.keyEqElement(key, rv) + } + case reflect.Interface: + if rv.IsNil() { + return + } + enc.encode(key, rv.Elem()) + case reflect.Map: + if rv.IsNil() { + return + } + enc.eTable(key, rv) + case reflect.Ptr: + if rv.IsNil() { + return + } + enc.encode(key, rv.Elem()) + case reflect.Struct: + enc.eTable(key, rv) + default: + panic(e("unsupported type for key '%s': %s", key, k)) + } +} + +// eElement encodes any value that can be an array element (primitives and +// arrays). +func (enc *Encoder) eElement(rv reflect.Value) { + switch v := rv.Interface().(type) { + case time.Time: + // Special case time.Time as a primitive. Has to come before + // TextMarshaler below because time.Time implements + // encoding.TextMarshaler, but we need to always use UTC. + enc.wf(v.UTC().Format("2006-01-02T15:04:05Z")) + return + case TextMarshaler: + // Special case. Use text marshaler if it's available for this value. + if s, err := v.MarshalText(); err != nil { + encPanic(err) + } else { + enc.writeQuoted(string(s)) + } + return + } + switch rv.Kind() { + case reflect.Bool: + enc.wf(strconv.FormatBool(rv.Bool())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64: + enc.wf(strconv.FormatInt(rv.Int(), 10)) + case reflect.Uint, reflect.Uint8, reflect.Uint16, + reflect.Uint32, reflect.Uint64: + enc.wf(strconv.FormatUint(rv.Uint(), 10)) + case reflect.Float32: + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) + case reflect.Float64: + enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) + case reflect.Array, reflect.Slice: + enc.eArrayOrSliceElement(rv) + case reflect.Interface: + enc.eElement(rv.Elem()) + case reflect.String: + enc.writeQuoted(rv.String()) + default: + panic(e("unexpected primitive type: %s", rv.Kind())) + } +} + +// By the TOML spec, all floats must have a decimal with at least one +// number on either side. +func floatAddDecimal(fstr string) string { + if !strings.Contains(fstr, ".") { + return fstr + ".0" + } + return fstr +} + +func (enc *Encoder) writeQuoted(s string) { + enc.wf("\"%s\"", quotedReplacer.Replace(s)) +} + +func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { + length := rv.Len() + enc.wf("[") + for i := 0; i < length; i++ { + elem := rv.Index(i) + enc.eElement(elem) + if i != length-1 { + enc.wf(", ") + } + } + enc.wf("]") +} + +func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) + } + for i := 0; i < rv.Len(); i++ { + trv := rv.Index(i) + if isNil(trv) { + continue + } + panicIfInvalidKey(key) + enc.newline() + enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll()) + enc.newline() + enc.eMapOrStruct(key, trv) + } +} + +func (enc *Encoder) eTable(key Key, rv reflect.Value) { + panicIfInvalidKey(key) + if len(key) == 1 { + // Output an extra newline between top-level tables. + // (The newline isn't written if nothing else has been written though.) + enc.newline() + } + if len(key) > 0 { + enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll()) + enc.newline() + } + enc.eMapOrStruct(key, rv) +} + +func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) { + switch rv := eindirect(rv); rv.Kind() { + case reflect.Map: + enc.eMap(key, rv) + case reflect.Struct: + enc.eStruct(key, rv) + default: + panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) + } +} + +func (enc *Encoder) eMap(key Key, rv reflect.Value) { + rt := rv.Type() + if rt.Key().Kind() != reflect.String { + encPanic(errNonString) + } + + // Sort keys so that we have deterministic output. And write keys directly + // underneath this key first, before writing sub-structs or sub-maps. + var mapKeysDirect, mapKeysSub []string + for _, mapKey := range rv.MapKeys() { + k := mapKey.String() + if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { + mapKeysSub = append(mapKeysSub, k) + } else { + mapKeysDirect = append(mapKeysDirect, k) + } + } + + var writeMapKeys = func(mapKeys []string) { + sort.Strings(mapKeys) + for _, mapKey := range mapKeys { + mrv := rv.MapIndex(reflect.ValueOf(mapKey)) + if isNil(mrv) { + // Don't write anything for nil fields. + continue + } + enc.encode(key.add(mapKey), mrv) + } + } + writeMapKeys(mapKeysDirect) + writeMapKeys(mapKeysSub) +} + +func (enc *Encoder) eStruct(key Key, rv reflect.Value) { + // Write keys for fields directly under this key first, because if we write + // a field that creates a new table, then all keys under it will be in that + // table (not the one we're writing here). + rt := rv.Type() + var fieldsDirect, fieldsSub [][]int + var addFields func(rt reflect.Type, rv reflect.Value, start []int) + addFields = func(rt reflect.Type, rv reflect.Value, start []int) { + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + // skip unexported fields + if f.PkgPath != "" && !f.Anonymous { + continue + } + frv := rv.Field(i) + if f.Anonymous { + t := f.Type + switch t.Kind() { + case reflect.Struct: + // Treat anonymous struct fields with + // tag names as though they are not + // anonymous, like encoding/json does. + if getOptions(f.Tag).name == "" { + addFields(t, frv, f.Index) + continue + } + case reflect.Ptr: + if t.Elem().Kind() == reflect.Struct && + getOptions(f.Tag).name == "" { + if !frv.IsNil() { + addFields(t.Elem(), frv.Elem(), f.Index) + } + continue + } + // Fall through to the normal field encoding logic below + // for non-struct anonymous fields. + } + } + + if typeIsHash(tomlTypeOfGo(frv)) { + fieldsSub = append(fieldsSub, append(start, f.Index...)) + } else { + fieldsDirect = append(fieldsDirect, append(start, f.Index...)) + } + } + } + addFields(rt, rv, nil) + + var writeFields = func(fields [][]int) { + for _, fieldIndex := range fields { + sft := rt.FieldByIndex(fieldIndex) + sf := rv.FieldByIndex(fieldIndex) + if isNil(sf) { + // Don't write anything for nil fields. + continue + } + + opts := getOptions(sft.Tag) + if opts.skip { + continue + } + keyName := sft.Name + if opts.name != "" { + keyName = opts.name + } + if opts.omitempty && isEmpty(sf) { + continue + } + if opts.omitzero && isZero(sf) { + continue + } + + enc.encode(key.add(keyName), sf) + } + } + writeFields(fieldsDirect) + writeFields(fieldsSub) +} + +// tomlTypeName returns the TOML type name of the Go value's type. It is +// used to determine whether the types of array elements are mixed (which is +// forbidden). If the Go value is nil, then it is illegal for it to be an array +// element, and valueIsNil is returned as true. + +// Returns the TOML type of a Go value. The type may be `nil`, which means +// no concrete TOML type could be found. +func tomlTypeOfGo(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() { + return nil + } + switch rv.Kind() { + case reflect.Bool: + return tomlBool + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, + reflect.Uint64: + return tomlInteger + case reflect.Float32, reflect.Float64: + return tomlFloat + case reflect.Array, reflect.Slice: + if typeEqual(tomlHash, tomlArrayType(rv)) { + return tomlArrayHash + } + return tomlArray + case reflect.Ptr, reflect.Interface: + return tomlTypeOfGo(rv.Elem()) + case reflect.String: + return tomlString + case reflect.Map: + return tomlHash + case reflect.Struct: + switch rv.Interface().(type) { + case time.Time: + return tomlDatetime + case TextMarshaler: + return tomlString + default: + return tomlHash + } + default: + panic("unexpected reflect.Kind: " + rv.Kind().String()) + } +} + +// tomlArrayType returns the element type of a TOML array. The type returned +// may be nil if it cannot be determined (e.g., a nil slice or a zero length +// slize). This function may also panic if it finds a type that cannot be +// expressed in TOML (such as nil elements, heterogeneous arrays or directly +// nested arrays of tables). +func tomlArrayType(rv reflect.Value) tomlType { + if isNil(rv) || !rv.IsValid() || rv.Len() == 0 { + return nil + } + firstType := tomlTypeOfGo(rv.Index(0)) + if firstType == nil { + encPanic(errArrayNilElement) + } + + rvlen := rv.Len() + for i := 1; i < rvlen; i++ { + elem := rv.Index(i) + switch elemType := tomlTypeOfGo(elem); { + case elemType == nil: + encPanic(errArrayNilElement) + case !typeEqual(firstType, elemType): + encPanic(errArrayMixedElementTypes) + } + } + // If we have a nested array, then we must make sure that the nested + // array contains ONLY primitives. + // This checks arbitrarily nested arrays. + if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) { + nest := tomlArrayType(eindirect(rv.Index(0))) + if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) { + encPanic(errArrayNoTable) + } + } + return firstType +} + +type tagOptions struct { + skip bool // "-" + name string + omitempty bool + omitzero bool +} + +func getOptions(tag reflect.StructTag) tagOptions { + t := tag.Get("toml") + if t == "-" { + return tagOptions{skip: true} + } + var opts tagOptions + parts := strings.Split(t, ",") + opts.name = parts[0] + for _, s := range parts[1:] { + switch s { + case "omitempty": + opts.omitempty = true + case "omitzero": + opts.omitzero = true + } + } + return opts +} + +func isZero(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint() == 0 + case reflect.Float32, reflect.Float64: + return rv.Float() == 0.0 + } + return false +} + +func isEmpty(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Array, reflect.Slice, reflect.Map, reflect.String: + return rv.Len() == 0 + case reflect.Bool: + return !rv.Bool() + } + return false +} + +func (enc *Encoder) newline() { + if enc.hasWritten { + enc.wf("\n") + } +} + +func (enc *Encoder) keyEqElement(key Key, val reflect.Value) { + if len(key) == 0 { + encPanic(errNoKey) + } + panicIfInvalidKey(key) + enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1)) + enc.eElement(val) + enc.newline() +} + +func (enc *Encoder) wf(format string, v ...interface{}) { + if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { + encPanic(err) + } + enc.hasWritten = true +} + +func (enc *Encoder) indentStr(key Key) string { + return strings.Repeat(enc.Indent, len(key)-1) +} + +func encPanic(err error) { + panic(tomlEncodeError{err}) +} + +func eindirect(v reflect.Value) reflect.Value { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + return eindirect(v.Elem()) + default: + return v + } +} + +func isNil(rv reflect.Value) bool { + switch rv.Kind() { + case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return rv.IsNil() + default: + return false + } +} + +func panicIfInvalidKey(key Key) { + for _, k := range key { + if len(k) == 0 { + encPanic(e("Key '%s' is not a valid table name. Key names "+ + "cannot be empty.", key.maybeQuotedAll())) + } + } +} + +func isValidKeyName(s string) bool { + return len(s) != 0 +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types.go new file mode 100644 index 0000000..d36e1dd --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types.go @@ -0,0 +1,19 @@ +// +build go1.2 + +package toml + +// In order to support Go 1.1, we define our own TextMarshaler and +// TextUnmarshaler types. For Go 1.2+, we just alias them with the +// standard library interfaces. + +import ( + "encoding" +) + +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. +type TextMarshaler encoding.TextMarshaler + +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined +// here so that Go 1.1 can be supported. +type TextUnmarshaler encoding.TextUnmarshaler diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go new file mode 100644 index 0000000..e8d503d --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go @@ -0,0 +1,18 @@ +// +build !go1.2 + +package toml + +// These interfaces were introduced in Go 1.2, so we add them manually when +// compiling for Go 1.1. + +// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here +// so that Go 1.1 can be supported. +type TextMarshaler interface { + MarshalText() (text []byte, err error) +} + +// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined +// here so that Go 1.1 can be supported. +type TextUnmarshaler interface { + UnmarshalText(text []byte) error +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/lex.go new file mode 100644 index 0000000..6dee7fc --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/lex.go @@ -0,0 +1,953 @@ +package toml + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +type itemType int + +const ( + itemError itemType = iota + itemNIL // used in the parser to indicate no type + itemEOF + itemText + itemString + itemRawString + itemMultilineString + itemRawMultilineString + itemBool + itemInteger + itemFloat + itemDatetime + itemArray // the start of an array + itemArrayEnd + itemTableStart + itemTableEnd + itemArrayTableStart + itemArrayTableEnd + itemKeyStart + itemCommentStart + itemInlineTableStart + itemInlineTableEnd +) + +const ( + eof = 0 + comma = ',' + tableStart = '[' + tableEnd = ']' + arrayTableStart = '[' + arrayTableEnd = ']' + tableSep = '.' + keySep = '=' + arrayStart = '[' + arrayEnd = ']' + commentStart = '#' + stringStart = '"' + stringEnd = '"' + rawStringStart = '\'' + rawStringEnd = '\'' + inlineTableStart = '{' + inlineTableEnd = '}' +) + +type stateFn func(lx *lexer) stateFn + +type lexer struct { + input string + start int + pos int + line int + state stateFn + items chan item + + // Allow for backing up up to three runes. + // This is necessary because TOML contains 3-rune tokens (""" and '''). + prevWidths [3]int + nprev int // how many of prevWidths are in use + // If we emit an eof, we can still back up, but it is not OK to call + // next again. + atEOF bool + + // A stack of state functions used to maintain context. + // The idea is to reuse parts of the state machine in various places. + // For example, values can appear at the top level or within arbitrarily + // nested arrays. The last state on the stack is used after a value has + // been lexed. Similarly for comments. + stack []stateFn +} + +type item struct { + typ itemType + val string + line int +} + +func (lx *lexer) nextItem() item { + for { + select { + case item := <-lx.items: + return item + default: + lx.state = lx.state(lx) + } + } +} + +func lex(input string) *lexer { + lx := &lexer{ + input: input, + state: lexTop, + line: 1, + items: make(chan item, 10), + stack: make([]stateFn, 0, 10), + } + return lx +} + +func (lx *lexer) push(state stateFn) { + lx.stack = append(lx.stack, state) +} + +func (lx *lexer) pop() stateFn { + if len(lx.stack) == 0 { + return lx.errorf("BUG in lexer: no states to pop") + } + last := lx.stack[len(lx.stack)-1] + lx.stack = lx.stack[0 : len(lx.stack)-1] + return last +} + +func (lx *lexer) current() string { + return lx.input[lx.start:lx.pos] +} + +func (lx *lexer) emit(typ itemType) { + lx.items <- item{typ, lx.current(), lx.line} + lx.start = lx.pos +} + +func (lx *lexer) emitTrim(typ itemType) { + lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} + lx.start = lx.pos +} + +func (lx *lexer) next() (r rune) { + if lx.atEOF { + panic("next called after EOF") + } + if lx.pos >= len(lx.input) { + lx.atEOF = true + return eof + } + + if lx.input[lx.pos] == '\n' { + lx.line++ + } + lx.prevWidths[2] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[0] + if lx.nprev < 3 { + lx.nprev++ + } + r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.prevWidths[0] = w + lx.pos += w + return r +} + +// ignore skips over the pending input before this point. +func (lx *lexer) ignore() { + lx.start = lx.pos +} + +// backup steps back one rune. Can be called only twice between calls to next. +func (lx *lexer) backup() { + if lx.atEOF { + lx.atEOF = false + return + } + if lx.nprev < 1 { + panic("backed up too far") + } + w := lx.prevWidths[0] + lx.prevWidths[0] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[2] + lx.nprev-- + lx.pos -= w + if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { + lx.line-- + } +} + +// accept consumes the next rune if it's equal to `valid`. +func (lx *lexer) accept(valid rune) bool { + if lx.next() == valid { + return true + } + lx.backup() + return false +} + +// peek returns but does not consume the next rune in the input. +func (lx *lexer) peek() rune { + r := lx.next() + lx.backup() + return r +} + +// skip ignores all input that matches the given predicate. +func (lx *lexer) skip(pred func(rune) bool) { + for { + r := lx.next() + if pred(r) { + continue + } + lx.backup() + lx.ignore() + return + } +} + +// errorf stops all lexing by emitting an error and returning `nil`. +// Note that any value that is a character is escaped if it's a special +// character (newlines, tabs, etc.). +func (lx *lexer) errorf(format string, values ...interface{}) stateFn { + lx.items <- item{ + itemError, + fmt.Sprintf(format, values...), + lx.line, + } + return nil +} + +// lexTop consumes elements at the top level of TOML data. +func lexTop(lx *lexer) stateFn { + r := lx.next() + if isWhitespace(r) || isNL(r) { + return lexSkip(lx, lexTop) + } + switch r { + case commentStart: + lx.push(lexTop) + return lexCommentStart + case tableStart: + return lexTableStart + case eof: + if lx.pos > lx.start { + return lx.errorf("unexpected EOF") + } + lx.emit(itemEOF) + return nil + } + + // At this point, the only valid item can be a key, so we back up + // and let the key lexer do the rest. + lx.backup() + lx.push(lexTopEnd) + return lexKeyStart +} + +// lexTopEnd is entered whenever a top-level item has been consumed. (A value +// or a table.) It must see only whitespace, and will turn back to lexTop +// upon a newline. If it sees EOF, it will quit the lexer successfully. +func lexTopEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case r == commentStart: + // a comment will read to a newline for us. + lx.push(lexTop) + return lexCommentStart + case isWhitespace(r): + return lexTopEnd + case isNL(r): + lx.ignore() + return lexTop + case r == eof: + lx.emit(itemEOF) + return nil + } + return lx.errorf("expected a top-level item to end with a newline, "+ + "comment, or EOF, but got %q instead", r) +} + +// lexTable lexes the beginning of a table. Namely, it makes sure that +// it starts with a character other than '.' and ']'. +// It assumes that '[' has already been consumed. +// It also handles the case that this is an item in an array of tables. +// e.g., '[[name]]'. +func lexTableStart(lx *lexer) stateFn { + if lx.peek() == arrayTableStart { + lx.next() + lx.emit(itemArrayTableStart) + lx.push(lexArrayTableEnd) + } else { + lx.emit(itemTableStart) + lx.push(lexTableEnd) + } + return lexTableNameStart +} + +func lexTableEnd(lx *lexer) stateFn { + lx.emit(itemTableEnd) + return lexTopEnd +} + +func lexArrayTableEnd(lx *lexer) stateFn { + if r := lx.next(); r != arrayTableEnd { + return lx.errorf("expected end of table array name delimiter %q, "+ + "but got %q instead", arrayTableEnd, r) + } + lx.emit(itemArrayTableEnd) + return lexTopEnd +} + +func lexTableNameStart(lx *lexer) stateFn { + lx.skip(isWhitespace) + switch r := lx.peek(); { + case r == tableEnd || r == eof: + return lx.errorf("unexpected end of table name " + + "(table names cannot be empty)") + case r == tableSep: + return lx.errorf("unexpected table separator " + + "(table names cannot be empty)") + case r == stringStart || r == rawStringStart: + lx.ignore() + lx.push(lexTableNameEnd) + return lexValue // reuse string lexing + default: + return lexBareTableName + } +} + +// lexBareTableName lexes the name of a table. It assumes that at least one +// valid character for the table has already been read. +func lexBareTableName(lx *lexer) stateFn { + r := lx.next() + if isBareKeyChar(r) { + return lexBareTableName + } + lx.backup() + lx.emit(itemText) + return lexTableNameEnd +} + +// lexTableNameEnd reads the end of a piece of a table name, optionally +// consuming whitespace. +func lexTableNameEnd(lx *lexer) stateFn { + lx.skip(isWhitespace) + switch r := lx.next(); { + case isWhitespace(r): + return lexTableNameEnd + case r == tableSep: + lx.ignore() + return lexTableNameStart + case r == tableEnd: + return lx.pop() + default: + return lx.errorf("expected '.' or ']' to end table name, "+ + "but got %q instead", r) + } +} + +// lexKeyStart consumes a key name up until the first non-whitespace character. +// lexKeyStart will ignore whitespace. +func lexKeyStart(lx *lexer) stateFn { + r := lx.peek() + switch { + case r == keySep: + return lx.errorf("unexpected key separator %q", keySep) + case isWhitespace(r) || isNL(r): + lx.next() + return lexSkip(lx, lexKeyStart) + case r == stringStart || r == rawStringStart: + lx.ignore() + lx.emit(itemKeyStart) + lx.push(lexKeyEnd) + return lexValue // reuse string lexing + default: + lx.ignore() + lx.emit(itemKeyStart) + return lexBareKey + } +} + +// lexBareKey consumes the text of a bare key. Assumes that the first character +// (which is not whitespace) has not yet been consumed. +func lexBareKey(lx *lexer) stateFn { + switch r := lx.next(); { + case isBareKeyChar(r): + return lexBareKey + case isWhitespace(r): + lx.backup() + lx.emit(itemText) + return lexKeyEnd + case r == keySep: + lx.backup() + lx.emit(itemText) + return lexKeyEnd + default: + return lx.errorf("bare keys cannot contain %q", r) + } +} + +// lexKeyEnd consumes the end of a key and trims whitespace (up to the key +// separator). +func lexKeyEnd(lx *lexer) stateFn { + switch r := lx.next(); { + case r == keySep: + return lexSkip(lx, lexValue) + case isWhitespace(r): + return lexSkip(lx, lexKeyEnd) + default: + return lx.errorf("expected key separator %q, but got %q instead", + keySep, r) + } +} + +// lexValue starts the consumption of a value anywhere a value is expected. +// lexValue will ignore whitespace. +// After a value is lexed, the last state on the next is popped and returned. +func lexValue(lx *lexer) stateFn { + // We allow whitespace to precede a value, but NOT newlines. + // In array syntax, the array states are responsible for ignoring newlines. + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexValue) + case isDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateStart + } + switch r { + case arrayStart: + lx.ignore() + lx.emit(itemArray) + return lexArrayValue + case inlineTableStart: + lx.ignore() + lx.emit(itemInlineTableStart) + return lexInlineTableValue + case stringStart: + if lx.accept(stringStart) { + if lx.accept(stringStart) { + lx.ignore() // Ignore """ + return lexMultilineString + } + lx.backup() + } + lx.ignore() // ignore the '"' + return lexString + case rawStringStart: + if lx.accept(rawStringStart) { + if lx.accept(rawStringStart) { + lx.ignore() // Ignore """ + return lexMultilineRawString + } + lx.backup() + } + lx.ignore() // ignore the "'" + return lexRawString + case '+', '-': + return lexNumberStart + case '.': // special error case, be kind to users + return lx.errorf("floats must start with a digit, not '.'") + } + if unicode.IsLetter(r) { + // Be permissive here; lexBool will give a nice error if the + // user wrote something like + // x = foo + // (i.e. not 'true' or 'false' but is something else word-like.) + lx.backup() + return lexBool + } + return lx.errorf("expected value but found %q instead", r) +} + +// lexArrayValue consumes one value in an array. It assumes that '[' or ',' +// have already been consumed. All whitespace and newlines are ignored. +func lexArrayValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValue) + case r == commentStart: + lx.push(lexArrayValue) + return lexCommentStart + case r == comma: + return lx.errorf("unexpected comma") + case r == arrayEnd: + // NOTE(caleb): The spec isn't clear about whether you can have + // a trailing comma or not, so we'll allow it. + return lexArrayEnd + } + + lx.backup() + lx.push(lexArrayValueEnd) + return lexValue +} + +// lexArrayValueEnd consumes everything between the end of an array value and +// the next value (or the end of the array): it ignores whitespace and newlines +// and expects either a ',' or a ']'. +func lexArrayValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValueEnd) + case r == commentStart: + lx.push(lexArrayValueEnd) + return lexCommentStart + case r == comma: + lx.ignore() + return lexArrayValue // move on to the next value + case r == arrayEnd: + return lexArrayEnd + } + return lx.errorf( + "expected a comma or array terminator %q, but got %q instead", + arrayEnd, r, + ) +} + +// lexArrayEnd finishes the lexing of an array. +// It assumes that a ']' has just been consumed. +func lexArrayEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemArrayEnd) + return lx.pop() +} + +// lexInlineTableValue consumes one key/value pair in an inline table. +// It assumes that '{' or ',' have already been consumed. Whitespace is ignored. +func lexInlineTableValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValue) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValue) + return lexCommentStart + case r == comma: + return lx.errorf("unexpected comma") + case r == inlineTableEnd: + return lexInlineTableEnd + } + lx.backup() + lx.push(lexInlineTableValueEnd) + return lexKeyStart +} + +// lexInlineTableValueEnd consumes everything between the end of an inline table +// key/value pair and the next pair (or the end of the table): +// it ignores whitespace and expects either a ',' or a '}'. +func lexInlineTableValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValueEnd) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValueEnd) + return lexCommentStart + case r == comma: + lx.ignore() + return lexInlineTableValue + case r == inlineTableEnd: + return lexInlineTableEnd + } + return lx.errorf("expected a comma or an inline table terminator %q, "+ + "but got %q instead", inlineTableEnd, r) +} + +// lexInlineTableEnd finishes the lexing of an inline table. +// It assumes that a '}' has just been consumed. +func lexInlineTableEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemInlineTableEnd) + return lx.pop() +} + +// lexString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. +func lexString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == eof: + return lx.errorf("unexpected EOF") + case isNL(r): + return lx.errorf("strings cannot contain newlines") + case r == '\\': + lx.push(lexString) + return lexStringEscape + case r == stringEnd: + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexString +} + +// lexMultilineString consumes the inner contents of a string. It assumes that +// the beginning '"""' has already been consumed and ignored. +func lexMultilineString(lx *lexer) stateFn { + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case '\\': + return lexMultilineStringEscape + case stringEnd: + if lx.accept(stringEnd) { + if lx.accept(stringEnd) { + lx.backup() + lx.backup() + lx.backup() + lx.emit(itemMultilineString) + lx.next() + lx.next() + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + } + } + return lexMultilineString +} + +// lexRawString consumes a raw string. Nothing can be escaped in such a string. +// It assumes that the beginning "'" has already been consumed and ignored. +func lexRawString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == eof: + return lx.errorf("unexpected EOF") + case isNL(r): + return lx.errorf("strings cannot contain newlines") + case r == rawStringEnd: + lx.backup() + lx.emit(itemRawString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexRawString +} + +// lexMultilineRawString consumes a raw string. Nothing can be escaped in such +// a string. It assumes that the beginning "'''" has already been consumed and +// ignored. +func lexMultilineRawString(lx *lexer) stateFn { + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case rawStringEnd: + if lx.accept(rawStringEnd) { + if lx.accept(rawStringEnd) { + lx.backup() + lx.backup() + lx.backup() + lx.emit(itemRawMultilineString) + lx.next() + lx.next() + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + } + } + return lexMultilineRawString +} + +// lexMultilineStringEscape consumes an escaped character. It assumes that the +// preceding '\\' has already been consumed. +func lexMultilineStringEscape(lx *lexer) stateFn { + // Handle the special case first: + if isNL(lx.next()) { + return lexMultilineString + } + lx.backup() + lx.push(lexMultilineString) + return lexStringEscape(lx) +} + +func lexStringEscape(lx *lexer) stateFn { + r := lx.next() + switch r { + case 'b': + fallthrough + case 't': + fallthrough + case 'n': + fallthrough + case 'f': + fallthrough + case 'r': + fallthrough + case '"': + fallthrough + case '\\': + return lx.pop() + case 'u': + return lexShortUnicodeEscape + case 'U': + return lexLongUnicodeEscape + } + return lx.errorf("invalid escape character %q; only the following "+ + "escape characters are allowed: "+ + `\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r) +} + +func lexShortUnicodeEscape(lx *lexer) stateFn { + var r rune + for i := 0; i < 4; i++ { + r = lx.next() + if !isHexadecimal(r) { + return lx.errorf(`expected four hexadecimal digits after '\u', `+ + "but got %q instead", lx.current()) + } + } + return lx.pop() +} + +func lexLongUnicodeEscape(lx *lexer) stateFn { + var r rune + for i := 0; i < 8; i++ { + r = lx.next() + if !isHexadecimal(r) { + return lx.errorf(`expected eight hexadecimal digits after '\U', `+ + "but got %q instead", lx.current()) + } + } + return lx.pop() +} + +// lexNumberOrDateStart consumes either an integer, a float, or datetime. +func lexNumberOrDateStart(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexNumberOrDate + } + switch r { + case '_': + return lexNumber + case 'e', 'E': + return lexFloat + case '.': + return lx.errorf("floats must start with a digit, not '.'") + } + return lx.errorf("expected a digit but got %q", r) +} + +// lexNumberOrDate consumes either an integer, float or datetime. +func lexNumberOrDate(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexNumberOrDate + } + switch r { + case '-': + return lexDatetime + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexDatetime consumes a Datetime, to a first approximation. +// The parser validates that it matches one of the accepted formats. +func lexDatetime(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexDatetime + } + switch r { + case '-', 'T', ':', '.', 'Z': + return lexDatetime + } + + lx.backup() + lx.emit(itemDatetime) + return lx.pop() +} + +// lexNumberStart consumes either an integer or a float. It assumes that a sign +// has already been read, but that *no* digits have been consumed. +// lexNumberStart will move to the appropriate integer or float states. +func lexNumberStart(lx *lexer) stateFn { + // We MUST see a digit. Even floats have to start with a digit. + r := lx.next() + if !isDigit(r) { + if r == '.' { + return lx.errorf("floats must start with a digit, not '.'") + } + return lx.errorf("expected a digit but got %q", r) + } + return lexNumber +} + +// lexNumber consumes an integer or a float after seeing the first digit. +func lexNumber(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexNumber + } + switch r { + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexFloat consumes the elements of a float. It allows any sequence of +// float-like characters, so floats emitted by the lexer are only a first +// approximation and must be validated by the parser. +func lexFloat(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexFloat + } + switch r { + case '_', '.', '-', '+', 'e', 'E': + return lexFloat + } + + lx.backup() + lx.emit(itemFloat) + return lx.pop() +} + +// lexBool consumes a bool string: 'true' or 'false. +func lexBool(lx *lexer) stateFn { + var rs []rune + for { + r := lx.next() + if !unicode.IsLetter(r) { + lx.backup() + break + } + rs = append(rs, r) + } + s := string(rs) + switch s { + case "true", "false": + lx.emit(itemBool) + return lx.pop() + } + return lx.errorf("expected value but found %q instead", s) +} + +// lexCommentStart begins the lexing of a comment. It will emit +// itemCommentStart and consume no characters, passing control to lexComment. +func lexCommentStart(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemCommentStart) + return lexComment +} + +// lexComment lexes an entire comment. It assumes that '#' has been consumed. +// It will consume *up to* the first newline character, and pass control +// back to the last state on the stack. +func lexComment(lx *lexer) stateFn { + r := lx.peek() + if isNL(r) || r == eof { + lx.emit(itemText) + return lx.pop() + } + lx.next() + return lexComment +} + +// lexSkip ignores all slurped input and moves on to the next state. +func lexSkip(lx *lexer, nextState stateFn) stateFn { + return func(lx *lexer) stateFn { + lx.ignore() + return nextState + } +} + +// isWhitespace returns true if `r` is a whitespace character according +// to the spec. +func isWhitespace(r rune) bool { + return r == '\t' || r == ' ' +} + +func isNL(r rune) bool { + return r == '\n' || r == '\r' +} + +func isDigit(r rune) bool { + return r >= '0' && r <= '9' +} + +func isHexadecimal(r rune) bool { + return (r >= '0' && r <= '9') || + (r >= 'a' && r <= 'f') || + (r >= 'A' && r <= 'F') +} + +func isBareKeyChar(r rune) bool { + return (r >= 'A' && r <= 'Z') || + (r >= 'a' && r <= 'z') || + (r >= '0' && r <= '9') || + r == '_' || + r == '-' +} + +func (itype itemType) String() string { + switch itype { + case itemError: + return "Error" + case itemNIL: + return "NIL" + case itemEOF: + return "EOF" + case itemText: + return "Text" + case itemString, itemRawString, itemMultilineString, itemRawMultilineString: + return "String" + case itemBool: + return "Bool" + case itemInteger: + return "Integer" + case itemFloat: + return "Float" + case itemDatetime: + return "DateTime" + case itemTableStart: + return "TableStart" + case itemTableEnd: + return "TableEnd" + case itemKeyStart: + return "KeyStart" + case itemArray: + return "Array" + case itemArrayEnd: + return "ArrayEnd" + case itemCommentStart: + return "CommentStart" + } + panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype))) +} + +func (item item) String() string { + return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/parse.go new file mode 100644 index 0000000..50869ef --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/parse.go @@ -0,0 +1,592 @@ +package toml + +import ( + "fmt" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +type parser struct { + mapping map[string]interface{} + types map[string]tomlType + lx *lexer + + // A list of keys in the order that they appear in the TOML data. + ordered []Key + + // the full key for the current hash in scope + context Key + + // the base key name for everything except hashes + currentKey string + + // rough approximation of line number + approxLine int + + // A map of 'key.group.names' to whether they were created implicitly. + implicits map[string]bool +} + +type parseError string + +func (pe parseError) Error() string { + return string(pe) +} + +func parse(data string) (p *parser, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + if err, ok = r.(parseError); ok { + return + } + panic(r) + } + }() + + p = &parser{ + mapping: make(map[string]interface{}), + types: make(map[string]tomlType), + lx: lex(data), + ordered: make([]Key, 0), + implicits: make(map[string]bool), + } + for { + item := p.next() + if item.typ == itemEOF { + break + } + p.topLevel(item) + } + + return p, nil +} + +func (p *parser) panicf(format string, v ...interface{}) { + msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s", + p.approxLine, p.current(), fmt.Sprintf(format, v...)) + panic(parseError(msg)) +} + +func (p *parser) next() item { + it := p.lx.nextItem() + if it.typ == itemError { + p.panicf("%s", it.val) + } + return it +} + +func (p *parser) bug(format string, v ...interface{}) { + panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) +} + +func (p *parser) expect(typ itemType) item { + it := p.next() + p.assertEqual(typ, it.typ) + return it +} + +func (p *parser) assertEqual(expected, got itemType) { + if expected != got { + p.bug("Expected '%s' but got '%s'.", expected, got) + } +} + +func (p *parser) topLevel(item item) { + switch item.typ { + case itemCommentStart: + p.approxLine = item.line + p.expect(itemText) + case itemTableStart: + kg := p.next() + p.approxLine = kg.line + + var key Key + for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() { + key = append(key, p.keyString(kg)) + } + p.assertEqual(itemTableEnd, kg.typ) + + p.establishContext(key, false) + p.setType("", tomlHash) + p.ordered = append(p.ordered, key) + case itemArrayTableStart: + kg := p.next() + p.approxLine = kg.line + + var key Key + for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() { + key = append(key, p.keyString(kg)) + } + p.assertEqual(itemArrayTableEnd, kg.typ) + + p.establishContext(key, true) + p.setType("", tomlArrayHash) + p.ordered = append(p.ordered, key) + case itemKeyStart: + kname := p.next() + p.approxLine = kname.line + p.currentKey = p.keyString(kname) + + val, typ := p.value(p.next()) + p.setValue(p.currentKey, val) + p.setType(p.currentKey, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + p.currentKey = "" + default: + p.bug("Unexpected type at top level: %s", item.typ) + } +} + +// Gets a string for a key (or part of a key in a table name). +func (p *parser) keyString(it item) string { + switch it.typ { + case itemText: + return it.val + case itemString, itemMultilineString, + itemRawString, itemRawMultilineString: + s, _ := p.value(it) + return s.(string) + default: + p.bug("Unexpected key type: %s", it.typ) + panic("unreachable") + } +} + +// value translates an expected value from the lexer into a Go value wrapped +// as an empty interface. +func (p *parser) value(it item) (interface{}, tomlType) { + switch it.typ { + case itemString: + return p.replaceEscapes(it.val), p.typeOfPrimitive(it) + case itemMultilineString: + trimmed := stripFirstNewline(stripEscapedWhitespace(it.val)) + return p.replaceEscapes(trimmed), p.typeOfPrimitive(it) + case itemRawString: + return it.val, p.typeOfPrimitive(it) + case itemRawMultilineString: + return stripFirstNewline(it.val), p.typeOfPrimitive(it) + case itemBool: + switch it.val { + case "true": + return true, p.typeOfPrimitive(it) + case "false": + return false, p.typeOfPrimitive(it) + } + p.bug("Expected boolean value, but got '%s'.", it.val) + case itemInteger: + if !numUnderscoresOK(it.val) { + p.panicf("Invalid integer %q: underscores must be surrounded by digits", + it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseInt(val, 10, 64) + if err != nil { + // Distinguish integer values. Normally, it'd be a bug if the lexer + // provides an invalid integer, but it's possible that the number is + // out of range of valid values (which the lexer cannot determine). + // So mark the former as a bug but the latter as a legitimate user + // error. + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panicf("Integer '%s' is out of the range of 64-bit "+ + "signed integers.", it.val) + } else { + p.bug("Expected integer value, but got '%s'.", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemFloat: + parts := strings.FieldsFunc(it.val, func(r rune) bool { + switch r { + case '.', 'e', 'E': + return true + } + return false + }) + for _, part := range parts { + if !numUnderscoresOK(part) { + p.panicf("Invalid float %q: underscores must be "+ + "surrounded by digits", it.val) + } + } + if !numPeriodsOK(it.val) { + // As a special case, numbers like '123.' or '1.e2', + // which are valid as far as Go/strconv are concerned, + // must be rejected because TOML says that a fractional + // part consists of '.' followed by 1+ digits. + p.panicf("Invalid float %q: '.' must be followed "+ + "by one or more digits", it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseFloat(val, 64) + if err != nil { + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panicf("Float '%s' is out of the range of 64-bit "+ + "IEEE-754 floating-point numbers.", it.val) + } else { + p.panicf("Invalid float value: %q", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemDatetime: + var t time.Time + var ok bool + var err error + for _, format := range []string{ + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05", + "2006-01-02", + } { + t, err = time.ParseInLocation(format, it.val, time.Local) + if err == nil { + ok = true + break + } + } + if !ok { + p.panicf("Invalid TOML Datetime: %q.", it.val) + } + return t, p.typeOfPrimitive(it) + case itemArray: + array := make([]interface{}, 0) + types := make([]tomlType, 0) + + for it = p.next(); it.typ != itemArrayEnd; it = p.next() { + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + val, typ := p.value(it) + array = append(array, val) + types = append(types, typ) + } + return array, p.typeOfArray(types) + case itemInlineTableStart: + var ( + hash = make(map[string]interface{}) + outerContext = p.context + outerKey = p.currentKey + ) + + p.context = append(p.context, p.currentKey) + p.currentKey = "" + for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() { + if it.typ != itemKeyStart { + p.bug("Expected key start but instead found %q, around line %d", + it.val, p.approxLine) + } + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + // retrieve key + k := p.next() + p.approxLine = k.line + kname := p.keyString(k) + + // retrieve value + p.currentKey = kname + val, typ := p.value(p.next()) + // make sure we keep metadata up to date + p.setType(kname, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + hash[kname] = val + } + p.context = outerContext + p.currentKey = outerKey + return hash, tomlHash + } + p.bug("Unexpected value type: %s", it.typ) + panic("unreachable") +} + +// numUnderscoresOK checks whether each underscore in s is surrounded by +// characters that are not underscores. +func numUnderscoresOK(s string) bool { + accept := false + for _, r := range s { + if r == '_' { + if !accept { + return false + } + accept = false + continue + } + accept = true + } + return accept +} + +// numPeriodsOK checks whether every period in s is followed by a digit. +func numPeriodsOK(s string) bool { + period := false + for _, r := range s { + if period && !isDigit(r) { + return false + } + period = r == '.' + } + return !period +} + +// establishContext sets the current context of the parser, +// where the context is either a hash or an array of hashes. Which one is +// set depends on the value of the `array` parameter. +// +// Establishing the context also makes sure that the key isn't a duplicate, and +// will create implicit hashes automatically. +func (p *parser) establishContext(key Key, array bool) { + var ok bool + + // Always start at the top level and drill down for our context. + hashContext := p.mapping + keyContext := make(Key, 0) + + // We only need implicit hashes for key[0:-1] + for _, k := range key[0 : len(key)-1] { + _, ok = hashContext[k] + keyContext = append(keyContext, k) + + // No key? Make an implicit hash and move on. + if !ok { + p.addImplicit(keyContext) + hashContext[k] = make(map[string]interface{}) + } + + // If the hash context is actually an array of tables, then set + // the hash context to the last element in that array. + // + // Otherwise, it better be a table, since this MUST be a key group (by + // virtue of it not being the last element in a key). + switch t := hashContext[k].(type) { + case []map[string]interface{}: + hashContext = t[len(t)-1] + case map[string]interface{}: + hashContext = t + default: + p.panicf("Key '%s' was already created as a hash.", keyContext) + } + } + + p.context = keyContext + if array { + // If this is the first element for this array, then allocate a new + // list of tables for it. + k := key[len(key)-1] + if _, ok := hashContext[k]; !ok { + hashContext[k] = make([]map[string]interface{}, 0, 5) + } + + // Add a new table. But make sure the key hasn't already been used + // for something else. + if hash, ok := hashContext[k].([]map[string]interface{}); ok { + hashContext[k] = append(hash, make(map[string]interface{})) + } else { + p.panicf("Key '%s' was already created and cannot be used as "+ + "an array.", keyContext) + } + } else { + p.setValue(key[len(key)-1], make(map[string]interface{})) + } + p.context = append(p.context, key[len(key)-1]) +} + +// setValue sets the given key to the given value in the current context. +// It will make sure that the key hasn't already been defined, account for +// implicit key groups. +func (p *parser) setValue(key string, value interface{}) { + var tmpHash interface{} + var ok bool + + hash := p.mapping + keyContext := make(Key, 0) + for _, k := range p.context { + keyContext = append(keyContext, k) + if tmpHash, ok = hash[k]; !ok { + p.bug("Context for key '%s' has not been established.", keyContext) + } + switch t := tmpHash.(type) { + case []map[string]interface{}: + // The context is a table of hashes. Pick the most recent table + // defined as the current hash. + hash = t[len(t)-1] + case map[string]interface{}: + hash = t + default: + p.bug("Expected hash to have type 'map[string]interface{}', but "+ + "it has '%T' instead.", tmpHash) + } + } + keyContext = append(keyContext, key) + + if _, ok := hash[key]; ok { + // Typically, if the given key has already been set, then we have + // to raise an error since duplicate keys are disallowed. However, + // it's possible that a key was previously defined implicitly. In this + // case, it is allowed to be redefined concretely. (See the + // `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.) + // + // But we have to make sure to stop marking it as an implicit. (So that + // another redefinition provokes an error.) + // + // Note that since it has already been defined (as a hash), we don't + // want to overwrite it. So our business is done. + if p.isImplicit(keyContext) { + p.removeImplicit(keyContext) + return + } + + // Otherwise, we have a concrete key trying to override a previous + // key, which is *always* wrong. + p.panicf("Key '%s' has already been defined.", keyContext) + } + hash[key] = value +} + +// setType sets the type of a particular value at a given key. +// It should be called immediately AFTER setValue. +// +// Note that if `key` is empty, then the type given will be applied to the +// current context (which is either a table or an array of tables). +func (p *parser) setType(key string, typ tomlType) { + keyContext := make(Key, 0, len(p.context)+1) + for _, k := range p.context { + keyContext = append(keyContext, k) + } + if len(key) > 0 { // allow type setting for hashes + keyContext = append(keyContext, key) + } + p.types[keyContext.String()] = typ +} + +// addImplicit sets the given Key as having been created implicitly. +func (p *parser) addImplicit(key Key) { + p.implicits[key.String()] = true +} + +// removeImplicit stops tagging the given key as having been implicitly +// created. +func (p *parser) removeImplicit(key Key) { + p.implicits[key.String()] = false +} + +// isImplicit returns true if the key group pointed to by the key was created +// implicitly. +func (p *parser) isImplicit(key Key) bool { + return p.implicits[key.String()] +} + +// current returns the full key name of the current context. +func (p *parser) current() string { + if len(p.currentKey) == 0 { + return p.context.String() + } + if len(p.context) == 0 { + return p.currentKey + } + return fmt.Sprintf("%s.%s", p.context, p.currentKey) +} + +func stripFirstNewline(s string) string { + if len(s) == 0 || s[0] != '\n' { + return s + } + return s[1:] +} + +func stripEscapedWhitespace(s string) string { + esc := strings.Split(s, "\\\n") + if len(esc) > 1 { + for i := 1; i < len(esc); i++ { + esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace) + } + } + return strings.Join(esc, "") +} + +func (p *parser) replaceEscapes(str string) string { + var replaced []rune + s := []byte(str) + r := 0 + for r < len(s) { + if s[r] != '\\' { + c, size := utf8.DecodeRune(s[r:]) + r += size + replaced = append(replaced, c) + continue + } + r += 1 + if r >= len(s) { + p.bug("Escape sequence at end of string.") + return "" + } + switch s[r] { + default: + p.bug("Expected valid escape code after \\, but got %q.", s[r]) + return "" + case 'b': + replaced = append(replaced, rune(0x0008)) + r += 1 + case 't': + replaced = append(replaced, rune(0x0009)) + r += 1 + case 'n': + replaced = append(replaced, rune(0x000A)) + r += 1 + case 'f': + replaced = append(replaced, rune(0x000C)) + r += 1 + case 'r': + replaced = append(replaced, rune(0x000D)) + r += 1 + case '"': + replaced = append(replaced, rune(0x0022)) + r += 1 + case '\\': + replaced = append(replaced, rune(0x005C)) + r += 1 + case 'u': + // At this point, we know we have a Unicode escape of the form + // `uXXXX` at [r, r+5). (Because the lexer guarantees this + // for us.) + escaped := p.asciiEscapeToUnicode(s[r+1 : r+5]) + replaced = append(replaced, escaped) + r += 5 + case 'U': + // At this point, we know we have a Unicode escape of the form + // `uXXXX` at [r, r+9). (Because the lexer guarantees this + // for us.) + escaped := p.asciiEscapeToUnicode(s[r+1 : r+9]) + replaced = append(replaced, escaped) + r += 9 + } + } + return string(replaced) +} + +func (p *parser) asciiEscapeToUnicode(bs []byte) rune { + s := string(bs) + hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) + if err != nil { + p.bug("Could not parse '%s' as a hexadecimal number, but the "+ + "lexer claims it's OK: %s", s, err) + } + if !utf8.ValidRune(rune(hex)) { + p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s) + } + return rune(hex) +} + +func isStringType(ty itemType) bool { + return ty == itemString || ty == itemMultilineString || + ty == itemRawString || ty == itemRawMultilineString +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_check.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_check.go new file mode 100644 index 0000000..c73f8af --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_check.go @@ -0,0 +1,91 @@ +package toml + +// tomlType represents any Go type that corresponds to a TOML type. +// While the first draft of the TOML spec has a simplistic type system that +// probably doesn't need this level of sophistication, we seem to be militating +// toward adding real composite types. +type tomlType interface { + typeString() string +} + +// typeEqual accepts any two types and returns true if they are equal. +func typeEqual(t1, t2 tomlType) bool { + if t1 == nil || t2 == nil { + return false + } + return t1.typeString() == t2.typeString() +} + +func typeIsHash(t tomlType) bool { + return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) +} + +type tomlBaseType string + +func (btype tomlBaseType) typeString() string { + return string(btype) +} + +func (btype tomlBaseType) String() string { + return btype.typeString() +} + +var ( + tomlInteger tomlBaseType = "Integer" + tomlFloat tomlBaseType = "Float" + tomlDatetime tomlBaseType = "Datetime" + tomlString tomlBaseType = "String" + tomlBool tomlBaseType = "Bool" + tomlArray tomlBaseType = "Array" + tomlHash tomlBaseType = "Hash" + tomlArrayHash tomlBaseType = "ArrayHash" +) + +// typeOfPrimitive returns a tomlType of any primitive value in TOML. +// Primitive values are: Integer, Float, Datetime, String and Bool. +// +// Passing a lexer item other than the following will cause a BUG message +// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime. +func (p *parser) typeOfPrimitive(lexItem item) tomlType { + switch lexItem.typ { + case itemInteger: + return tomlInteger + case itemFloat: + return tomlFloat + case itemDatetime: + return tomlDatetime + case itemString: + return tomlString + case itemMultilineString: + return tomlString + case itemRawString: + return tomlString + case itemRawMultilineString: + return tomlString + case itemBool: + return tomlBool + } + p.bug("Cannot infer primitive type of lex item '%s'.", lexItem) + panic("unreachable") +} + +// typeOfArray returns a tomlType for an array given a list of types of its +// values. +// +// In the current spec, if an array is homogeneous, then its type is always +// "Array". If the array is not homogeneous, an error is generated. +func (p *parser) typeOfArray(types []tomlType) tomlType { + // Empty arrays are cool. + if len(types) == 0 { + return tomlArray + } + + theType := types[0] + for _, t := range types[1:] { + if !typeEqual(theType, t) { + p.panicf("Array contains values of type '%s' and '%s', but "+ + "arrays must be homogeneous.", theType, t) + } + } + return tomlArray +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_fields.go new file mode 100644 index 0000000..608997c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/BurntSushi/toml/type_fields.go @@ -0,0 +1,242 @@ +package toml + +// Struct field handling is adapted from code in encoding/json: +// +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the Go distribution. + +import ( + "reflect" + "sort" + "sync" +) + +// A field represents a single field found in a struct. +type field struct { + name string // the name of the field (`toml` tag included) + tag bool // whether field has a `toml` tag + index []int // represents the depth of an anonymous field + typ reflect.Type // the type of the field +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from toml tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that TOML should recognize for the given +// type. The algorithm is breadth-first search over the set of structs to +// include - the top struct and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" && !sf.Anonymous { // unexported + continue + } + opts := getOptions(sf.Tag) + if opts.skip { + continue + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := opts.name != "" + name := opts.name + if name == "" { + name = sf.Name + } + fields = append(fields, field{name, tagged, index, ft}) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + f := field{name: ft.Name(), index: index, typ: ft} + next = append(next, f) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with TOML tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// TOML tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS new file mode 100644 index 0000000..fbe4ec4 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -0,0 +1,90 @@ +# This is the official list of Go-MySQL-Driver authors for copyright purposes. + +# If you are submitting a patch, please add your name or the name of the +# organization which holds the copyright to this list in alphabetical order. + +# Names should be added to this file as +# Name +# The email address is not required for organizations. +# Please keep the list sorted. + + +# Individual Persons + +Aaron Hopkins +Achille Roussel +Alexey Palazhchenko +Andrew Reid +Arne Hormann +Asta Xie +Bulat Gaifullin +Carlos Nieto +Chris Moos +Craig Wilson +Daniel Montoya +Daniel Nichter +Daniël van Eeden +Dave Protasowski +DisposaBoy +Egor Smolyakov +Evan Shaw +Frederick Mayle +Gustavo Kristic +Hajime Nakagami +Hanno Braun +Henri Yandell +Hirotaka Yamamoto +ICHINOSE Shogo +INADA Naoki +Jacek Szwec +James Harr +Jeff Hodges +Jeffrey Charles +Jian Zhen +Joshua Prunier +Julien Lefevre +Julien Schmidt +Justin Li +Justin Nuß +Kamil Dziedzic +Kevin Malachowski +Kieron Woodhouse +Lennart Rudolph +Leonardo YongUk Kim +Linh Tran Tuan +Lion Yang +Luca Looz +Lucas Liu +Luke Scott +Maciej Zimnoch +Michael Woolnough +Nicola Peduzzi +Olivier Mengué +oscarzhao +Paul Bonser +Peter Schultz +Rebecca Chin +Reed Allman +Richard Wilkes +Robert Russell +Runrioter Wung +Shuode Li +Soroush Pour +Stan Putrya +Stanley Gunawan +Thomas Wodarek +Xiangyu Hu +Xiaobing Jiang +Xiuming Chen +Zhenye Xie + +# Organizations + +Barracuda Networks, Inc. +Counting Ltd. +Google Inc. +InfoSum Ltd. +Keybase Inc. +Percona LLC +Pivotal Inc. +Stripe Inc. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/LICENSE new file mode 100644 index 0000000..14e2f77 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go new file mode 100644 index 0000000..be41f2e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go @@ -0,0 +1,19 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build appengine + +package mysql + +import ( + "google.golang.org/appengine/cloudsql" +) + +func init() { + RegisterDial("cloudsql", cloudsql.Dial) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go new file mode 100644 index 0000000..2f61ecd --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go @@ -0,0 +1,420 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/x509" + "encoding/pem" + "sync" +) + +// server pub keys registry +var ( + serverPubKeyLock sync.RWMutex + serverPubKeyRegistry map[string]*rsa.PublicKey +) + +// RegisterServerPubKey registers a server RSA public key which can be used to +// send data in a secure manner to the server without receiving the public key +// in a potentially insecure way from the server first. +// Registered keys can afterwards be used adding serverPubKey= to the DSN. +// +// Note: The provided rsa.PublicKey instance is exclusively owned by the driver +// after registering it and may not be modified. +// +// data, err := ioutil.ReadFile("mykey.pem") +// if err != nil { +// log.Fatal(err) +// } +// +// block, _ := pem.Decode(data) +// if block == nil || block.Type != "PUBLIC KEY" { +// log.Fatal("failed to decode PEM block containing public key") +// } +// +// pub, err := x509.ParsePKIXPublicKey(block.Bytes) +// if err != nil { +// log.Fatal(err) +// } +// +// if rsaPubKey, ok := pub.(*rsa.PublicKey); ok { +// mysql.RegisterServerPubKey("mykey", rsaPubKey) +// } else { +// log.Fatal("not a RSA public key") +// } +// +func RegisterServerPubKey(name string, pubKey *rsa.PublicKey) { + serverPubKeyLock.Lock() + if serverPubKeyRegistry == nil { + serverPubKeyRegistry = make(map[string]*rsa.PublicKey) + } + + serverPubKeyRegistry[name] = pubKey + serverPubKeyLock.Unlock() +} + +// DeregisterServerPubKey removes the public key registered with the given name. +func DeregisterServerPubKey(name string) { + serverPubKeyLock.Lock() + if serverPubKeyRegistry != nil { + delete(serverPubKeyRegistry, name) + } + serverPubKeyLock.Unlock() +} + +func getServerPubKey(name string) (pubKey *rsa.PublicKey) { + serverPubKeyLock.RLock() + if v, ok := serverPubKeyRegistry[name]; ok { + pubKey = v + } + serverPubKeyLock.RUnlock() + return +} + +// Hash password using pre 4.1 (old password) method +// https://github.com/atcurtis/mariadb/blob/master/mysys/my_rnd.c +type myRnd struct { + seed1, seed2 uint32 +} + +const myRndMaxVal = 0x3FFFFFFF + +// Pseudo random number generator +func newMyRnd(seed1, seed2 uint32) *myRnd { + return &myRnd{ + seed1: seed1 % myRndMaxVal, + seed2: seed2 % myRndMaxVal, + } +} + +// Tested to be equivalent to MariaDB's floating point variant +// http://play.golang.org/p/QHvhd4qved +// http://play.golang.org/p/RG0q4ElWDx +func (r *myRnd) NextByte() byte { + r.seed1 = (r.seed1*3 + r.seed2) % myRndMaxVal + r.seed2 = (r.seed1 + r.seed2 + 33) % myRndMaxVal + + return byte(uint64(r.seed1) * 31 / myRndMaxVal) +} + +// Generate binary hash from byte string using insecure pre 4.1 method +func pwHash(password []byte) (result [2]uint32) { + var add uint32 = 7 + var tmp uint32 + + result[0] = 1345345333 + result[1] = 0x12345671 + + for _, c := range password { + // skip spaces and tabs in password + if c == ' ' || c == '\t' { + continue + } + + tmp = uint32(c) + result[0] ^= (((result[0] & 63) + add) * tmp) + (result[0] << 8) + result[1] += (result[1] << 8) ^ result[0] + add += tmp + } + + // Remove sign bit (1<<31)-1) + result[0] &= 0x7FFFFFFF + result[1] &= 0x7FFFFFFF + + return +} + +// Hash password using insecure pre 4.1 method +func scrambleOldPassword(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + scramble = scramble[:8] + + hashPw := pwHash([]byte(password)) + hashSc := pwHash(scramble) + + r := newMyRnd(hashPw[0]^hashSc[0], hashPw[1]^hashSc[1]) + + var out [8]byte + for i := range out { + out[i] = r.NextByte() + 64 + } + + mask := r.NextByte() + for i := range out { + out[i] ^= mask + } + + return out[:] +} + +// Hash password using 4.1+ method (SHA1) +func scramblePassword(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + // stage1Hash = SHA1(password) + crypt := sha1.New() + crypt.Write([]byte(password)) + stage1 := crypt.Sum(nil) + + // scrambleHash = SHA1(scramble + SHA1(stage1Hash)) + // inner Hash + crypt.Reset() + crypt.Write(stage1) + hash := crypt.Sum(nil) + + // outer Hash + crypt.Reset() + crypt.Write(scramble) + crypt.Write(hash) + scramble = crypt.Sum(nil) + + // token = scrambleHash XOR stage1Hash + for i := range scramble { + scramble[i] ^= stage1[i] + } + return scramble +} + +// Hash password using MySQL 8+ method (SHA256) +func scrambleSHA256Password(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + // XOR(SHA256(password), SHA256(SHA256(SHA256(password)), scramble)) + + crypt := sha256.New() + crypt.Write([]byte(password)) + message1 := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1) + message1Hash := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1Hash) + crypt.Write(scramble) + message2 := crypt.Sum(nil) + + for i := range message1 { + message1[i] ^= message2[i] + } + + return message1 +} + +func encryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, error) { + plain := make([]byte, len(password)+1) + copy(plain, password) + for i := range plain { + j := i % len(seed) + plain[i] ^= seed[j] + } + sha1 := sha1.New() + return rsa.EncryptOAEP(sha1, rand.Reader, pub, plain, nil) +} + +func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) error { + enc, err := encryptPassword(mc.cfg.Passwd, seed, pub) + if err != nil { + return err + } + return mc.writeAuthSwitchPacket(enc, false) +} + +func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, bool, error) { + switch plugin { + case "caching_sha2_password": + authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) + return authResp, false, nil + + case "mysql_old_password": + if !mc.cfg.AllowOldPasswords { + return nil, false, ErrOldPassword + } + // Note: there are edge cases where this should work but doesn't; + // this is currently "wontfix": + // https://github.com/go-sql-driver/mysql/issues/184 + authResp := scrambleOldPassword(authData[:8], mc.cfg.Passwd) + return authResp, true, nil + + case "mysql_clear_password": + if !mc.cfg.AllowCleartextPasswords { + return nil, false, ErrCleartextPassword + } + // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html + // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html + return []byte(mc.cfg.Passwd), true, nil + + case "mysql_native_password": + if !mc.cfg.AllowNativePasswords { + return nil, false, ErrNativePassword + } + // https://dev.mysql.com/doc/internals/en/secure-password-authentication.html + // Native password authentication only need and will need 20-byte challenge. + authResp := scramblePassword(authData[:20], mc.cfg.Passwd) + return authResp, false, nil + + case "sha256_password": + if len(mc.cfg.Passwd) == 0 { + return nil, true, nil + } + if mc.cfg.tls != nil || mc.cfg.Net == "unix" { + // write cleartext auth packet + return []byte(mc.cfg.Passwd), true, nil + } + + pubKey := mc.cfg.pubKey + if pubKey == nil { + // request public key from server + return []byte{1}, false, nil + } + + // encrypted password + enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) + return enc, false, err + + default: + errLog.Print("unknown auth plugin:", plugin) + return nil, false, ErrUnknownPlugin + } +} + +func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { + // Read Result Packet + authData, newPlugin, err := mc.readAuthResult() + if err != nil { + return err + } + + // handle auth plugin switch, if requested + if newPlugin != "" { + // If CLIENT_PLUGIN_AUTH capability is not supported, no new cipher is + // sent and we have to keep using the cipher sent in the init packet. + if authData == nil { + authData = oldAuthData + } else { + // copy data from read buffer to owned slice + copy(oldAuthData, authData) + } + + plugin = newPlugin + + authResp, addNUL, err := mc.auth(authData, plugin) + if err != nil { + return err + } + if err = mc.writeAuthSwitchPacket(authResp, addNUL); err != nil { + return err + } + + // Read Result Packet + authData, newPlugin, err = mc.readAuthResult() + if err != nil { + return err + } + + // Do not allow to change the auth plugin more than once + if newPlugin != "" { + return ErrMalformPkt + } + } + + switch plugin { + + // https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/ + case "caching_sha2_password": + switch len(authData) { + case 0: + return nil // auth successful + case 1: + switch authData[0] { + case cachingSha2PasswordFastAuthSuccess: + if err = mc.readResultOK(); err == nil { + return nil // auth successful + } + + case cachingSha2PasswordPerformFullAuthentication: + if mc.cfg.tls != nil || mc.cfg.Net == "unix" { + // write cleartext auth packet + err = mc.writeAuthSwitchPacket([]byte(mc.cfg.Passwd), true) + if err != nil { + return err + } + } else { + pubKey := mc.cfg.pubKey + if pubKey == nil { + // request public key from server + data := mc.buf.takeSmallBuffer(4 + 1) + data[4] = cachingSha2PasswordRequestPublicKey + mc.writePacket(data) + + // parse public key + data, err := mc.readPacket() + if err != nil { + return err + } + + block, _ := pem.Decode(data[1:]) + pkix, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + pubKey = pkix.(*rsa.PublicKey) + } + + // send encrypted password + err = mc.sendEncryptedPassword(oldAuthData, pubKey) + if err != nil { + return err + } + } + return mc.readResultOK() + + default: + return ErrMalformPkt + } + default: + return ErrMalformPkt + } + + case "sha256_password": + switch len(authData) { + case 0: + return nil // auth successful + default: + block, _ := pem.Decode(authData) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + + // send encrypted password + err = mc.sendEncryptedPassword(oldAuthData, pub.(*rsa.PublicKey)) + if err != nil { + return err + } + return mc.readResultOK() + } + + default: + return nil // auth successful + } + + return err +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go new file mode 100644 index 0000000..eb4748b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -0,0 +1,147 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "io" + "net" + "time" +) + +const defaultBufSize = 4096 + +// A buffer which is used for both reading and writing. +// This is possible since communication on each connection is synchronous. +// In other words, we can't write and read simultaneously on the same connection. +// The buffer is similar to bufio.Reader / Writer but zero-copy-ish +// Also highly optimized for this particular use case. +type buffer struct { + buf []byte + nc net.Conn + idx int + length int + timeout time.Duration +} + +func newBuffer(nc net.Conn) buffer { + var b [defaultBufSize]byte + return buffer{ + buf: b[:], + nc: nc, + } +} + +// fill reads into the buffer until at least _need_ bytes are in it +func (b *buffer) fill(need int) error { + n := b.length + + // move existing data to the beginning + if n > 0 && b.idx > 0 { + copy(b.buf[0:n], b.buf[b.idx:]) + } + + // grow buffer if necessary + // TODO: let the buffer shrink again at some point + // Maybe keep the org buf slice and swap back? + if need > len(b.buf) { + // Round up to the next multiple of the default size + newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) + copy(newBuf, b.buf) + b.buf = newBuf + } + + b.idx = 0 + + for { + if b.timeout > 0 { + if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil { + return err + } + } + + nn, err := b.nc.Read(b.buf[n:]) + n += nn + + switch err { + case nil: + if n < need { + continue + } + b.length = n + return nil + + case io.EOF: + if n >= need { + b.length = n + return nil + } + return io.ErrUnexpectedEOF + + default: + return err + } + } +} + +// returns next N bytes from buffer. +// The returned slice is only guaranteed to be valid until the next read +func (b *buffer) readNext(need int) ([]byte, error) { + if b.length < need { + // refill + if err := b.fill(need); err != nil { + return nil, err + } + } + + offset := b.idx + b.idx += need + b.length -= need + return b.buf[offset:b.idx], nil +} + +// returns a buffer with the requested size. +// If possible, a slice from the existing buffer is returned. +// Otherwise a bigger buffer is made. +// Only one buffer (total) can be used at a time. +func (b *buffer) takeBuffer(length int) []byte { + if b.length > 0 { + return nil + } + + // test (cheap) general case first + if length <= defaultBufSize || length <= cap(b.buf) { + return b.buf[:length] + } + + if length < maxPacketSize { + b.buf = make([]byte, length) + return b.buf + } + return make([]byte, length) +} + +// shortcut which can be used if the requested buffer is guaranteed to be +// smaller than defaultBufSize +// Only one buffer (total) can be used at a time. +func (b *buffer) takeSmallBuffer(length int) []byte { + if b.length > 0 { + return nil + } + return b.buf[:length] +} + +// takeCompleteBuffer returns the complete existing buffer. +// This can be used if the necessary buffer size is unknown. +// Only one buffer (total) can be used at a time. +func (b *buffer) takeCompleteBuffer() []byte { + if b.length > 0 { + return nil + } + return b.buf +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go new file mode 100644 index 0000000..136c9e4 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go @@ -0,0 +1,251 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2014 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +const defaultCollation = "utf8_general_ci" +const binaryCollation = "binary" + +// A list of available collations mapped to the internal ID. +// To update this map use the following MySQL query: +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +var collations = map[string]byte{ + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + "ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + "utf16_general_ci": 54, + "utf16_bin": 55, + "utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + "utf32_general_ci": 60, + "utf32_bin": 61, + "utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + "ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + "utf16_unicode_ci": 101, + "utf16_icelandic_ci": 102, + "utf16_latvian_ci": 103, + "utf16_romanian_ci": 104, + "utf16_slovenian_ci": 105, + "utf16_polish_ci": 106, + "utf16_estonian_ci": 107, + "utf16_spanish_ci": 108, + "utf16_swedish_ci": 109, + "utf16_turkish_ci": 110, + "utf16_czech_ci": 111, + "utf16_danish_ci": 112, + "utf16_lithuanian_ci": 113, + "utf16_slovak_ci": 114, + "utf16_spanish2_ci": 115, + "utf16_roman_ci": 116, + "utf16_persian_ci": 117, + "utf16_esperanto_ci": 118, + "utf16_hungarian_ci": 119, + "utf16_sinhala_ci": 120, + "utf16_german2_ci": 121, + "utf16_croatian_ci": 122, + "utf16_unicode_520_ci": 123, + "utf16_vietnamese_ci": 124, + "ucs2_unicode_ci": 128, + "ucs2_icelandic_ci": 129, + "ucs2_latvian_ci": 130, + "ucs2_romanian_ci": 131, + "ucs2_slovenian_ci": 132, + "ucs2_polish_ci": 133, + "ucs2_estonian_ci": 134, + "ucs2_spanish_ci": 135, + "ucs2_swedish_ci": 136, + "ucs2_turkish_ci": 137, + "ucs2_czech_ci": 138, + "ucs2_danish_ci": 139, + "ucs2_lithuanian_ci": 140, + "ucs2_slovak_ci": 141, + "ucs2_spanish2_ci": 142, + "ucs2_roman_ci": 143, + "ucs2_persian_ci": 144, + "ucs2_esperanto_ci": 145, + "ucs2_hungarian_ci": 146, + "ucs2_sinhala_ci": 147, + "ucs2_german2_ci": 148, + "ucs2_croatian_ci": 149, + "ucs2_unicode_520_ci": 150, + "ucs2_vietnamese_ci": 151, + "ucs2_general_mysql500_ci": 159, + "utf32_unicode_ci": 160, + "utf32_icelandic_ci": 161, + "utf32_latvian_ci": 162, + "utf32_romanian_ci": 163, + "utf32_slovenian_ci": 164, + "utf32_polish_ci": 165, + "utf32_estonian_ci": 166, + "utf32_spanish_ci": 167, + "utf32_swedish_ci": 168, + "utf32_turkish_ci": 169, + "utf32_czech_ci": 170, + "utf32_danish_ci": 171, + "utf32_lithuanian_ci": 172, + "utf32_slovak_ci": 173, + "utf32_spanish2_ci": 174, + "utf32_roman_ci": 175, + "utf32_persian_ci": 176, + "utf32_esperanto_ci": 177, + "utf32_hungarian_ci": 178, + "utf32_sinhala_ci": 179, + "utf32_german2_ci": 180, + "utf32_croatian_ci": 181, + "utf32_unicode_520_ci": 182, + "utf32_vietnamese_ci": 183, + "utf8_unicode_ci": 192, + "utf8_icelandic_ci": 193, + "utf8_latvian_ci": 194, + "utf8_romanian_ci": 195, + "utf8_slovenian_ci": 196, + "utf8_polish_ci": 197, + "utf8_estonian_ci": 198, + "utf8_spanish_ci": 199, + "utf8_swedish_ci": 200, + "utf8_turkish_ci": 201, + "utf8_czech_ci": 202, + "utf8_danish_ci": 203, + "utf8_lithuanian_ci": 204, + "utf8_slovak_ci": 205, + "utf8_spanish2_ci": 206, + "utf8_roman_ci": 207, + "utf8_persian_ci": 208, + "utf8_esperanto_ci": 209, + "utf8_hungarian_ci": 210, + "utf8_sinhala_ci": 211, + "utf8_german2_ci": 212, + "utf8_croatian_ci": 213, + "utf8_unicode_520_ci": 214, + "utf8_vietnamese_ci": 215, + "utf8_general_mysql500_ci": 223, + "utf8mb4_unicode_ci": 224, + "utf8mb4_icelandic_ci": 225, + "utf8mb4_latvian_ci": 226, + "utf8mb4_romanian_ci": 227, + "utf8mb4_slovenian_ci": 228, + "utf8mb4_polish_ci": 229, + "utf8mb4_estonian_ci": 230, + "utf8mb4_spanish_ci": 231, + "utf8mb4_swedish_ci": 232, + "utf8mb4_turkish_ci": 233, + "utf8mb4_czech_ci": 234, + "utf8mb4_danish_ci": 235, + "utf8mb4_lithuanian_ci": 236, + "utf8mb4_slovak_ci": 237, + "utf8mb4_spanish2_ci": 238, + "utf8mb4_roman_ci": 239, + "utf8mb4_persian_ci": 240, + "utf8mb4_esperanto_ci": 241, + "utf8mb4_hungarian_ci": 242, + "utf8mb4_sinhala_ci": 243, + "utf8mb4_german2_ci": 244, + "utf8mb4_croatian_ci": 245, + "utf8mb4_unicode_520_ci": 246, + "utf8mb4_vietnamese_ci": 247, +} + +// A blacklist of collations which is unsafe to interpolate parameters. +// These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. +var unsafeCollations = map[string]bool{ + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go new file mode 100644 index 0000000..911be20 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go @@ -0,0 +1,654 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql" + "database/sql/driver" + "io" + "net" + "strconv" + "strings" + "time" +) + +// a copy of context.Context for Go 1.7 and earlier +type mysqlContext interface { + Done() <-chan struct{} + Err() error + + // defined in context.Context, but not used in this driver: + // Deadline() (deadline time.Time, ok bool) + // Value(key interface{}) interface{} +} + +type mysqlConn struct { + buf buffer + netConn net.Conn + affectedRows uint64 + insertId uint64 + cfg *Config + maxAllowedPacket int + maxWriteSize int + writeTimeout time.Duration + flags clientFlag + status statusFlag + sequence uint8 + parseTime bool + + // for context support (Go 1.8+) + watching bool + watcher chan<- mysqlContext + closech chan struct{} + finished chan<- struct{} + canceled atomicError // set non-nil if conn is canceled + closed atomicBool // set when conn is closed, before closech is closed +} + +// Handles parameters set in DSN after the connection is established +func (mc *mysqlConn) handleParams() (err error) { + for param, val := range mc.cfg.Params { + switch param { + // Charset + case "charset": + charsets := strings.Split(val, ",") + for i := range charsets { + // ignore errors here - a charset may not exist + err = mc.exec("SET NAMES " + charsets[i]) + if err == nil { + break + } + } + if err != nil { + return + } + + // System Vars + default: + err = mc.exec("SET " + param + "=" + val + "") + if err != nil { + return + } + } + } + + return +} + +func (mc *mysqlConn) markBadConn(err error) error { + if mc == nil { + return err + } + if err != errBadConnNoWrite { + return err + } + return driver.ErrBadConn +} + +func (mc *mysqlConn) Begin() (driver.Tx, error) { + return mc.begin(false) +} + +func (mc *mysqlConn) begin(readOnly bool) (driver.Tx, error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return nil, driver.ErrBadConn + } + var q string + if readOnly { + q = "START TRANSACTION READ ONLY" + } else { + q = "START TRANSACTION" + } + err := mc.exec(q) + if err == nil { + return &mysqlTx{mc}, err + } + return nil, mc.markBadConn(err) +} + +func (mc *mysqlConn) Close() (err error) { + // Makes Close idempotent + if !mc.closed.IsSet() { + err = mc.writeCommandPacket(comQuit) + } + + mc.cleanup() + + return +} + +// Closes the network connection and unsets internal variables. Do not call this +// function after successfully authentication, call Close instead. This function +// is called before auth or on auth failure because MySQL will have already +// closed the network connection. +func (mc *mysqlConn) cleanup() { + if !mc.closed.TrySet(true) { + return + } + + // Makes cleanup idempotent + close(mc.closech) + if mc.netConn == nil { + return + } + if err := mc.netConn.Close(); err != nil { + errLog.Print(err) + } +} + +func (mc *mysqlConn) error() error { + if mc.closed.IsSet() { + if err := mc.canceled.Value(); err != nil { + return err + } + return ErrInvalidConn + } + return nil +} + +func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return nil, driver.ErrBadConn + } + // Send command + err := mc.writeCommandPacketStr(comStmtPrepare, query) + if err != nil { + return nil, mc.markBadConn(err) + } + + stmt := &mysqlStmt{ + mc: mc, + } + + // Read Result + columnCount, err := stmt.readPrepareResultPacket() + if err == nil { + if stmt.paramCount > 0 { + if err = mc.readUntilEOF(); err != nil { + return nil, err + } + } + + if columnCount > 0 { + err = mc.readUntilEOF() + } + } + + return stmt, err +} + +func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) { + // Number of ? should be same to len(args) + if strings.Count(query, "?") != len(args) { + return "", driver.ErrSkip + } + + buf := mc.buf.takeCompleteBuffer() + if buf == nil { + // can not take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return "", ErrInvalidConn + } + buf = buf[:0] + argPos := 0 + + for i := 0; i < len(query); i++ { + q := strings.IndexByte(query[i:], '?') + if q == -1 { + buf = append(buf, query[i:]...) + break + } + buf = append(buf, query[i:i+q]...) + i += q + + arg := args[argPos] + argPos++ + + if arg == nil { + buf = append(buf, "NULL"...) + continue + } + + switch v := arg.(type) { + case int64: + buf = strconv.AppendInt(buf, v, 10) + case float64: + buf = strconv.AppendFloat(buf, v, 'g', -1, 64) + case bool: + if v { + buf = append(buf, '1') + } else { + buf = append(buf, '0') + } + case time.Time: + if v.IsZero() { + buf = append(buf, "'0000-00-00'"...) + } else { + v := v.In(mc.cfg.Loc) + v = v.Add(time.Nanosecond * 500) // To round under microsecond + year := v.Year() + year100 := year / 100 + year1 := year % 100 + month := v.Month() + day := v.Day() + hour := v.Hour() + minute := v.Minute() + second := v.Second() + micro := v.Nanosecond() / 1000 + + buf = append(buf, []byte{ + '\'', + digits10[year100], digits01[year100], + digits10[year1], digits01[year1], + '-', + digits10[month], digits01[month], + '-', + digits10[day], digits01[day], + ' ', + digits10[hour], digits01[hour], + ':', + digits10[minute], digits01[minute], + ':', + digits10[second], digits01[second], + }...) + + if micro != 0 { + micro10000 := micro / 10000 + micro100 := micro / 100 % 100 + micro1 := micro % 100 + buf = append(buf, []byte{ + '.', + digits10[micro10000], digits01[micro10000], + digits10[micro100], digits01[micro100], + digits10[micro1], digits01[micro1], + }...) + } + buf = append(buf, '\'') + } + case []byte: + if v == nil { + buf = append(buf, "NULL"...) + } else { + buf = append(buf, "_binary'"...) + if mc.status&statusNoBackslashEscapes == 0 { + buf = escapeBytesBackslash(buf, v) + } else { + buf = escapeBytesQuotes(buf, v) + } + buf = append(buf, '\'') + } + case string: + buf = append(buf, '\'') + if mc.status&statusNoBackslashEscapes == 0 { + buf = escapeStringBackslash(buf, v) + } else { + buf = escapeStringQuotes(buf, v) + } + buf = append(buf, '\'') + default: + return "", driver.ErrSkip + } + + if len(buf)+4 > mc.maxAllowedPacket { + return "", driver.ErrSkip + } + } + if argPos != len(args) { + return "", driver.ErrSkip + } + return string(buf), nil +} + +func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return nil, driver.ErrBadConn + } + if len(args) != 0 { + if !mc.cfg.InterpolateParams { + return nil, driver.ErrSkip + } + // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement + prepared, err := mc.interpolateParams(query, args) + if err != nil { + return nil, err + } + query = prepared + } + mc.affectedRows = 0 + mc.insertId = 0 + + err := mc.exec(query) + if err == nil { + return &mysqlResult{ + affectedRows: int64(mc.affectedRows), + insertId: int64(mc.insertId), + }, err + } + return nil, mc.markBadConn(err) +} + +// Internal function to execute commands +func (mc *mysqlConn) exec(query string) error { + // Send command + if err := mc.writeCommandPacketStr(comQuery, query); err != nil { + return mc.markBadConn(err) + } + + // Read Result + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return err + } + + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { + return err + } + + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } + } + + return mc.discardResults() +} + +func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) { + return mc.query(query, args) +} + +func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return nil, driver.ErrBadConn + } + if len(args) != 0 { + if !mc.cfg.InterpolateParams { + return nil, driver.ErrSkip + } + // try client-side prepare to reduce roundtrip + prepared, err := mc.interpolateParams(query, args) + if err != nil { + return nil, err + } + query = prepared + } + // Send command + err := mc.writeCommandPacketStr(comQuery, query) + if err == nil { + // Read Result + var resLen int + resLen, err = mc.readResultSetHeaderPacket() + if err == nil { + rows := new(textRows) + rows.mc = mc + + if resLen == 0 { + rows.rs.done = true + + switch err := rows.NextResultSet(); err { + case nil, io.EOF: + return rows, nil + default: + return nil, err + } + } + + // Columns + rows.rs.columns, err = mc.readColumns(resLen) + return rows, err + } + } + return nil, mc.markBadConn(err) +} + +// Gets the value of the given MySQL System Variable +// The returned byte slice is only valid until the next read +func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { + // Send command + if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil { + return nil, err + } + + // Read Result + resLen, err := mc.readResultSetHeaderPacket() + if err == nil { + rows := new(textRows) + rows.mc = mc + rows.rs.columns = []mysqlField{{fieldType: fieldTypeVarChar}} + + if resLen > 0 { + // Columns + if err := mc.readUntilEOF(); err != nil { + return nil, err + } + } + + dest := make([]driver.Value, resLen) + if err = rows.readRow(dest); err == nil { + return dest[0].([]byte), mc.readUntilEOF() + } + } + return nil, err +} + +// finish is called when the query has canceled. +func (mc *mysqlConn) cancel(err error) { + mc.canceled.Set(err) + mc.cleanup() +} + +// finish is called when the query has succeeded. +func (mc *mysqlConn) finish() { + if !mc.watching || mc.finished == nil { + return + } + select { + case mc.finished <- struct{}{}: + mc.watching = false + case <-mc.closech: + } +} + +// Ping implements driver.Pinger interface +func (mc *mysqlConn) Ping(ctx context.Context) (err error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return driver.ErrBadConn + } + + if err = mc.watchCancel(ctx); err != nil { + return + } + defer mc.finish() + + if err = mc.writeCommandPacket(comPing); err != nil { + return + } + + return mc.readResultOK() +} + +// BeginTx implements driver.ConnBeginTx interface +func (mc *mysqlConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { + level, err := mapIsolationLevel(opts.Isolation) + if err != nil { + return nil, err + } + err = mc.exec("SET TRANSACTION ISOLATION LEVEL " + level) + if err != nil { + return nil, err + } + } + + return mc.begin(opts.ReadOnly) +} + +func (mc *mysqlConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := mc.query(query, dargs) + if err != nil { + mc.finish() + return nil, err + } + rows.finish = mc.finish + return rows, err +} + +func (mc *mysqlConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + return mc.Exec(query, dargs) +} + +func (mc *mysqlConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + stmt, err := mc.Prepare(query) + mc.finish() + if err != nil { + return nil, err + } + + select { + default: + case <-ctx.Done(): + stmt.Close() + return nil, ctx.Err() + } + return stmt, nil +} + +func (stmt *mysqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := stmt.query(dargs) + if err != nil { + stmt.mc.finish() + return nil, err + } + rows.finish = stmt.mc.finish + return rows, err +} + +func (stmt *mysqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + defer stmt.mc.finish() + + return stmt.Exec(dargs) +} + +func (mc *mysqlConn) watchCancel(ctx context.Context) error { + if mc.watching { + // Reach here if canceled, + // so the connection is already invalid + mc.cleanup() + return nil + } + if ctx.Done() == nil { + return nil + } + + mc.watching = true + select { + default: + case <-ctx.Done(): + return ctx.Err() + } + if mc.watcher == nil { + return nil + } + + mc.watcher <- ctx + + return nil +} + +func (mc *mysqlConn) startWatcher() { + watcher := make(chan mysqlContext, 1) + mc.watcher = watcher + finished := make(chan struct{}) + mc.finished = finished + go func() { + for { + var ctx mysqlContext + select { + case ctx = <-watcher: + case <-mc.closech: + return + } + + select { + case <-ctx.Done(): + mc.cancel(ctx.Err()) + case <-finished: + case <-mc.closech: + return + } + } + }() +} + +func (mc *mysqlConn) CheckNamedValue(nv *driver.NamedValue) (err error) { + nv.Value, err = converter{}.ConvertValue(nv.Value) + return +} + +// ResetSession implements driver.SessionResetter. +// (From Go 1.10) +func (mc *mysqlConn) ResetSession(ctx context.Context) error { + if mc.closed.IsSet() { + return driver.ErrBadConn + } + return nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/const.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/const.go new file mode 100644 index 0000000..b1e6b85 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/const.go @@ -0,0 +1,174 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +const ( + defaultAuthPlugin = "mysql_native_password" + defaultMaxAllowedPacket = 4 << 20 // 4 MiB + minProtocolVersion = 10 + maxPacketSize = 1<<24 - 1 + timeFormat = "2006-01-02 15:04:05.999999" +) + +// MySQL constants documentation: +// http://dev.mysql.com/doc/internals/en/client-server-protocol.html + +const ( + iOK byte = 0x00 + iAuthMoreData byte = 0x01 + iLocalInFile byte = 0xfb + iEOF byte = 0xfe + iERR byte = 0xff +) + +// https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags +type clientFlag uint32 + +const ( + clientLongPassword clientFlag = 1 << iota + clientFoundRows + clientLongFlag + clientConnectWithDB + clientNoSchema + clientCompress + clientODBC + clientLocalFiles + clientIgnoreSpace + clientProtocol41 + clientInteractive + clientSSL + clientIgnoreSIGPIPE + clientTransactions + clientReserved + clientSecureConn + clientMultiStatements + clientMultiResults + clientPSMultiResults + clientPluginAuth + clientConnectAttrs + clientPluginAuthLenEncClientData + clientCanHandleExpiredPasswords + clientSessionTrack + clientDeprecateEOF +) + +const ( + comQuit byte = iota + 1 + comInitDB + comQuery + comFieldList + comCreateDB + comDropDB + comRefresh + comShutdown + comStatistics + comProcessInfo + comConnect + comProcessKill + comDebug + comPing + comTime + comDelayedInsert + comChangeUser + comBinlogDump + comTableDump + comConnectOut + comRegisterSlave + comStmtPrepare + comStmtExecute + comStmtSendLongData + comStmtClose + comStmtReset + comSetOption + comStmtFetch +) + +// https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnType +type fieldType byte + +const ( + fieldTypeDecimal fieldType = iota + fieldTypeTiny + fieldTypeShort + fieldTypeLong + fieldTypeFloat + fieldTypeDouble + fieldTypeNULL + fieldTypeTimestamp + fieldTypeLongLong + fieldTypeInt24 + fieldTypeDate + fieldTypeTime + fieldTypeDateTime + fieldTypeYear + fieldTypeNewDate + fieldTypeVarChar + fieldTypeBit +) +const ( + fieldTypeJSON fieldType = iota + 0xf5 + fieldTypeNewDecimal + fieldTypeEnum + fieldTypeSet + fieldTypeTinyBLOB + fieldTypeMediumBLOB + fieldTypeLongBLOB + fieldTypeBLOB + fieldTypeVarString + fieldTypeString + fieldTypeGeometry +) + +type fieldFlag uint16 + +const ( + flagNotNULL fieldFlag = 1 << iota + flagPriKey + flagUniqueKey + flagMultipleKey + flagBLOB + flagUnsigned + flagZeroFill + flagBinary + flagEnum + flagAutoIncrement + flagTimestamp + flagSet + flagUnknown1 + flagUnknown2 + flagUnknown3 + flagUnknown4 +) + +// http://dev.mysql.com/doc/internals/en/status-flags.html +type statusFlag uint16 + +const ( + statusInTrans statusFlag = 1 << iota + statusInAutocommit + statusReserved // Not in documentation + statusMoreResultsExists + statusNoGoodIndexUsed + statusNoIndexUsed + statusCursorExists + statusLastRowSent + statusDbDropped + statusNoBackslashEscapes + statusMetadataChanged + statusQueryWasSlow + statusPsOutParams + statusInTransReadonly + statusSessionStateChanged +) + +const ( + cachingSha2PasswordRequestPublicKey = 2 + cachingSha2PasswordFastAuthSuccess = 3 + cachingSha2PasswordPerformFullAuthentication = 4 +) diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go new file mode 100644 index 0000000..ba12978 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go @@ -0,0 +1,165 @@ +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package mysql provides a MySQL driver for Go's database/sql package. +// +// The driver should be used via the database/sql package: +// +// import "database/sql" +// import _ "github.com/go-sql-driver/mysql" +// +// db, err := sql.Open("mysql", "user:password@/dbname") +// +// See https://github.com/go-sql-driver/mysql#usage for details +package mysql + +import ( + "database/sql" + "database/sql/driver" + "net" + "sync" +) + +// MySQLDriver is exported to make the driver directly accessible. +// In general the driver is used via the database/sql package. +type MySQLDriver struct{} + +// DialFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDial +type DialFunc func(addr string) (net.Conn, error) + +var ( + dialsLock sync.RWMutex + dials map[string]DialFunc +) + +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +func RegisterDial(net string, dial DialFunc) { + dialsLock.Lock() + defer dialsLock.Unlock() + if dials == nil { + dials = make(map[string]DialFunc) + } + dials[net] = dial +} + +// Open new Connection. +// See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how +// the DSN string is formated +func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + } + mc.cfg, err = ParseDSN(dsn) + if err != nil { + return nil, err + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + mc.netConn, err = dial(mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) + } + if err != nil { + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, addNUL, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, addNUL, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +func init() { + sql.Register("mysql", &MySQLDriver{}) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go new file mode 100644 index 0000000..be014ba --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -0,0 +1,611 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rsa" + "crypto/tls" + "errors" + "fmt" + "net" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +var ( + errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") + errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") + errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") + errInvalidDSNUnsafeCollation = errors.New("invalid DSN: interpolateParams can not be used with unsafe collations") +) + +// Config is a configuration parsed from a DSN string. +// If a new Config is created instead of being parsed from a DSN string, +// the NewConfig function should be used, which sets default values. +type Config struct { + User string // Username + Passwd string // Password (requires User) + Net string // Network type + Addr string // Network address (requires Net) + DBName string // Database name + Params map[string]string // Connection parameters + Collation string // Connection collation + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + ServerPubKey string // Server public key name + pubKey *rsa.PublicKey // Server public key + TLSConfig string // TLS configuration name + tls *tls.Config // TLS configuration + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + + AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE + AllowCleartextPasswords bool // Allows the cleartext client side plugin + AllowNativePasswords bool // Allows the native password authentication method + AllowOldPasswords bool // Allows the old insecure password method + ClientFoundRows bool // Return number of matching rows instead of rows changed + ColumnsWithAlias bool // Prepend table alias to column names + InterpolateParams bool // Interpolate placeholders into query string + MultiStatements bool // Allow multiple statements in one query + ParseTime bool // Parse time values to time.Time + RejectReadOnly bool // Reject read-only connections +} + +// NewConfig creates a new Config and sets default values. +func NewConfig() *Config { + return &Config{ + Collation: defaultCollation, + Loc: time.UTC, + MaxAllowedPacket: defaultMaxAllowedPacket, + AllowNativePasswords: true, + } +} + +func (cfg *Config) normalize() error { + if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { + return errInvalidDSNUnsafeCollation + } + + // Set default network if empty + if cfg.Net == "" { + cfg.Net = "tcp" + } + + // Set default address if empty + if cfg.Addr == "" { + switch cfg.Net { + case "tcp": + cfg.Addr = "127.0.0.1:3306" + case "unix": + cfg.Addr = "/tmp/mysql.sock" + default: + return errors.New("default addr for network '" + cfg.Net + "' unknown") + } + + } else if cfg.Net == "tcp" { + cfg.Addr = ensureHavePort(cfg.Addr) + } + + if cfg.tls != nil { + if cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + } + + return nil +} + +// FormatDSN formats the given Config into a DSN string which can be passed to +// the driver. +func (cfg *Config) FormatDSN() string { + var buf bytes.Buffer + + // [username[:password]@] + if len(cfg.User) > 0 { + buf.WriteString(cfg.User) + if len(cfg.Passwd) > 0 { + buf.WriteByte(':') + buf.WriteString(cfg.Passwd) + } + buf.WriteByte('@') + } + + // [protocol[(address)]] + if len(cfg.Net) > 0 { + buf.WriteString(cfg.Net) + if len(cfg.Addr) > 0 { + buf.WriteByte('(') + buf.WriteString(cfg.Addr) + buf.WriteByte(')') + } + } + + // /dbname + buf.WriteByte('/') + buf.WriteString(cfg.DBName) + + // [?param1=value1&...¶mN=valueN] + hasParam := false + + if cfg.AllowAllFiles { + hasParam = true + buf.WriteString("?allowAllFiles=true") + } + + if cfg.AllowCleartextPasswords { + if hasParam { + buf.WriteString("&allowCleartextPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowCleartextPasswords=true") + } + } + + if !cfg.AllowNativePasswords { + if hasParam { + buf.WriteString("&allowNativePasswords=false") + } else { + hasParam = true + buf.WriteString("?allowNativePasswords=false") + } + } + + if cfg.AllowOldPasswords { + if hasParam { + buf.WriteString("&allowOldPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowOldPasswords=true") + } + } + + if cfg.ClientFoundRows { + if hasParam { + buf.WriteString("&clientFoundRows=true") + } else { + hasParam = true + buf.WriteString("?clientFoundRows=true") + } + } + + if col := cfg.Collation; col != defaultCollation && len(col) > 0 { + if hasParam { + buf.WriteString("&collation=") + } else { + hasParam = true + buf.WriteString("?collation=") + } + buf.WriteString(col) + } + + if cfg.ColumnsWithAlias { + if hasParam { + buf.WriteString("&columnsWithAlias=true") + } else { + hasParam = true + buf.WriteString("?columnsWithAlias=true") + } + } + + if cfg.InterpolateParams { + if hasParam { + buf.WriteString("&interpolateParams=true") + } else { + hasParam = true + buf.WriteString("?interpolateParams=true") + } + } + + if cfg.Loc != time.UTC && cfg.Loc != nil { + if hasParam { + buf.WriteString("&loc=") + } else { + hasParam = true + buf.WriteString("?loc=") + } + buf.WriteString(url.QueryEscape(cfg.Loc.String())) + } + + if cfg.MultiStatements { + if hasParam { + buf.WriteString("&multiStatements=true") + } else { + hasParam = true + buf.WriteString("?multiStatements=true") + } + } + + if cfg.ParseTime { + if hasParam { + buf.WriteString("&parseTime=true") + } else { + hasParam = true + buf.WriteString("?parseTime=true") + } + } + + if cfg.ReadTimeout > 0 { + if hasParam { + buf.WriteString("&readTimeout=") + } else { + hasParam = true + buf.WriteString("?readTimeout=") + } + buf.WriteString(cfg.ReadTimeout.String()) + } + + if cfg.RejectReadOnly { + if hasParam { + buf.WriteString("&rejectReadOnly=true") + } else { + hasParam = true + buf.WriteString("?rejectReadOnly=true") + } + } + + if len(cfg.ServerPubKey) > 0 { + if hasParam { + buf.WriteString("&serverPubKey=") + } else { + hasParam = true + buf.WriteString("?serverPubKey=") + } + buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + } + + if cfg.Timeout > 0 { + if hasParam { + buf.WriteString("&timeout=") + } else { + hasParam = true + buf.WriteString("?timeout=") + } + buf.WriteString(cfg.Timeout.String()) + } + + if len(cfg.TLSConfig) > 0 { + if hasParam { + buf.WriteString("&tls=") + } else { + hasParam = true + buf.WriteString("?tls=") + } + buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + } + + if cfg.WriteTimeout > 0 { + if hasParam { + buf.WriteString("&writeTimeout=") + } else { + hasParam = true + buf.WriteString("?writeTimeout=") + } + buf.WriteString(cfg.WriteTimeout.String()) + } + + if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { + if hasParam { + buf.WriteString("&maxAllowedPacket=") + } else { + hasParam = true + buf.WriteString("?maxAllowedPacket=") + } + buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) + + } + + // other params + if cfg.Params != nil { + var params []string + for param := range cfg.Params { + params = append(params, param) + } + sort.Strings(params) + for _, param := range params { + if hasParam { + buf.WriteByte('&') + } else { + hasParam = true + buf.WriteByte('?') + } + + buf.WriteString(param) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(cfg.Params[param])) + } + } + + return buf.String() +} + +// ParseDSN parses the DSN string to a Config +func ParseDSN(dsn string) (cfg *Config, err error) { + // New config with some default values + cfg = NewConfig() + + // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] + // Find the last '/' (since the password or the net addr might contain a '/') + foundSlash := false + for i := len(dsn) - 1; i >= 0; i-- { + if dsn[i] == '/' { + foundSlash = true + var j, k int + + // left part is empty if i <= 0 + if i > 0 { + // [username[:password]@][protocol[(address)]] + // Find the last '@' in dsn[:i] + for j = i; j >= 0; j-- { + if dsn[j] == '@' { + // username[:password] + // Find the first ':' in dsn[:j] + for k = 0; k < j; k++ { + if dsn[k] == ':' { + cfg.Passwd = dsn[k+1 : j] + break + } + } + cfg.User = dsn[:k] + + break + } + } + + // [protocol[(address)]] + // Find the first '(' in dsn[j+1:i] + for k = j + 1; k < i; k++ { + if dsn[k] == '(' { + // dsn[i-1] must be == ')' if an address is specified + if dsn[i-1] != ')' { + if strings.ContainsRune(dsn[k+1:i], ')') { + return nil, errInvalidDSNUnescaped + } + return nil, errInvalidDSNAddr + } + cfg.Addr = dsn[k+1 : i-1] + break + } + } + cfg.Net = dsn[j+1 : k] + } + + // dbname[?param1=value1&...¶mN=valueN] + // Find the first '?' in dsn[i+1:] + for j = i + 1; j < len(dsn); j++ { + if dsn[j] == '?' { + if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { + return + } + break + } + } + cfg.DBName = dsn[i+1 : j] + + break + } + } + + if !foundSlash && len(dsn) > 0 { + return nil, errInvalidDSNNoSlash + } + + if err = cfg.normalize(); err != nil { + return nil, err + } + return +} + +// parseDSNParams parses the DSN "query string" +// Values must be url.QueryEscape'ed +func parseDSNParams(cfg *Config, params string) (err error) { + for _, v := range strings.Split(params, "&") { + param := strings.SplitN(v, "=", 2) + if len(param) != 2 { + continue + } + + // cfg params + switch value := param[1]; param[0] { + // Disable INFILE whitelist / enable all files + case "allowAllFiles": + var isBool bool + cfg.AllowAllFiles, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use cleartext authentication mode (MySQL 5.5.10+) + case "allowCleartextPasswords": + var isBool bool + cfg.AllowCleartextPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use native password authentication + case "allowNativePasswords": + var isBool bool + cfg.AllowNativePasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use old authentication mode (pre MySQL 4.1) + case "allowOldPasswords": + var isBool bool + cfg.AllowOldPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Switch "rowsAffected" mode + case "clientFoundRows": + var isBool bool + cfg.ClientFoundRows, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Collation + case "collation": + cfg.Collation = value + break + + case "columnsWithAlias": + var isBool bool + cfg.ColumnsWithAlias, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Compression + case "compress": + return errors.New("compression not implemented yet") + + // Enable client side placeholder substitution + case "interpolateParams": + var isBool bool + cfg.InterpolateParams, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Time Location + case "loc": + if value, err = url.QueryUnescape(value); err != nil { + return + } + cfg.Loc, err = time.LoadLocation(value) + if err != nil { + return + } + + // multiple statements in one query + case "multiStatements": + var isBool bool + cfg.MultiStatements, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // time.Time parsing + case "parseTime": + var isBool bool + cfg.ParseTime, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // I/O read Timeout + case "readTimeout": + cfg.ReadTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // Reject read-only connections + case "rejectReadOnly": + var isBool bool + cfg.RejectReadOnly, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Server public key + case "serverPubKey": + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for server pub key name: %v", err) + } + + if pubKey := getServerPubKey(name); pubKey != nil { + cfg.ServerPubKey = name + cfg.pubKey = pubKey + } else { + return errors.New("invalid value / unknown server pub key name: " + name) + } + + // Strict mode + case "strict": + panic("strict mode has been removed. See https://github.com/go-sql-driver/mysql/wiki/strict-mode") + + // Dial Timeout + case "timeout": + cfg.Timeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // TLS-Encryption + case "tls": + boolValue, isBool := readBool(value) + if isBool { + if boolValue { + cfg.TLSConfig = "true" + cfg.tls = &tls.Config{} + } else { + cfg.TLSConfig = "false" + } + } else if vl := strings.ToLower(value); vl == "skip-verify" { + cfg.TLSConfig = vl + cfg.tls = &tls.Config{InsecureSkipVerify: true} + } else { + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for TLS config name: %v", err) + } + + if tlsConfig := getTLSConfigClone(name); tlsConfig != nil { + cfg.TLSConfig = name + cfg.tls = tlsConfig + } else { + return errors.New("invalid value / unknown config name: " + name) + } + } + + // I/O write Timeout + case "writeTimeout": + cfg.WriteTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + case "maxAllowedPacket": + cfg.MaxAllowedPacket, err = strconv.Atoi(value) + if err != nil { + return + } + default: + // lazy init + if cfg.Params == nil { + cfg.Params = make(map[string]string) + } + + if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { + return + } + } + } + + return +} + +func ensureHavePort(addr string) string { + if _, _, err := net.SplitHostPort(addr); err != nil { + return net.JoinHostPort(addr, "3306") + } + return addr +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors.go new file mode 100644 index 0000000..760782f --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors.go @@ -0,0 +1,65 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "errors" + "fmt" + "log" + "os" +) + +// Various errors the driver might return. Can change between driver versions. +var ( + ErrInvalidConn = errors.New("invalid connection") + ErrMalformPkt = errors.New("malformed packet") + ErrNoTLS = errors.New("TLS requested but server does not support TLS") + ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") + ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") + ErrUnknownPlugin = errors.New("this authentication plugin is not supported") + ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") + ErrPktSync = errors.New("commands out of sync. You can't run this command now") + ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") + ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") + ErrBusyBuffer = errors.New("busy buffer") + + // errBadConnNoWrite is used for connection errors where nothing was sent to the database yet. + // If this happens first in a function starting a database interaction, it should be replaced by driver.ErrBadConn + // to trigger a resend. + // See https://github.com/go-sql-driver/mysql/pull/302 + errBadConnNoWrite = errors.New("bad connection") +) + +var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) + +// Logger is used to log critical error messages. +type Logger interface { + Print(v ...interface{}) +} + +// SetLogger is used to set the logger for critical errors. +// The initial logger is os.Stderr. +func SetLogger(logger Logger) error { + if logger == nil { + return errors.New("logger is nil") + } + errLog = logger + return nil +} + +// MySQLError is an error type which represents a single MySQL error +type MySQLError struct { + Number uint16 + Message string +} + +func (me *MySQLError) Error() string { + return fmt.Sprintf("Error %d: %s", me.Number, me.Message) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/fields.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/fields.go new file mode 100644 index 0000000..e1e2ece --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/fields.go @@ -0,0 +1,194 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql" + "reflect" +) + +func (mf *mysqlField) typeDatabaseName() string { + switch mf.fieldType { + case fieldTypeBit: + return "BIT" + case fieldTypeBLOB: + if mf.charSet != collations[binaryCollation] { + return "TEXT" + } + return "BLOB" + case fieldTypeDate: + return "DATE" + case fieldTypeDateTime: + return "DATETIME" + case fieldTypeDecimal: + return "DECIMAL" + case fieldTypeDouble: + return "DOUBLE" + case fieldTypeEnum: + return "ENUM" + case fieldTypeFloat: + return "FLOAT" + case fieldTypeGeometry: + return "GEOMETRY" + case fieldTypeInt24: + return "MEDIUMINT" + case fieldTypeJSON: + return "JSON" + case fieldTypeLong: + return "INT" + case fieldTypeLongBLOB: + if mf.charSet != collations[binaryCollation] { + return "LONGTEXT" + } + return "LONGBLOB" + case fieldTypeLongLong: + return "BIGINT" + case fieldTypeMediumBLOB: + if mf.charSet != collations[binaryCollation] { + return "MEDIUMTEXT" + } + return "MEDIUMBLOB" + case fieldTypeNewDate: + return "DATE" + case fieldTypeNewDecimal: + return "DECIMAL" + case fieldTypeNULL: + return "NULL" + case fieldTypeSet: + return "SET" + case fieldTypeShort: + return "SMALLINT" + case fieldTypeString: + if mf.charSet == collations[binaryCollation] { + return "BINARY" + } + return "CHAR" + case fieldTypeTime: + return "TIME" + case fieldTypeTimestamp: + return "TIMESTAMP" + case fieldTypeTiny: + return "TINYINT" + case fieldTypeTinyBLOB: + if mf.charSet != collations[binaryCollation] { + return "TINYTEXT" + } + return "TINYBLOB" + case fieldTypeVarChar: + if mf.charSet == collations[binaryCollation] { + return "VARBINARY" + } + return "VARCHAR" + case fieldTypeVarString: + if mf.charSet == collations[binaryCollation] { + return "VARBINARY" + } + return "VARCHAR" + case fieldTypeYear: + return "YEAR" + default: + return "" + } +} + +var ( + scanTypeFloat32 = reflect.TypeOf(float32(0)) + scanTypeFloat64 = reflect.TypeOf(float64(0)) + scanTypeInt8 = reflect.TypeOf(int8(0)) + scanTypeInt16 = reflect.TypeOf(int16(0)) + scanTypeInt32 = reflect.TypeOf(int32(0)) + scanTypeInt64 = reflect.TypeOf(int64(0)) + scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) + scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) + scanTypeNullTime = reflect.TypeOf(NullTime{}) + scanTypeUint8 = reflect.TypeOf(uint8(0)) + scanTypeUint16 = reflect.TypeOf(uint16(0)) + scanTypeUint32 = reflect.TypeOf(uint32(0)) + scanTypeUint64 = reflect.TypeOf(uint64(0)) + scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) + scanTypeUnknown = reflect.TypeOf(new(interface{})) +) + +type mysqlField struct { + tableName string + name string + length uint32 + flags fieldFlag + fieldType fieldType + decimals byte + charSet uint8 +} + +func (mf *mysqlField) scanType() reflect.Type { + switch mf.fieldType { + case fieldTypeTiny: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint8 + } + return scanTypeInt8 + } + return scanTypeNullInt + + case fieldTypeShort, fieldTypeYear: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint16 + } + return scanTypeInt16 + } + return scanTypeNullInt + + case fieldTypeInt24, fieldTypeLong: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint32 + } + return scanTypeInt32 + } + return scanTypeNullInt + + case fieldTypeLongLong: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint64 + } + return scanTypeInt64 + } + return scanTypeNullInt + + case fieldTypeFloat: + if mf.flags&flagNotNULL != 0 { + return scanTypeFloat32 + } + return scanTypeNullFloat + + case fieldTypeDouble: + if mf.flags&flagNotNULL != 0 { + return scanTypeFloat64 + } + return scanTypeNullFloat + + case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, + fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, + fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, + fieldTypeTime: + return scanTypeRawBytes + + case fieldTypeDate, fieldTypeNewDate, + fieldTypeTimestamp, fieldTypeDateTime: + // NullTime is always returned for more consistent behavior as it can + // handle both cases of parseTime regardless if the field is nullable. + return scanTypeNullTime + + default: + return scanTypeUnknown + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/infile.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/infile.go new file mode 100644 index 0000000..273cb0b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/infile.go @@ -0,0 +1,182 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "fmt" + "io" + "os" + "strings" + "sync" +) + +var ( + fileRegister map[string]bool + fileRegisterLock sync.RWMutex + readerRegister map[string]func() io.Reader + readerRegisterLock sync.RWMutex +) + +// RegisterLocalFile adds the given file to the file whitelist, +// so that it can be used by "LOAD DATA LOCAL INFILE ". +// Alternatively you can allow the use of all local files with +// the DSN parameter 'allowAllFiles=true' +// +// filePath := "/home/gopher/data.csv" +// mysql.RegisterLocalFile(filePath) +// err := db.Exec("LOAD DATA LOCAL INFILE '" + filePath + "' INTO TABLE foo") +// if err != nil { +// ... +// +func RegisterLocalFile(filePath string) { + fileRegisterLock.Lock() + // lazy map init + if fileRegister == nil { + fileRegister = make(map[string]bool) + } + + fileRegister[strings.Trim(filePath, `"`)] = true + fileRegisterLock.Unlock() +} + +// DeregisterLocalFile removes the given filepath from the whitelist. +func DeregisterLocalFile(filePath string) { + fileRegisterLock.Lock() + delete(fileRegister, strings.Trim(filePath, `"`)) + fileRegisterLock.Unlock() +} + +// RegisterReaderHandler registers a handler function which is used +// to receive a io.Reader. +// The Reader can be used by "LOAD DATA LOCAL INFILE Reader::". +// If the handler returns a io.ReadCloser Close() is called when the +// request is finished. +// +// mysql.RegisterReaderHandler("data", func() io.Reader { +// var csvReader io.Reader // Some Reader that returns CSV data +// ... // Open Reader here +// return csvReader +// }) +// err := db.Exec("LOAD DATA LOCAL INFILE 'Reader::data' INTO TABLE foo") +// if err != nil { +// ... +// +func RegisterReaderHandler(name string, handler func() io.Reader) { + readerRegisterLock.Lock() + // lazy map init + if readerRegister == nil { + readerRegister = make(map[string]func() io.Reader) + } + + readerRegister[name] = handler + readerRegisterLock.Unlock() +} + +// DeregisterReaderHandler removes the ReaderHandler function with +// the given name from the registry. +func DeregisterReaderHandler(name string) { + readerRegisterLock.Lock() + delete(readerRegister, name) + readerRegisterLock.Unlock() +} + +func deferredClose(err *error, closer io.Closer) { + closeErr := closer.Close() + if *err == nil { + *err = closeErr + } +} + +func (mc *mysqlConn) handleInFileRequest(name string) (err error) { + var rdr io.Reader + var data []byte + packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP + if mc.maxWriteSize < packetSize { + packetSize = mc.maxWriteSize + } + + if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader + // The server might return an an absolute path. See issue #355. + name = name[idx+8:] + + readerRegisterLock.RLock() + handler, inMap := readerRegister[name] + readerRegisterLock.RUnlock() + + if inMap { + rdr = handler() + if rdr != nil { + if cl, ok := rdr.(io.Closer); ok { + defer deferredClose(&err, cl) + } + } else { + err = fmt.Errorf("Reader '%s' is ", name) + } + } else { + err = fmt.Errorf("Reader '%s' is not registered", name) + } + } else { // File + name = strings.Trim(name, `"`) + fileRegisterLock.RLock() + fr := fileRegister[name] + fileRegisterLock.RUnlock() + if mc.cfg.AllowAllFiles || fr { + var file *os.File + var fi os.FileInfo + + if file, err = os.Open(name); err == nil { + defer deferredClose(&err, file) + + // get file size + if fi, err = file.Stat(); err == nil { + rdr = file + if fileSize := int(fi.Size()); fileSize < packetSize { + packetSize = fileSize + } + } + } + } else { + err = fmt.Errorf("local file '%s' is not registered", name) + } + } + + // send content packets + // if packetSize == 0, the Reader contains no data + if err == nil && packetSize > 0 { + data := make([]byte, 4+packetSize) + var n int + for err == nil { + n, err = rdr.Read(data[4:]) + if n > 0 { + if ioErr := mc.writePacket(data[:4+n]); ioErr != nil { + return ioErr + } + } + } + if err == io.EOF { + err = nil + } + } + + // send empty packet (termination) + if data == nil { + data = make([]byte, 4) + } + if ioErr := mc.writePacket(data[:4]); ioErr != nil { + return ioErr + } + + // read OK packet + if err == nil { + return mc.readResultOK() + } + + mc.readPacket() + return err +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go new file mode 100644 index 0000000..170aaa0 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go @@ -0,0 +1,1298 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/tls" + "database/sql/driver" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "time" +) + +// Packets documentation: +// http://dev.mysql.com/doc/internals/en/client-server-protocol.html + +// Read packet to buffer 'data' +func (mc *mysqlConn) readPacket() ([]byte, error) { + var prevData []byte + for { + // read packet header + data, err := mc.buf.readNext(4) + if err != nil { + if cerr := mc.canceled.Value(); cerr != nil { + return nil, cerr + } + errLog.Print(err) + mc.Close() + return nil, ErrInvalidConn + } + + // packet length [24 bit] + pktLen := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) + + // check packet sync [8 bit] + if data[3] != mc.sequence { + if data[3] > mc.sequence { + return nil, ErrPktSyncMul + } + return nil, ErrPktSync + } + mc.sequence++ + + // packets with length 0 terminate a previous packet which is a + // multiple of (2^24)−1 bytes long + if pktLen == 0 { + // there was no previous packet + if prevData == nil { + errLog.Print(ErrMalformPkt) + mc.Close() + return nil, ErrInvalidConn + } + + return prevData, nil + } + + // read packet body [pktLen bytes] + data, err = mc.buf.readNext(pktLen) + if err != nil { + if cerr := mc.canceled.Value(); cerr != nil { + return nil, cerr + } + errLog.Print(err) + mc.Close() + return nil, ErrInvalidConn + } + + // return data if this was the last packet + if pktLen < maxPacketSize { + // zero allocations for non-split packets + if prevData == nil { + return data, nil + } + + return append(prevData, data...), nil + } + + prevData = append(prevData, data...) + } +} + +// Write packet buffer 'data' +func (mc *mysqlConn) writePacket(data []byte) error { + pktLen := len(data) - 4 + + if pktLen > mc.maxAllowedPacket { + return ErrPktTooLarge + } + + for { + var size int + if pktLen >= maxPacketSize { + data[0] = 0xff + data[1] = 0xff + data[2] = 0xff + size = maxPacketSize + } else { + data[0] = byte(pktLen) + data[1] = byte(pktLen >> 8) + data[2] = byte(pktLen >> 16) + size = pktLen + } + data[3] = mc.sequence + + // Write packet + if mc.writeTimeout > 0 { + if err := mc.netConn.SetWriteDeadline(time.Now().Add(mc.writeTimeout)); err != nil { + return err + } + } + + n, err := mc.netConn.Write(data[:4+size]) + if err == nil && n == 4+size { + mc.sequence++ + if size != maxPacketSize { + return nil + } + pktLen -= size + data = data[size:] + continue + } + + // Handle error + if err == nil { // n != len(data) + mc.cleanup() + errLog.Print(ErrMalformPkt) + } else { + if cerr := mc.canceled.Value(); cerr != nil { + return cerr + } + if n == 0 && pktLen == len(data)-4 { + // only for the first loop iteration when nothing was written yet + return errBadConnNoWrite + } + mc.cleanup() + errLog.Print(err) + } + return ErrInvalidConn + } +} + +/****************************************************************************** +* Initialization Process * +******************************************************************************/ + +// Handshake Initialization Packet +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake +func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) { + data, err = mc.readPacket() + if err != nil { + // for init we can rewrite this to ErrBadConn for sql.Driver to retry, since + // in connection initialization we don't risk retrying non-idempotent actions. + if err == ErrInvalidConn { + return nil, "", driver.ErrBadConn + } + return + } + + if data[0] == iERR { + return nil, "", mc.handleErrorPacket(data) + } + + // protocol version [1 byte] + if data[0] < minProtocolVersion { + return nil, "", fmt.Errorf( + "unsupported protocol version %d. Version %d or higher is required", + data[0], + minProtocolVersion, + ) + } + + // server version [null terminated string] + // connection id [4 bytes] + pos := 1 + bytes.IndexByte(data[1:], 0x00) + 1 + 4 + + // first part of the password cipher [8 bytes] + authData := data[pos : pos+8] + + // (filler) always 0x00 [1 byte] + pos += 8 + 1 + + // capability flags (lower 2 bytes) [2 bytes] + mc.flags = clientFlag(binary.LittleEndian.Uint16(data[pos : pos+2])) + if mc.flags&clientProtocol41 == 0 { + return nil, "", ErrOldProtocol + } + if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { + return nil, "", ErrNoTLS + } + pos += 2 + + if len(data) > pos { + // character set [1 byte] + // status flags [2 bytes] + // capability flags (upper 2 bytes) [2 bytes] + // length of auth-plugin-data [1 byte] + // reserved (all [00]) [10 bytes] + pos += 1 + 2 + 2 + 1 + 10 + + // second part of the password cipher [mininum 13 bytes], + // where len=MAX(13, length of auth-plugin-data - 8) + // + // The web documentation is ambiguous about the length. However, + // according to mysql-5.7/sql/auth/sql_authentication.cc line 538, + // the 13th byte is "\0 byte, terminating the second part of + // a scramble". So the second part of the password cipher is + // a NULL terminated string that's at least 13 bytes with the + // last byte being NULL. + // + // The official Python library uses the fixed length 12 + // which seems to work but technically could have a hidden bug. + authData = append(authData, data[pos:pos+12]...) + pos += 13 + + // EOF if version (>= 5.5.7 and < 5.5.10) or (>= 5.6.0 and < 5.6.2) + // \NUL otherwise + if end := bytes.IndexByte(data[pos:], 0x00); end != -1 { + plugin = string(data[pos : pos+end]) + } else { + plugin = string(data[pos:]) + } + + // make a memory safe copy of the cipher slice + var b [20]byte + copy(b[:], authData) + return b[:], plugin, nil + } + + // make a memory safe copy of the cipher slice + var b [8]byte + copy(b[:], authData) + return b[:], plugin, nil +} + +// Client Authentication Packet +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse +func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, plugin string) error { + // Adjust client flags based on server support + clientFlags := clientProtocol41 | + clientSecureConn | + clientLongPassword | + clientTransactions | + clientLocalFiles | + clientPluginAuth | + clientMultiResults | + mc.flags&clientLongFlag + + if mc.cfg.ClientFoundRows { + clientFlags |= clientFoundRows + } + + // To enable TLS / SSL + if mc.cfg.tls != nil { + clientFlags |= clientSSL + } + + if mc.cfg.MultiStatements { + clientFlags |= clientMultiStatements + } + + // encode length of the auth plugin data + var authRespLEIBuf [9]byte + authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(authResp))) + if len(authRespLEI) > 1 { + // if the length can not be written in 1 byte, it must be written as a + // length encoded integer + clientFlags |= clientPluginAuthLenEncClientData + } + + pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1 + if addNUL { + pktLen++ + } + + // To specify a db name + if n := len(mc.cfg.DBName); n > 0 { + clientFlags |= clientConnectWithDB + pktLen += n + 1 + } + + // Calculate packet length and get buffer with that size + data := mc.buf.takeSmallBuffer(pktLen + 4) + if data == nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return errBadConnNoWrite + } + + // ClientFlags [32 bit] + data[4] = byte(clientFlags) + data[5] = byte(clientFlags >> 8) + data[6] = byte(clientFlags >> 16) + data[7] = byte(clientFlags >> 24) + + // MaxPacketSize [32 bit] (none) + data[8] = 0x00 + data[9] = 0x00 + data[10] = 0x00 + data[11] = 0x00 + + // Charset [1 byte] + var found bool + data[12], found = collations[mc.cfg.Collation] + if !found { + // Note possibility for false negatives: + // could be triggered although the collation is valid if the + // collations map does not contain entries the server supports. + return errors.New("unknown collation") + } + + // SSL Connection Request Packet + // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest + if mc.cfg.tls != nil { + // Send TLS / SSL request packet + if err := mc.writePacket(data[:(4+4+1+23)+4]); err != nil { + return err + } + + // Switch to TLS + tlsConn := tls.Client(mc.netConn, mc.cfg.tls) + if err := tlsConn.Handshake(); err != nil { + return err + } + mc.netConn = tlsConn + mc.buf.nc = tlsConn + } + + // Filler [23 bytes] (all 0x00) + pos := 13 + for ; pos < 13+23; pos++ { + data[pos] = 0 + } + + // User [null terminated string] + if len(mc.cfg.User) > 0 { + pos += copy(data[pos:], mc.cfg.User) + } + data[pos] = 0x00 + pos++ + + // Auth Data [length encoded integer] + pos += copy(data[pos:], authRespLEI) + pos += copy(data[pos:], authResp) + if addNUL { + data[pos] = 0x00 + pos++ + } + + // Databasename [null terminated string] + if len(mc.cfg.DBName) > 0 { + pos += copy(data[pos:], mc.cfg.DBName) + data[pos] = 0x00 + pos++ + } + + pos += copy(data[pos:], plugin) + data[pos] = 0x00 + + // Send Auth packet + return mc.writePacket(data) +} + +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte, addNUL bool) error { + pktLen := 4 + len(authData) + if addNUL { + pktLen++ + } + data := mc.buf.takeSmallBuffer(pktLen) + if data == nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return errBadConnNoWrite + } + + // Add the auth data [EOF] + copy(data[4:], authData) + if addNUL { + data[pktLen-1] = 0x00 + } + + return mc.writePacket(data) +} + +/****************************************************************************** +* Command Packets * +******************************************************************************/ + +func (mc *mysqlConn) writeCommandPacket(command byte) error { + // Reset Packet Sequence + mc.sequence = 0 + + data := mc.buf.takeSmallBuffer(4 + 1) + if data == nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return errBadConnNoWrite + } + + // Add command byte + data[4] = command + + // Send CMD packet + return mc.writePacket(data) +} + +func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { + // Reset Packet Sequence + mc.sequence = 0 + + pktLen := 1 + len(arg) + data := mc.buf.takeBuffer(pktLen + 4) + if data == nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return errBadConnNoWrite + } + + // Add command byte + data[4] = command + + // Add arg + copy(data[5:], arg) + + // Send CMD packet + return mc.writePacket(data) +} + +func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { + // Reset Packet Sequence + mc.sequence = 0 + + data := mc.buf.takeSmallBuffer(4 + 1 + 4) + if data == nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return errBadConnNoWrite + } + + // Add command byte + data[4] = command + + // Add arg [32 bit] + data[5] = byte(arg) + data[6] = byte(arg >> 8) + data[7] = byte(arg >> 16) + data[8] = byte(arg >> 24) + + // Send CMD packet + return mc.writePacket(data) +} + +/****************************************************************************** +* Result Packets * +******************************************************************************/ + +func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { + data, err := mc.readPacket() + if err != nil { + return nil, "", err + } + + // packet indicator + switch data[0] { + + case iOK: + return nil, "", mc.handleOkPacket(data) + + case iAuthMoreData: + return data[1:], "", err + + case iEOF: + if len(data) < 1 { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest + return nil, "mysql_old_password", nil + } + pluginEndIndex := bytes.IndexByte(data, 0x00) + if pluginEndIndex < 0 { + return nil, "", ErrMalformPkt + } + plugin := string(data[1:pluginEndIndex]) + authData := data[pluginEndIndex+1:] + return authData, plugin, nil + + default: // Error otherwise + return nil, "", mc.handleErrorPacket(data) + } +} + +// Returns error if Packet is not an 'Result OK'-Packet +func (mc *mysqlConn) readResultOK() error { + data, err := mc.readPacket() + if err != nil { + return err + } + + if data[0] == iOK { + return mc.handleOkPacket(data) + } + return mc.handleErrorPacket(data) +} + +// Result Set Header Packet +// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::Resultset +func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { + data, err := mc.readPacket() + if err == nil { + switch data[0] { + + case iOK: + return 0, mc.handleOkPacket(data) + + case iERR: + return 0, mc.handleErrorPacket(data) + + case iLocalInFile: + return 0, mc.handleInFileRequest(string(data[1:])) + } + + // column count + num, _, n := readLengthEncodedInteger(data) + if n-len(data) == 0 { + return int(num), nil + } + + return 0, ErrMalformPkt + } + return 0, err +} + +// Error Packet +// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-ERR_Packet +func (mc *mysqlConn) handleErrorPacket(data []byte) error { + if data[0] != iERR { + return ErrMalformPkt + } + + // 0xff [1 byte] + + // Error Number [16 bit uint] + errno := binary.LittleEndian.Uint16(data[1:3]) + + // 1792: ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION + // 1290: ER_OPTION_PREVENTS_STATEMENT (returned by Aurora during failover) + if (errno == 1792 || errno == 1290) && mc.cfg.RejectReadOnly { + // Oops; we are connected to a read-only connection, and won't be able + // to issue any write statements. Since RejectReadOnly is configured, + // we throw away this connection hoping this one would have write + // permission. This is specifically for a possible race condition + // during failover (e.g. on AWS Aurora). See README.md for more. + // + // We explicitly close the connection before returning + // driver.ErrBadConn to ensure that `database/sql` purges this + // connection and initiates a new one for next statement next time. + mc.Close() + return driver.ErrBadConn + } + + pos := 3 + + // SQL State [optional: # + 5bytes string] + if data[3] == 0x23 { + //sqlstate := string(data[4 : 4+5]) + pos = 9 + } + + // Error Message [string] + return &MySQLError{ + Number: errno, + Message: string(data[pos:]), + } +} + +func readStatus(b []byte) statusFlag { + return statusFlag(b[0]) | statusFlag(b[1])<<8 +} + +// Ok Packet +// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet +func (mc *mysqlConn) handleOkPacket(data []byte) error { + var n, m int + + // 0x00 [1 byte] + + // Affected rows [Length Coded Binary] + mc.affectedRows, _, n = readLengthEncodedInteger(data[1:]) + + // Insert id [Length Coded Binary] + mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) + + // server_status [2 bytes] + mc.status = readStatus(data[1+n+m : 1+n+m+2]) + if mc.status&statusMoreResultsExists != 0 { + return nil + } + + // warning count [2 bytes] + + return nil +} + +// Read Packets as Field Packets until EOF-Packet or an Error appears +// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41 +func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { + columns := make([]mysqlField, count) + + for i := 0; ; i++ { + data, err := mc.readPacket() + if err != nil { + return nil, err + } + + // EOF Packet + if data[0] == iEOF && (len(data) == 5 || len(data) == 1) { + if i == count { + return columns, nil + } + return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns)) + } + + // Catalog + pos, err := skipLengthEncodedString(data) + if err != nil { + return nil, err + } + + // Database [len coded string] + n, err := skipLengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + // Table [len coded string] + if mc.cfg.ColumnsWithAlias { + tableName, _, n, err := readLengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + columns[i].tableName = string(tableName) + } else { + n, err = skipLengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + } + + // Original table [len coded string] + n, err = skipLengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + // Name [len coded string] + name, _, n, err := readLengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + columns[i].name = string(name) + pos += n + + // Original name [len coded string] + n, err = skipLengthEncodedString(data[pos:]) + if err != nil { + return nil, err + } + pos += n + + // Filler [uint8] + pos++ + + // Charset [charset, collation uint8] + columns[i].charSet = data[pos] + pos += 2 + + // Length [uint32] + columns[i].length = binary.LittleEndian.Uint32(data[pos : pos+4]) + pos += 4 + + // Field type [uint8] + columns[i].fieldType = fieldType(data[pos]) + pos++ + + // Flags [uint16] + columns[i].flags = fieldFlag(binary.LittleEndian.Uint16(data[pos : pos+2])) + pos += 2 + + // Decimals [uint8] + columns[i].decimals = data[pos] + //pos++ + + // Default value [len coded binary] + //if pos < len(data) { + // defaultVal, _, err = bytesToLengthCodedBinary(data[pos:]) + //} + } +} + +// Read Packets as Field Packets until EOF-Packet or an Error appears +// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::ResultsetRow +func (rows *textRows) readRow(dest []driver.Value) error { + mc := rows.mc + + if rows.rs.done { + return io.EOF + } + + data, err := mc.readPacket() + if err != nil { + return err + } + + // EOF Packet + if data[0] == iEOF && len(data) == 5 { + // server_status [2 bytes] + rows.mc.status = readStatus(data[3:]) + rows.rs.done = true + if !rows.HasNextResultSet() { + rows.mc = nil + } + return io.EOF + } + if data[0] == iERR { + rows.mc = nil + return mc.handleErrorPacket(data) + } + + // RowSet Packet + var n int + var isNull bool + pos := 0 + + for i := range dest { + // Read bytes and convert to string + dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) + pos += n + if err == nil { + if !isNull { + if !mc.parseTime { + continue + } else { + switch rows.rs.columns[i].fieldType { + case fieldTypeTimestamp, fieldTypeDateTime, + fieldTypeDate, fieldTypeNewDate: + dest[i], err = parseDateTime( + string(dest[i].([]byte)), + mc.cfg.Loc, + ) + if err == nil { + continue + } + default: + continue + } + } + + } else { + dest[i] = nil + continue + } + } + return err // err != nil + } + + return nil +} + +// Reads Packets until EOF-Packet or an Error appears. Returns count of Packets read +func (mc *mysqlConn) readUntilEOF() error { + for { + data, err := mc.readPacket() + if err != nil { + return err + } + + switch data[0] { + case iERR: + return mc.handleErrorPacket(data) + case iEOF: + if len(data) == 5 { + mc.status = readStatus(data[3:]) + } + return nil + } + } +} + +/****************************************************************************** +* Prepared Statements * +******************************************************************************/ + +// Prepare Result Packets +// http://dev.mysql.com/doc/internals/en/com-stmt-prepare-response.html +func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) { + data, err := stmt.mc.readPacket() + if err == nil { + // packet indicator [1 byte] + if data[0] != iOK { + return 0, stmt.mc.handleErrorPacket(data) + } + + // statement id [4 bytes] + stmt.id = binary.LittleEndian.Uint32(data[1:5]) + + // Column count [16 bit uint] + columnCount := binary.LittleEndian.Uint16(data[5:7]) + + // Param count [16 bit uint] + stmt.paramCount = int(binary.LittleEndian.Uint16(data[7:9])) + + // Reserved [8 bit] + + // Warning count [16 bit uint] + + return columnCount, nil + } + return 0, err +} + +// http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html +func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { + maxLen := stmt.mc.maxAllowedPacket - 1 + pktLen := maxLen + + // After the header (bytes 0-3) follows before the data: + // 1 byte command + // 4 bytes stmtID + // 2 bytes paramID + const dataOffset = 1 + 4 + 2 + + // Cannot use the write buffer since + // a) the buffer is too small + // b) it is in use + data := make([]byte, 4+1+4+2+len(arg)) + + copy(data[4+dataOffset:], arg) + + for argLen := len(arg); argLen > 0; argLen -= pktLen - dataOffset { + if dataOffset+argLen < maxLen { + pktLen = dataOffset + argLen + } + + stmt.mc.sequence = 0 + // Add command byte [1 byte] + data[4] = comStmtSendLongData + + // Add stmtID [32 bit] + data[5] = byte(stmt.id) + data[6] = byte(stmt.id >> 8) + data[7] = byte(stmt.id >> 16) + data[8] = byte(stmt.id >> 24) + + // Add paramID [16 bit] + data[9] = byte(paramID) + data[10] = byte(paramID >> 8) + + // Send CMD packet + err := stmt.mc.writePacket(data[:4+pktLen]) + if err == nil { + data = data[pktLen-dataOffset:] + continue + } + return err + + } + + // Reset Packet Sequence + stmt.mc.sequence = 0 + return nil +} + +// Execute Prepared Statement +// http://dev.mysql.com/doc/internals/en/com-stmt-execute.html +func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { + if len(args) != stmt.paramCount { + return fmt.Errorf( + "argument count mismatch (got: %d; has: %d)", + len(args), + stmt.paramCount, + ) + } + + const minPktLen = 4 + 1 + 4 + 1 + 4 + mc := stmt.mc + + // Determine threshould dynamically to avoid packet size shortage. + longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1) + if longDataSize < 64 { + longDataSize = 64 + } + + // Reset packet-sequence + mc.sequence = 0 + + var data []byte + + if len(args) == 0 { + data = mc.buf.takeBuffer(minPktLen) + } else { + data = mc.buf.takeCompleteBuffer() + } + if data == nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return errBadConnNoWrite + } + + // command [1 byte] + data[4] = comStmtExecute + + // statement_id [4 bytes] + data[5] = byte(stmt.id) + data[6] = byte(stmt.id >> 8) + data[7] = byte(stmt.id >> 16) + data[8] = byte(stmt.id >> 24) + + // flags (0: CURSOR_TYPE_NO_CURSOR) [1 byte] + data[9] = 0x00 + + // iteration_count (uint32(1)) [4 bytes] + data[10] = 0x01 + data[11] = 0x00 + data[12] = 0x00 + data[13] = 0x00 + + if len(args) > 0 { + pos := minPktLen + + var nullMask []byte + if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { + // buffer has to be extended but we don't know by how much so + // we depend on append after all data with known sizes fit. + // We stop at that because we deal with a lot of columns here + // which makes the required allocation size hard to guess. + tmp := make([]byte, pos+maskLen+typesLen) + copy(tmp[:pos], data[:pos]) + data = tmp + nullMask = data[pos : pos+maskLen] + pos += maskLen + } else { + nullMask = data[pos : pos+maskLen] + for i := 0; i < maskLen; i++ { + nullMask[i] = 0 + } + pos += maskLen + } + + // newParameterBoundFlag 1 [1 byte] + data[pos] = 0x01 + pos++ + + // type of each parameter [len(args)*2 bytes] + paramTypes := data[pos:] + pos += len(args) * 2 + + // value of each parameter [n bytes] + paramValues := data[pos:pos] + valuesCap := cap(paramValues) + + for i, arg := range args { + // build NULL-bitmap + if arg == nil { + nullMask[i/8] |= 1 << (uint(i) & 7) + paramTypes[i+i] = byte(fieldTypeNULL) + paramTypes[i+i+1] = 0x00 + continue + } + + // cache types and values + switch v := arg.(type) { + case int64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x00 + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + + case float64: + paramTypes[i+i] = byte(fieldTypeDouble) + paramTypes[i+i+1] = 0x00 + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + math.Float64bits(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(math.Float64bits(v))..., + ) + } + + case bool: + paramTypes[i+i] = byte(fieldTypeTiny) + paramTypes[i+i+1] = 0x00 + + if v { + paramValues = append(paramValues, 0x01) + } else { + paramValues = append(paramValues, 0x00) + } + + case []byte: + // Common case (non-nil value) first + if v != nil { + paramTypes[i+i] = byte(fieldTypeString) + paramTypes[i+i+1] = 0x00 + + if len(v) < longDataSize { + paramValues = appendLengthEncodedInteger(paramValues, + uint64(len(v)), + ) + paramValues = append(paramValues, v...) + } else { + if err := stmt.writeCommandLongData(i, v); err != nil { + return err + } + } + continue + } + + // Handle []byte(nil) as a NULL value + nullMask[i/8] |= 1 << (uint(i) & 7) + paramTypes[i+i] = byte(fieldTypeNULL) + paramTypes[i+i+1] = 0x00 + + case string: + paramTypes[i+i] = byte(fieldTypeString) + paramTypes[i+i+1] = 0x00 + + if len(v) < longDataSize { + paramValues = appendLengthEncodedInteger(paramValues, + uint64(len(v)), + ) + paramValues = append(paramValues, v...) + } else { + if err := stmt.writeCommandLongData(i, []byte(v)); err != nil { + return err + } + } + + case time.Time: + paramTypes[i+i] = byte(fieldTypeString) + paramTypes[i+i+1] = 0x00 + + var a [64]byte + var b = a[:0] + + if v.IsZero() { + b = append(b, "0000-00-00"...) + } else { + b = v.In(mc.cfg.Loc).AppendFormat(b, timeFormat) + } + + paramValues = appendLengthEncodedInteger(paramValues, + uint64(len(b)), + ) + paramValues = append(paramValues, b...) + + default: + return fmt.Errorf("cannot convert type: %T", arg) + } + } + + // Check if param values exceeded the available buffer + // In that case we must build the data packet with the new values buffer + if valuesCap != cap(paramValues) { + data = append(data[:pos], paramValues...) + mc.buf.buf = data + } + + pos += len(paramValues) + data = data[:pos] + } + + return mc.writePacket(data) +} + +func (mc *mysqlConn) discardResults() error { + for mc.status&statusMoreResultsExists != 0 { + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return err + } + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { + return err + } + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } + } + } + return nil +} + +// http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html +func (rows *binaryRows) readRow(dest []driver.Value) error { + data, err := rows.mc.readPacket() + if err != nil { + return err + } + + // packet indicator [1 byte] + if data[0] != iOK { + // EOF Packet + if data[0] == iEOF && len(data) == 5 { + rows.mc.status = readStatus(data[3:]) + rows.rs.done = true + if !rows.HasNextResultSet() { + rows.mc = nil + } + return io.EOF + } + mc := rows.mc + rows.mc = nil + + // Error otherwise + return mc.handleErrorPacket(data) + } + + // NULL-bitmap, [(column-count + 7 + 2) / 8 bytes] + pos := 1 + (len(dest)+7+2)>>3 + nullMask := data[1:pos] + + for i := range dest { + // Field is NULL + // (byte >> bit-pos) % 2 == 1 + if ((nullMask[(i+2)>>3] >> uint((i+2)&7)) & 1) == 1 { + dest[i] = nil + continue + } + + // Convert to byte-coded string + switch rows.rs.columns[i].fieldType { + case fieldTypeNULL: + dest[i] = nil + continue + + // Numeric Types + case fieldTypeTiny: + if rows.rs.columns[i].flags&flagUnsigned != 0 { + dest[i] = int64(data[pos]) + } else { + dest[i] = int64(int8(data[pos])) + } + pos++ + continue + + case fieldTypeShort, fieldTypeYear: + if rows.rs.columns[i].flags&flagUnsigned != 0 { + dest[i] = int64(binary.LittleEndian.Uint16(data[pos : pos+2])) + } else { + dest[i] = int64(int16(binary.LittleEndian.Uint16(data[pos : pos+2]))) + } + pos += 2 + continue + + case fieldTypeInt24, fieldTypeLong: + if rows.rs.columns[i].flags&flagUnsigned != 0 { + dest[i] = int64(binary.LittleEndian.Uint32(data[pos : pos+4])) + } else { + dest[i] = int64(int32(binary.LittleEndian.Uint32(data[pos : pos+4]))) + } + pos += 4 + continue + + case fieldTypeLongLong: + if rows.rs.columns[i].flags&flagUnsigned != 0 { + val := binary.LittleEndian.Uint64(data[pos : pos+8]) + if val > math.MaxInt64 { + dest[i] = uint64ToString(val) + } else { + dest[i] = int64(val) + } + } else { + dest[i] = int64(binary.LittleEndian.Uint64(data[pos : pos+8])) + } + pos += 8 + continue + + case fieldTypeFloat: + dest[i] = math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4])) + pos += 4 + continue + + case fieldTypeDouble: + dest[i] = math.Float64frombits(binary.LittleEndian.Uint64(data[pos : pos+8])) + pos += 8 + continue + + // Length coded Binary Strings + case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, + fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, + fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON: + var isNull bool + var n int + dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) + pos += n + if err == nil { + if !isNull { + continue + } else { + dest[i] = nil + continue + } + } + return err + + case + fieldTypeDate, fieldTypeNewDate, // Date YYYY-MM-DD + fieldTypeTime, // Time [-][H]HH:MM:SS[.fractal] + fieldTypeTimestamp, fieldTypeDateTime: // Timestamp YYYY-MM-DD HH:MM:SS[.fractal] + + num, isNull, n := readLengthEncodedInteger(data[pos:]) + pos += n + + switch { + case isNull: + dest[i] = nil + continue + case rows.rs.columns[i].fieldType == fieldTypeTime: + // database/sql does not support an equivalent to TIME, return a string + var dstlen uint8 + switch decimals := rows.rs.columns[i].decimals; decimals { + case 0x00, 0x1f: + dstlen = 8 + case 1, 2, 3, 4, 5, 6: + dstlen = 8 + 1 + decimals + default: + return fmt.Errorf( + "protocol error, illegal decimals value %d", + rows.rs.columns[i].decimals, + ) + } + dest[i], err = formatBinaryTime(data[pos:pos+int(num)], dstlen) + case rows.mc.parseTime: + dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.Loc) + default: + var dstlen uint8 + if rows.rs.columns[i].fieldType == fieldTypeDate { + dstlen = 10 + } else { + switch decimals := rows.rs.columns[i].decimals; decimals { + case 0x00, 0x1f: + dstlen = 19 + case 1, 2, 3, 4, 5, 6: + dstlen = 19 + 1 + decimals + default: + return fmt.Errorf( + "protocol error, illegal decimals value %d", + rows.rs.columns[i].decimals, + ) + } + } + dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen) + } + + if err == nil { + pos += int(num) + continue + } else { + return err + } + + // Please report if this happens! + default: + return fmt.Errorf("unknown field type %d", rows.rs.columns[i].fieldType) + } + } + + return nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/result.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/result.go new file mode 100644 index 0000000..c6438d0 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/result.go @@ -0,0 +1,22 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +type mysqlResult struct { + affectedRows int64 + insertId int64 +} + +func (res *mysqlResult) LastInsertId() (int64, error) { + return res.insertId, nil +} + +func (res *mysqlResult) RowsAffected() (int64, error) { + return res.affectedRows, nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go new file mode 100644 index 0000000..d3b1e28 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go @@ -0,0 +1,216 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "io" + "math" + "reflect" +) + +type resultSet struct { + columns []mysqlField + columnNames []string + done bool +} + +type mysqlRows struct { + mc *mysqlConn + rs resultSet + finish func() +} + +type binaryRows struct { + mysqlRows +} + +type textRows struct { + mysqlRows +} + +func (rows *mysqlRows) Columns() []string { + if rows.rs.columnNames != nil { + return rows.rs.columnNames + } + + columns := make([]string, len(rows.rs.columns)) + if rows.mc != nil && rows.mc.cfg.ColumnsWithAlias { + for i := range columns { + if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 { + columns[i] = tableName + "." + rows.rs.columns[i].name + } else { + columns[i] = rows.rs.columns[i].name + } + } + } else { + for i := range columns { + columns[i] = rows.rs.columns[i].name + } + } + + rows.rs.columnNames = columns + return columns +} + +func (rows *mysqlRows) ColumnTypeDatabaseTypeName(i int) string { + return rows.rs.columns[i].typeDatabaseName() +} + +// func (rows *mysqlRows) ColumnTypeLength(i int) (length int64, ok bool) { +// return int64(rows.rs.columns[i].length), true +// } + +func (rows *mysqlRows) ColumnTypeNullable(i int) (nullable, ok bool) { + return rows.rs.columns[i].flags&flagNotNULL == 0, true +} + +func (rows *mysqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) { + column := rows.rs.columns[i] + decimals := int64(column.decimals) + + switch column.fieldType { + case fieldTypeDecimal, fieldTypeNewDecimal: + if decimals > 0 { + return int64(column.length) - 2, decimals, true + } + return int64(column.length) - 1, decimals, true + case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeTime: + return decimals, decimals, true + case fieldTypeFloat, fieldTypeDouble: + if decimals == 0x1f { + return math.MaxInt64, math.MaxInt64, true + } + return math.MaxInt64, decimals, true + } + + return 0, 0, false +} + +func (rows *mysqlRows) ColumnTypeScanType(i int) reflect.Type { + return rows.rs.columns[i].scanType() +} + +func (rows *mysqlRows) Close() (err error) { + if f := rows.finish; f != nil { + f() + rows.finish = nil + } + + mc := rows.mc + if mc == nil { + return nil + } + if err := mc.error(); err != nil { + return err + } + + // Remove unread packets from stream + if !rows.rs.done { + err = mc.readUntilEOF() + } + if err == nil { + if err = mc.discardResults(); err != nil { + return err + } + } + + rows.mc = nil + return err +} + +func (rows *mysqlRows) HasNextResultSet() (b bool) { + if rows.mc == nil { + return false + } + return rows.mc.status&statusMoreResultsExists != 0 +} + +func (rows *mysqlRows) nextResultSet() (int, error) { + if rows.mc == nil { + return 0, io.EOF + } + if err := rows.mc.error(); err != nil { + return 0, err + } + + // Remove unread packets from stream + if !rows.rs.done { + if err := rows.mc.readUntilEOF(); err != nil { + return 0, err + } + rows.rs.done = true + } + + if !rows.HasNextResultSet() { + rows.mc = nil + return 0, io.EOF + } + rows.rs = resultSet{} + return rows.mc.readResultSetHeaderPacket() +} + +func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) { + for { + resLen, err := rows.nextResultSet() + if err != nil { + return 0, err + } + + if resLen > 0 { + return resLen, nil + } + + rows.rs.done = true + } +} + +func (rows *binaryRows) NextResultSet() error { + resLen, err := rows.nextNotEmptyResultSet() + if err != nil { + return err + } + + rows.rs.columns, err = rows.mc.readColumns(resLen) + return err +} + +func (rows *binaryRows) Next(dest []driver.Value) error { + if mc := rows.mc; mc != nil { + if err := mc.error(); err != nil { + return err + } + + // Fetch next row from stream + return rows.readRow(dest) + } + return io.EOF +} + +func (rows *textRows) NextResultSet() (err error) { + resLen, err := rows.nextNotEmptyResultSet() + if err != nil { + return err + } + + rows.rs.columns, err = rows.mc.readColumns(resLen) + return err +} + +func (rows *textRows) Next(dest []driver.Value) error { + if mc := rows.mc; mc != nil { + if err := mc.error(); err != nil { + return err + } + + // Fetch next row from stream + return rows.readRow(dest) + } + return io.EOF +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go new file mode 100644 index 0000000..ce7fe4c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go @@ -0,0 +1,211 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "fmt" + "io" + "reflect" + "strconv" +) + +type mysqlStmt struct { + mc *mysqlConn + id uint32 + paramCount int +} + +func (stmt *mysqlStmt) Close() error { + if stmt.mc == nil || stmt.mc.closed.IsSet() { + // driver.Stmt.Close can be called more than once, thus this function + // has to be idempotent. + // See also Issue #450 and golang/go#16019. + //errLog.Print(ErrInvalidConn) + return driver.ErrBadConn + } + + err := stmt.mc.writeCommandPacketUint32(comStmtClose, stmt.id) + stmt.mc = nil + return err +} + +func (stmt *mysqlStmt) NumInput() int { + return stmt.paramCount +} + +func (stmt *mysqlStmt) ColumnConverter(idx int) driver.ValueConverter { + return converter{} +} + +func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { + if stmt.mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return nil, driver.ErrBadConn + } + // Send command + err := stmt.writeExecutePacket(args) + if err != nil { + return nil, stmt.mc.markBadConn(err) + } + + mc := stmt.mc + + mc.affectedRows = 0 + mc.insertId = 0 + + // Read Result + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return nil, err + } + + if resLen > 0 { + // Columns + if err = mc.readUntilEOF(); err != nil { + return nil, err + } + + // Rows + if err := mc.readUntilEOF(); err != nil { + return nil, err + } + } + + if err := mc.discardResults(); err != nil { + return nil, err + } + + return &mysqlResult{ + affectedRows: int64(mc.affectedRows), + insertId: int64(mc.insertId), + }, nil +} + +func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { + return stmt.query(args) +} + +func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { + if stmt.mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return nil, driver.ErrBadConn + } + // Send command + err := stmt.writeExecutePacket(args) + if err != nil { + return nil, stmt.mc.markBadConn(err) + } + + mc := stmt.mc + + // Read Result + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return nil, err + } + + rows := new(binaryRows) + + if resLen > 0 { + rows.mc = mc + rows.rs.columns, err = mc.readColumns(resLen) + } else { + rows.rs.done = true + + switch err := rows.NextResultSet(); err { + case nil, io.EOF: + return rows, nil + default: + return nil, err + } + } + + return rows, err +} + +type converter struct{} + +// ConvertValue mirrors the reference/default converter in database/sql/driver +// with _one_ exception. We support uint64 with their high bit and the default +// implementation does not. This function should be kept in sync with +// database/sql/driver defaultConverter.ConvertValue() except for that +// deliberate difference. +func (c converter) ConvertValue(v interface{}) (driver.Value, error) { + if driver.IsValue(v) { + return v, nil + } + + if vr, ok := v.(driver.Valuer); ok { + sv, err := callValuerValue(vr) + if err != nil { + return nil, err + } + if !driver.IsValue(sv) { + return nil, fmt.Errorf("non-Value type %T returned from Value", sv) + } + return sv, nil + } + + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Ptr: + // indirect pointers + if rv.IsNil() { + return nil, nil + } else { + return c.ConvertValue(rv.Elem().Interface()) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int(), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: + return int64(rv.Uint()), nil + case reflect.Uint64: + u64 := rv.Uint() + if u64 >= 1<<63 { + return strconv.FormatUint(u64, 10), nil + } + return int64(u64), nil + case reflect.Float32, reflect.Float64: + return rv.Float(), nil + case reflect.Bool: + return rv.Bool(), nil + case reflect.Slice: + ek := rv.Type().Elem().Kind() + if ek == reflect.Uint8 { + return rv.Bytes(), nil + } + return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek) + case reflect.String: + return rv.String(), nil + } + return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) +} + +var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() + +// callValuerValue returns vr.Value(), with one exception: +// If vr.Value is an auto-generated method on a pointer type and the +// pointer is nil, it would panic at runtime in the panicwrap +// method. Treat it like nil instead. +// +// This is so people can implement driver.Value on value types and +// still use nil pointers to those types to mean nil/NULL, just like +// string/*string. +// +// This is an exact copy of the same-named unexported function from the +// database/sql package. +func callValuerValue(vr driver.Valuer) (v driver.Value, err error) { + if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr && + rv.IsNil() && + rv.Type().Elem().Implements(valuerReflectType) { + return nil, nil + } + return vr.Value() +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/transaction.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/transaction.go new file mode 100644 index 0000000..417d727 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/transaction.go @@ -0,0 +1,31 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +type mysqlTx struct { + mc *mysqlConn +} + +func (tx *mysqlTx) Commit() (err error) { + if tx.mc == nil || tx.mc.closed.IsSet() { + return ErrInvalidConn + } + err = tx.mc.exec("COMMIT") + tx.mc = nil + return +} + +func (tx *mysqlTx) Rollback() (err error) { + if tx.mc == nil || tx.mc.closed.IsSet() { + return ErrInvalidConn + } + err = tx.mc.exec("ROLLBACK") + tx.mc = nil + return +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go new file mode 100644 index 0000000..cb3650b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go @@ -0,0 +1,755 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/tls" + "database/sql" + "database/sql/driver" + "encoding/binary" + "errors" + "fmt" + "io" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Registry for custom tls.Configs +var ( + tlsConfigLock sync.RWMutex + tlsConfigRegistry map[string]*tls.Config +) + +// RegisterTLSConfig registers a custom tls.Config to be used with sql.Open. +// Use the key as a value in the DSN where tls=value. +// +// Note: The provided tls.Config is exclusively owned by the driver after +// registering it. +// +// rootCertPool := x509.NewCertPool() +// pem, err := ioutil.ReadFile("/path/ca-cert.pem") +// if err != nil { +// log.Fatal(err) +// } +// if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { +// log.Fatal("Failed to append PEM.") +// } +// clientCert := make([]tls.Certificate, 0, 1) +// certs, err := tls.LoadX509KeyPair("/path/client-cert.pem", "/path/client-key.pem") +// if err != nil { +// log.Fatal(err) +// } +// clientCert = append(clientCert, certs) +// mysql.RegisterTLSConfig("custom", &tls.Config{ +// RootCAs: rootCertPool, +// Certificates: clientCert, +// }) +// db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") +// +func RegisterTLSConfig(key string, config *tls.Config) error { + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { + return fmt.Errorf("key '%s' is reserved", key) + } + + tlsConfigLock.Lock() + if tlsConfigRegistry == nil { + tlsConfigRegistry = make(map[string]*tls.Config) + } + + tlsConfigRegistry[key] = config + tlsConfigLock.Unlock() + return nil +} + +// DeregisterTLSConfig removes the tls.Config associated with key. +func DeregisterTLSConfig(key string) { + tlsConfigLock.Lock() + if tlsConfigRegistry != nil { + delete(tlsConfigRegistry, key) + } + tlsConfigLock.Unlock() +} + +func getTLSConfigClone(key string) (config *tls.Config) { + tlsConfigLock.RLock() + if v, ok := tlsConfigRegistry[key]; ok { + config = v.Clone() + } + tlsConfigLock.RUnlock() + return +} + +// Returns the bool value of the input. +// The 2nd return value indicates if the input was a valid bool value +func readBool(input string) (value bool, valid bool) { + switch input { + case "1", "true", "TRUE", "True": + return true, true + case "0", "false", "FALSE", "False": + return false, true + } + + // Not a valid bool value + return +} + +/****************************************************************************** +* Time related utils * +******************************************************************************/ + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} + +// Scan implements the Scanner interface. +// The value type must be time.Time or string / []byte (formatted time-string), +// otherwise Scan fails. +func (nt *NullTime) Scan(value interface{}) (err error) { + if value == nil { + nt.Time, nt.Valid = time.Time{}, false + return + } + + switch v := value.(type) { + case time.Time: + nt.Time, nt.Valid = v, true + return + case []byte: + nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Valid = (err == nil) + return + case string: + nt.Time, err = parseDateTime(v, time.UTC) + nt.Valid = (err == nil) + return + } + + nt.Valid = false + return fmt.Errorf("Can't convert %T to time.Time", value) +} + +// Value implements the driver Valuer interface. +func (nt NullTime) Value() (driver.Value, error) { + if !nt.Valid { + return nil, nil + } + return nt.Time, nil +} + +func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { + base := "0000-00-00 00:00:00.0000000" + switch len(str) { + case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM" + if str == base[:len(str)] { + return + } + t, err = time.Parse(timeFormat[:len(str)], str) + default: + err = fmt.Errorf("invalid time string: %s", str) + return + } + + // Adjust location + if err == nil && loc != time.UTC { + y, mo, d := t.Date() + h, mi, s := t.Clock() + t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil + } + + return +} + +func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Value, error) { + switch num { + case 0: + return time.Time{}, nil + case 4: + return time.Date( + int(binary.LittleEndian.Uint16(data[:2])), // year + time.Month(data[2]), // month + int(data[3]), // day + 0, 0, 0, 0, + loc, + ), nil + case 7: + return time.Date( + int(binary.LittleEndian.Uint16(data[:2])), // year + time.Month(data[2]), // month + int(data[3]), // day + int(data[4]), // hour + int(data[5]), // minutes + int(data[6]), // seconds + 0, + loc, + ), nil + case 11: + return time.Date( + int(binary.LittleEndian.Uint16(data[:2])), // year + time.Month(data[2]), // month + int(data[3]), // day + int(data[4]), // hour + int(data[5]), // minutes + int(data[6]), // seconds + int(binary.LittleEndian.Uint32(data[7:11]))*1000, // nanoseconds + loc, + ), nil + } + return nil, fmt.Errorf("invalid DATETIME packet length %d", num) +} + +// zeroDateTime is used in formatBinaryDateTime to avoid an allocation +// if the DATE or DATETIME has the zero value. +// It must never be changed. +// The current behavior depends on database/sql copying the result. +var zeroDateTime = []byte("0000-00-00 00:00:00.000000") + +const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" + +func appendMicrosecs(dst, src []byte, decimals int) []byte { + if decimals <= 0 { + return dst + } + if len(src) == 0 { + return append(dst, ".000000"[:decimals+1]...) + } + + microsecs := binary.LittleEndian.Uint32(src[:4]) + p1 := byte(microsecs / 10000) + microsecs -= 10000 * uint32(p1) + p2 := byte(microsecs / 100) + microsecs -= 100 * uint32(p2) + p3 := byte(microsecs) + + switch decimals { + default: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + digits10[p3], digits01[p3], + ) + case 1: + return append(dst, '.', + digits10[p1], + ) + case 2: + return append(dst, '.', + digits10[p1], digits01[p1], + ) + case 3: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], + ) + case 4: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + ) + case 5: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + digits10[p3], + ) + } +} + +func formatBinaryDateTime(src []byte, length uint8) (driver.Value, error) { + // length expects the deterministic length of the zero value, + // negative time and 100+ hours are automatically added if needed + if len(src) == 0 { + return zeroDateTime[:length], nil + } + var dst []byte // return value + var p1, p2, p3 byte // current digit pair + + switch length { + case 10, 19, 21, 22, 23, 24, 25, 26: + default: + t := "DATE" + if length > 10 { + t += "TIME" + } + return nil, fmt.Errorf("illegal %s length %d", t, length) + } + switch len(src) { + case 4, 7, 11: + default: + t := "DATE" + if length > 10 { + t += "TIME" + } + return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) + } + dst = make([]byte, 0, length) + // start with the date + year := binary.LittleEndian.Uint16(src[:2]) + pt := year / 100 + p1 = byte(year - 100*uint16(pt)) + p2, p3 = src[2], src[3] + dst = append(dst, + digits10[pt], digits01[pt], + digits10[p1], digits01[p1], '-', + digits10[p2], digits01[p2], '-', + digits10[p3], digits01[p3], + ) + if length == 10 { + return dst, nil + } + if len(src) == 4 { + return append(dst, zeroDateTime[10:length]...), nil + } + dst = append(dst, ' ') + p1 = src[4] // hour + src = src[5:] + + // p1 is 2-digit hour, src is after hour + p2, p3 = src[0], src[1] + dst = append(dst, + digits10[p1], digits01[p1], ':', + digits10[p2], digits01[p2], ':', + digits10[p3], digits01[p3], + ) + return appendMicrosecs(dst, src[2:], int(length)-20), nil +} + +func formatBinaryTime(src []byte, length uint8) (driver.Value, error) { + // length expects the deterministic length of the zero value, + // negative time and 100+ hours are automatically added if needed + if len(src) == 0 { + return zeroDateTime[11 : 11+length], nil + } + var dst []byte // return value + + switch length { + case + 8, // time (can be up to 10 when negative and 100+ hours) + 10, 11, 12, 13, 14, 15: // time with fractional seconds + default: + return nil, fmt.Errorf("illegal TIME length %d", length) + } + switch len(src) { + case 8, 12: + default: + return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) + } + // +2 to enable negative time and 100+ hours + dst = make([]byte, 0, length+2) + if src[0] == 1 { + dst = append(dst, '-') + } + days := binary.LittleEndian.Uint32(src[1:5]) + hours := int64(days)*24 + int64(src[5]) + + if hours >= 100 { + dst = strconv.AppendInt(dst, hours, 10) + } else { + dst = append(dst, digits10[hours], digits01[hours]) + } + + min, sec := src[6], src[7] + dst = append(dst, ':', + digits10[min], digits01[min], ':', + digits10[sec], digits01[sec], + ) + return appendMicrosecs(dst, src[8:], int(length)-9), nil +} + +/****************************************************************************** +* Convert from and to bytes * +******************************************************************************/ + +func uint64ToBytes(n uint64) []byte { + return []byte{ + byte(n), + byte(n >> 8), + byte(n >> 16), + byte(n >> 24), + byte(n >> 32), + byte(n >> 40), + byte(n >> 48), + byte(n >> 56), + } +} + +func uint64ToString(n uint64) []byte { + var a [20]byte + i := 20 + + // U+0030 = 0 + // ... + // U+0039 = 9 + + var q uint64 + for n >= 10 { + i-- + q = n / 10 + a[i] = uint8(n-q*10) + 0x30 + n = q + } + + i-- + a[i] = uint8(n) + 0x30 + + return a[i:] +} + +// treats string value as unsigned integer representation +func stringToInt(b []byte) int { + val := 0 + for i := range b { + val *= 10 + val += int(b[i] - 0x30) + } + return val +} + +// returns the string read as a bytes slice, wheter the value is NULL, +// the number of bytes read and an error, in case the string is longer than +// the input slice +func readLengthEncodedString(b []byte) ([]byte, bool, int, error) { + // Get length + num, isNull, n := readLengthEncodedInteger(b) + if num < 1 { + return b[n:n], isNull, n, nil + } + + n += int(num) + + // Check data length + if len(b) >= n { + return b[n-int(num) : n : n], false, n, nil + } + return nil, false, n, io.EOF +} + +// returns the number of bytes skipped and an error, in case the string is +// longer than the input slice +func skipLengthEncodedString(b []byte) (int, error) { + // Get length + num, _, n := readLengthEncodedInteger(b) + if num < 1 { + return n, nil + } + + n += int(num) + + // Check data length + if len(b) >= n { + return n, nil + } + return n, io.EOF +} + +// returns the number read, whether the value is NULL and the number of bytes read +func readLengthEncodedInteger(b []byte) (uint64, bool, int) { + // See issue #349 + if len(b) == 0 { + return 0, true, 1 + } + + switch b[0] { + // 251: NULL + case 0xfb: + return 0, true, 1 + + // 252: value of following 2 + case 0xfc: + return uint64(b[1]) | uint64(b[2])<<8, false, 3 + + // 253: value of following 3 + case 0xfd: + return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16, false, 4 + + // 254: value of following 8 + case 0xfe: + return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16 | + uint64(b[4])<<24 | uint64(b[5])<<32 | uint64(b[6])<<40 | + uint64(b[7])<<48 | uint64(b[8])<<56, + false, 9 + } + + // 0-250: value of first byte + return uint64(b[0]), false, 1 +} + +// encodes a uint64 value and appends it to the given bytes slice +func appendLengthEncodedInteger(b []byte, n uint64) []byte { + switch { + case n <= 250: + return append(b, byte(n)) + + case n <= 0xffff: + return append(b, 0xfc, byte(n), byte(n>>8)) + + case n <= 0xffffff: + return append(b, 0xfd, byte(n), byte(n>>8), byte(n>>16)) + } + return append(b, 0xfe, byte(n), byte(n>>8), byte(n>>16), byte(n>>24), + byte(n>>32), byte(n>>40), byte(n>>48), byte(n>>56)) +} + +// reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. +// If cap(buf) is not enough, reallocate new buffer. +func reserveBuffer(buf []byte, appendSize int) []byte { + newSize := len(buf) + appendSize + if cap(buf) < newSize { + // Grow buffer exponentially + newBuf := make([]byte, len(buf)*2+appendSize) + copy(newBuf, buf) + buf = newBuf + } + return buf[:newSize] +} + +// escapeBytesBackslash escapes []byte with backslashes (\) +// This escapes the contents of a string (provided as []byte) by adding backslashes before special +// characters, and turning others into specific escape sequences, such as +// turning newlines into \n and null bytes into \0. +// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L823-L932 +func escapeBytesBackslash(buf, v []byte) []byte { + pos := len(buf) + buf = reserveBuffer(buf, len(v)*2) + + for _, c := range v { + switch c { + case '\x00': + buf[pos] = '\\' + buf[pos+1] = '0' + pos += 2 + case '\n': + buf[pos] = '\\' + buf[pos+1] = 'n' + pos += 2 + case '\r': + buf[pos] = '\\' + buf[pos+1] = 'r' + pos += 2 + case '\x1a': + buf[pos] = '\\' + buf[pos+1] = 'Z' + pos += 2 + case '\'': + buf[pos] = '\\' + buf[pos+1] = '\'' + pos += 2 + case '"': + buf[pos] = '\\' + buf[pos+1] = '"' + pos += 2 + case '\\': + buf[pos] = '\\' + buf[pos+1] = '\\' + pos += 2 + default: + buf[pos] = c + pos++ + } + } + + return buf[:pos] +} + +// escapeStringBackslash is similar to escapeBytesBackslash but for string. +func escapeStringBackslash(buf []byte, v string) []byte { + pos := len(buf) + buf = reserveBuffer(buf, len(v)*2) + + for i := 0; i < len(v); i++ { + c := v[i] + switch c { + case '\x00': + buf[pos] = '\\' + buf[pos+1] = '0' + pos += 2 + case '\n': + buf[pos] = '\\' + buf[pos+1] = 'n' + pos += 2 + case '\r': + buf[pos] = '\\' + buf[pos+1] = 'r' + pos += 2 + case '\x1a': + buf[pos] = '\\' + buf[pos+1] = 'Z' + pos += 2 + case '\'': + buf[pos] = '\\' + buf[pos+1] = '\'' + pos += 2 + case '"': + buf[pos] = '\\' + buf[pos+1] = '"' + pos += 2 + case '\\': + buf[pos] = '\\' + buf[pos+1] = '\\' + pos += 2 + default: + buf[pos] = c + pos++ + } + } + + return buf[:pos] +} + +// escapeBytesQuotes escapes apostrophes in []byte by doubling them up. +// This escapes the contents of a string by doubling up any apostrophes that +// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in +// effect on the server. +// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L963-L1038 +func escapeBytesQuotes(buf, v []byte) []byte { + pos := len(buf) + buf = reserveBuffer(buf, len(v)*2) + + for _, c := range v { + if c == '\'' { + buf[pos] = '\'' + buf[pos+1] = '\'' + pos += 2 + } else { + buf[pos] = c + pos++ + } + } + + return buf[:pos] +} + +// escapeStringQuotes is similar to escapeBytesQuotes but for string. +func escapeStringQuotes(buf []byte, v string) []byte { + pos := len(buf) + buf = reserveBuffer(buf, len(v)*2) + + for i := 0; i < len(v); i++ { + c := v[i] + if c == '\'' { + buf[pos] = '\'' + buf[pos+1] = '\'' + pos += 2 + } else { + buf[pos] = c + pos++ + } + } + + return buf[:pos] +} + +/****************************************************************************** +* Sync utils * +******************************************************************************/ + +// noCopy may be embedded into structs which must not be copied +// after the first use. +// +// See https://github.com/golang/go/issues/8005#issuecomment-190753527 +// for details. +type noCopy struct{} + +// Lock is a no-op used by -copylocks checker from `go vet`. +func (*noCopy) Lock() {} + +// atomicBool is a wrapper around uint32 for usage as a boolean value with +// atomic access. +type atomicBool struct { + _noCopy noCopy + value uint32 +} + +// IsSet returns wether the current boolean value is true +func (ab *atomicBool) IsSet() bool { + return atomic.LoadUint32(&ab.value) > 0 +} + +// Set sets the value of the bool regardless of the previous value +func (ab *atomicBool) Set(value bool) { + if value { + atomic.StoreUint32(&ab.value, 1) + } else { + atomic.StoreUint32(&ab.value, 0) + } +} + +// TrySet sets the value of the bool and returns wether the value changed +func (ab *atomicBool) TrySet(value bool) bool { + if value { + return atomic.SwapUint32(&ab.value, 1) == 0 + } + return atomic.SwapUint32(&ab.value, 0) > 0 +} + +// atomicError is a wrapper for atomically accessed error values +type atomicError struct { + _noCopy noCopy + value atomic.Value +} + +// Set sets the error value regardless of the previous value. +// The value must not be nil +func (ae *atomicError) Set(value error) { + ae.value.Store(value) +} + +// Value returns the current error value +func (ae *atomicError) Value() error { + if v := ae.value.Load(); v != nil { + // this will panic if the value doesn't implement the error interface + return v.(error) + } + return nil +} + +func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { + dargs := make([]driver.Value, len(named)) + for n, param := range named { + if len(param.Name) > 0 { + // TODO: support the use of Named Parameters #561 + return nil, errors.New("mysql: driver does not support the use of Named Parameters") + } + dargs[n] = param.Value + } + return dargs, nil +} + +func mapIsolationLevel(level driver.IsolationLevel) (string, error) { + switch sql.IsolationLevel(level) { + case sql.LevelRepeatableRead: + return "REPEATABLE READ", nil + case sql.LevelReadCommitted: + return "READ COMMITTED", nil + case sql.LevelReadUncommitted: + return "READ UNCOMMITTED", nil + case sql.LevelSerializable: + return "SERIALIZABLE", nil + default: + return "", fmt.Errorf("mysql: unsupported isolation level: %v", level) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/LICENSE new file mode 100644 index 0000000..0d31edf --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/LICENSE @@ -0,0 +1,23 @@ + Copyright (c) 2013, Jason Moiron + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/bind.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/bind.go new file mode 100644 index 0000000..0fdc443 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/bind.go @@ -0,0 +1,208 @@ +package sqlx + +import ( + "bytes" + "errors" + "reflect" + "strconv" + "strings" + + "github.com/jmoiron/sqlx/reflectx" +) + +// Bindvar types supported by Rebind, BindMap and BindStruct. +const ( + UNKNOWN = iota + QUESTION + DOLLAR + NAMED +) + +// BindType returns the bindtype for a given database given a drivername. +func BindType(driverName string) int { + switch driverName { + case "postgres", "pgx", "pq-timeouts", "cloudsqlpostgres": + return DOLLAR + case "mysql": + return QUESTION + case "sqlite3": + return QUESTION + case "oci8", "ora", "goracle": + return NAMED + } + return UNKNOWN +} + +// FIXME: this should be able to be tolerant of escaped ?'s in queries without +// losing much speed, and should be to avoid confusion. + +// Rebind a query from the default bindtype (QUESTION) to the target bindtype. +func Rebind(bindType int, query string) string { + switch bindType { + case QUESTION, UNKNOWN: + return query + } + + // Add space enough for 10 params before we have to allocate + rqb := make([]byte, 0, len(query)+10) + + var i, j int + + for i = strings.Index(query, "?"); i != -1; i = strings.Index(query, "?") { + rqb = append(rqb, query[:i]...) + + switch bindType { + case DOLLAR: + rqb = append(rqb, '$') + case NAMED: + rqb = append(rqb, ':', 'a', 'r', 'g') + } + + j++ + rqb = strconv.AppendInt(rqb, int64(j), 10) + + query = query[i+1:] + } + + return string(append(rqb, query...)) +} + +// Experimental implementation of Rebind which uses a bytes.Buffer. The code is +// much simpler and should be more resistant to odd unicode, but it is twice as +// slow. Kept here for benchmarking purposes and to possibly replace Rebind if +// problems arise with its somewhat naive handling of unicode. +func rebindBuff(bindType int, query string) string { + if bindType != DOLLAR { + return query + } + + b := make([]byte, 0, len(query)) + rqb := bytes.NewBuffer(b) + j := 1 + for _, r := range query { + if r == '?' { + rqb.WriteRune('$') + rqb.WriteString(strconv.Itoa(j)) + j++ + } else { + rqb.WriteRune(r) + } + } + + return rqb.String() +} + +// In expands slice values in args, returning the modified query string +// and a new arg list that can be executed by a database. The `query` should +// use the `?` bindVar. The return value uses the `?` bindVar. +func In(query string, args ...interface{}) (string, []interface{}, error) { + // argMeta stores reflect.Value and length for slices and + // the value itself for non-slice arguments + type argMeta struct { + v reflect.Value + i interface{} + length int + } + + var flatArgsCount int + var anySlices bool + + meta := make([]argMeta, len(args)) + + for i, arg := range args { + v := reflect.ValueOf(arg) + t := reflectx.Deref(v.Type()) + + // []byte is a driver.Value type so it should not be expanded + if t.Kind() == reflect.Slice && t != reflect.TypeOf([]byte{}) { + meta[i].length = v.Len() + meta[i].v = v + + anySlices = true + flatArgsCount += meta[i].length + + if meta[i].length == 0 { + return "", nil, errors.New("empty slice passed to 'in' query") + } + } else { + meta[i].i = arg + flatArgsCount++ + } + } + + // don't do any parsing if there aren't any slices; note that this means + // some errors that we might have caught below will not be returned. + if !anySlices { + return query, args, nil + } + + newArgs := make([]interface{}, 0, flatArgsCount) + buf := bytes.NewBuffer(make([]byte, 0, len(query)+len(", ?")*flatArgsCount)) + + var arg, offset int + + for i := strings.IndexByte(query[offset:], '?'); i != -1; i = strings.IndexByte(query[offset:], '?') { + if arg >= len(meta) { + // if an argument wasn't passed, lets return an error; this is + // not actually how database/sql Exec/Query works, but since we are + // creating an argument list programmatically, we want to be able + // to catch these programmer errors earlier. + return "", nil, errors.New("number of bindVars exceeds arguments") + } + + argMeta := meta[arg] + arg++ + + // not a slice, continue. + // our questionmark will either be written before the next expansion + // of a slice or after the loop when writing the rest of the query + if argMeta.length == 0 { + offset = offset + i + 1 + newArgs = append(newArgs, argMeta.i) + continue + } + + // write everything up to and including our ? character + buf.WriteString(query[:offset+i+1]) + + for si := 1; si < argMeta.length; si++ { + buf.WriteString(", ?") + } + + newArgs = appendReflectSlice(newArgs, argMeta.v, argMeta.length) + + // slice the query and reset the offset. this avoids some bookkeeping for + // the write after the loop + query = query[offset+i+1:] + offset = 0 + } + + buf.WriteString(query) + + if arg < len(meta) { + return "", nil, errors.New("number of bindVars less than number arguments") + } + + return buf.String(), newArgs, nil +} + +func appendReflectSlice(args []interface{}, v reflect.Value, vlen int) []interface{} { + switch val := v.Interface().(type) { + case []interface{}: + args = append(args, val...) + case []int: + for i := range val { + args = append(args, val[i]) + } + case []string: + for i := range val { + args = append(args, val[i]) + } + default: + for si := 0; si < vlen; si++ { + args = append(args, v.Index(si).Interface()) + } + } + + return args +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/doc.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/doc.go new file mode 100644 index 0000000..e2b4e60 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/doc.go @@ -0,0 +1,12 @@ +// Package sqlx provides general purpose extensions to database/sql. +// +// It is intended to seamlessly wrap database/sql and provide convenience +// methods which are useful in the development of database driven applications. +// None of the underlying database/sql methods are changed. Instead all extended +// behavior is implemented through new methods defined on wrapper types. +// +// Additions include scanning into structs, named query support, rebinding +// queries for different drivers, convenient shorthands for common error handling +// and more. +// +package sqlx diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named.go new file mode 100644 index 0000000..69eb954 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named.go @@ -0,0 +1,346 @@ +package sqlx + +// Named Query Support +// +// * BindMap - bind query bindvars to map/struct args +// * NamedExec, NamedQuery - named query w/ struct or map +// * NamedStmt - a pre-compiled named query which is a prepared statement +// +// Internal Interfaces: +// +// * compileNamedQuery - rebind a named query, returning a query and list of names +// * bindArgs, bindMapArgs, bindAnyArgs - given a list of names, return an arglist +// +import ( + "database/sql" + "errors" + "fmt" + "reflect" + "strconv" + "unicode" + + "github.com/jmoiron/sqlx/reflectx" +) + +// NamedStmt is a prepared statement that executes named queries. Prepare it +// how you would execute a NamedQuery, but pass in a struct or map when executing. +type NamedStmt struct { + Params []string + QueryString string + Stmt *Stmt +} + +// Close closes the named statement. +func (n *NamedStmt) Close() error { + return n.Stmt.Close() +} + +// Exec executes a named statement using the struct passed. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) Exec(arg interface{}) (sql.Result, error) { + args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) + if err != nil { + return *new(sql.Result), err + } + return n.Stmt.Exec(args...) +} + +// Query executes a named statement using the struct argument, returning rows. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) Query(arg interface{}) (*sql.Rows, error) { + args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) + if err != nil { + return nil, err + } + return n.Stmt.Query(args...) +} + +// QueryRow executes a named statement against the database. Because sqlx cannot +// create a *sql.Row with an error condition pre-set for binding errors, sqlx +// returns a *sqlx.Row instead. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) QueryRow(arg interface{}) *Row { + args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) + if err != nil { + return &Row{err: err} + } + return n.Stmt.QueryRowx(args...) +} + +// MustExec execs a NamedStmt, panicing on error +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) MustExec(arg interface{}) sql.Result { + res, err := n.Exec(arg) + if err != nil { + panic(err) + } + return res +} + +// Queryx using this NamedStmt +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) Queryx(arg interface{}) (*Rows, error) { + r, err := n.Query(arg) + if err != nil { + return nil, err + } + return &Rows{Rows: r, Mapper: n.Stmt.Mapper, unsafe: isUnsafe(n)}, err +} + +// QueryRowx this NamedStmt. Because of limitations with QueryRow, this is +// an alias for QueryRow. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) QueryRowx(arg interface{}) *Row { + return n.QueryRow(arg) +} + +// Select using this NamedStmt +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) Select(dest interface{}, arg interface{}) error { + rows, err := n.Queryx(arg) + if err != nil { + return err + } + // if something happens here, we want to make sure the rows are Closed + defer rows.Close() + return scanAll(rows, dest, false) +} + +// Get using this NamedStmt +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) Get(dest interface{}, arg interface{}) error { + r := n.QueryRowx(arg) + return r.scanAny(dest, false) +} + +// Unsafe creates an unsafe version of the NamedStmt +func (n *NamedStmt) Unsafe() *NamedStmt { + r := &NamedStmt{Params: n.Params, Stmt: n.Stmt, QueryString: n.QueryString} + r.Stmt.unsafe = true + return r +} + +// A union interface of preparer and binder, required to be able to prepare +// named statements (as the bindtype must be determined). +type namedPreparer interface { + Preparer + binder +} + +func prepareNamed(p namedPreparer, query string) (*NamedStmt, error) { + bindType := BindType(p.DriverName()) + q, args, err := compileNamedQuery([]byte(query), bindType) + if err != nil { + return nil, err + } + stmt, err := Preparex(p, q) + if err != nil { + return nil, err + } + return &NamedStmt{ + QueryString: q, + Params: args, + Stmt: stmt, + }, nil +} + +func bindAnyArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) { + if maparg, ok := arg.(map[string]interface{}); ok { + return bindMapArgs(names, maparg) + } + return bindArgs(names, arg, m) +} + +// private interface to generate a list of interfaces from a given struct +// type, given a list of names to pull out of the struct. Used by public +// BindStruct interface. +func bindArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) { + arglist := make([]interface{}, 0, len(names)) + + // grab the indirected value of arg + v := reflect.ValueOf(arg) + for v = reflect.ValueOf(arg); v.Kind() == reflect.Ptr; { + v = v.Elem() + } + + err := m.TraversalsByNameFunc(v.Type(), names, func(i int, t []int) error { + if len(t) == 0 { + return fmt.Errorf("could not find name %s in %#v", names[i], arg) + } + + val := reflectx.FieldByIndexesReadOnly(v, t) + arglist = append(arglist, val.Interface()) + + return nil + }) + + return arglist, err +} + +// like bindArgs, but for maps. +func bindMapArgs(names []string, arg map[string]interface{}) ([]interface{}, error) { + arglist := make([]interface{}, 0, len(names)) + + for _, name := range names { + val, ok := arg[name] + if !ok { + return arglist, fmt.Errorf("could not find name %s in %#v", name, arg) + } + arglist = append(arglist, val) + } + return arglist, nil +} + +// bindStruct binds a named parameter query with fields from a struct argument. +// The rules for binding field names to parameter names follow the same +// conventions as for StructScan, including obeying the `db` struct tags. +func bindStruct(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) { + bound, names, err := compileNamedQuery([]byte(query), bindType) + if err != nil { + return "", []interface{}{}, err + } + + arglist, err := bindArgs(names, arg, m) + if err != nil { + return "", []interface{}{}, err + } + + return bound, arglist, nil +} + +// bindMap binds a named parameter query with a map of arguments. +func bindMap(bindType int, query string, args map[string]interface{}) (string, []interface{}, error) { + bound, names, err := compileNamedQuery([]byte(query), bindType) + if err != nil { + return "", []interface{}{}, err + } + + arglist, err := bindMapArgs(names, args) + return bound, arglist, err +} + +// -- Compilation of Named Queries + +// Allow digits and letters in bind params; additionally runes are +// checked against underscores, meaning that bind params can have be +// alphanumeric with underscores. Mind the difference between unicode +// digits and numbers, where '5' is a digit but '五' is not. +var allowedBindRunes = []*unicode.RangeTable{unicode.Letter, unicode.Digit} + +// FIXME: this function isn't safe for unicode named params, as a failing test +// can testify. This is not a regression but a failure of the original code +// as well. It should be modified to range over runes in a string rather than +// bytes, even though this is less convenient and slower. Hopefully the +// addition of the prepared NamedStmt (which will only do this once) will make +// up for the slightly slower ad-hoc NamedExec/NamedQuery. + +// compile a NamedQuery into an unbound query (using the '?' bindvar) and +// a list of names. +func compileNamedQuery(qs []byte, bindType int) (query string, names []string, err error) { + names = make([]string, 0, 10) + rebound := make([]byte, 0, len(qs)) + + inName := false + last := len(qs) - 1 + currentVar := 1 + name := make([]byte, 0, 10) + + for i, b := range qs { + // a ':' while we're in a name is an error + if b == ':' { + // if this is the second ':' in a '::' escape sequence, append a ':' + if inName && i > 0 && qs[i-1] == ':' { + rebound = append(rebound, ':') + inName = false + continue + } else if inName { + err = errors.New("unexpected `:` while reading named param at " + strconv.Itoa(i)) + return query, names, err + } + inName = true + name = []byte{} + // if we're in a name, and this is an allowed character, continue + } else if inName && (unicode.IsOneOf(allowedBindRunes, rune(b)) || b == '_' || b == '.') && i != last { + // append the byte to the name if we are in a name and not on the last byte + name = append(name, b) + // if we're in a name and it's not an allowed character, the name is done + } else if inName { + inName = false + // if this is the final byte of the string and it is part of the name, then + // make sure to add it to the name + if i == last && unicode.IsOneOf(allowedBindRunes, rune(b)) { + name = append(name, b) + } + // add the string representation to the names list + names = append(names, string(name)) + // add a proper bindvar for the bindType + switch bindType { + // oracle only supports named type bind vars even for positional + case NAMED: + rebound = append(rebound, ':') + rebound = append(rebound, name...) + case QUESTION, UNKNOWN: + rebound = append(rebound, '?') + case DOLLAR: + rebound = append(rebound, '$') + for _, b := range strconv.Itoa(currentVar) { + rebound = append(rebound, byte(b)) + } + currentVar++ + } + // add this byte to string unless it was not part of the name + if i != last { + rebound = append(rebound, b) + } else if !unicode.IsOneOf(allowedBindRunes, rune(b)) { + rebound = append(rebound, b) + } + } else { + // this is a normal byte and should just go onto the rebound query + rebound = append(rebound, b) + } + } + + return string(rebound), names, err +} + +// BindNamed binds a struct or a map to a query with named parameters. +// DEPRECATED: use sqlx.Named` instead of this, it may be removed in future. +func BindNamed(bindType int, query string, arg interface{}) (string, []interface{}, error) { + return bindNamedMapper(bindType, query, arg, mapper()) +} + +// Named takes a query using named parameters and an argument and +// returns a new query with a list of args that can be executed by +// a database. The return value uses the `?` bindvar. +func Named(query string, arg interface{}) (string, []interface{}, error) { + return bindNamedMapper(QUESTION, query, arg, mapper()) +} + +func bindNamedMapper(bindType int, query string, arg interface{}, m *reflectx.Mapper) (string, []interface{}, error) { + if maparg, ok := arg.(map[string]interface{}); ok { + return bindMap(bindType, query, maparg) + } + return bindStruct(bindType, query, arg, m) +} + +// NamedQuery binds a named query and then runs Query on the result using the +// provided Ext (sqlx.Tx, sqlx.Db). It works with both structs and with +// map[string]interface{} types. +func NamedQuery(e Ext, query string, arg interface{}) (*Rows, error) { + q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e)) + if err != nil { + return nil, err + } + return e.Queryx(q, args...) +} + +// NamedExec uses BindStruct to get a query executable by the driver and +// then runs Exec on the result. Returns an error from the binding +// or the query excution itself. +func NamedExec(e Ext, query string, arg interface{}) (sql.Result, error) { + q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e)) + if err != nil { + return nil, err + } + return e.Exec(q, args...) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named_context.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named_context.go new file mode 100644 index 0000000..9405007 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/named_context.go @@ -0,0 +1,132 @@ +// +build go1.8 + +package sqlx + +import ( + "context" + "database/sql" +) + +// A union interface of contextPreparer and binder, required to be able to +// prepare named statements with context (as the bindtype must be determined). +type namedPreparerContext interface { + PreparerContext + binder +} + +func prepareNamedContext(ctx context.Context, p namedPreparerContext, query string) (*NamedStmt, error) { + bindType := BindType(p.DriverName()) + q, args, err := compileNamedQuery([]byte(query), bindType) + if err != nil { + return nil, err + } + stmt, err := PreparexContext(ctx, p, q) + if err != nil { + return nil, err + } + return &NamedStmt{ + QueryString: q, + Params: args, + Stmt: stmt, + }, nil +} + +// ExecContext executes a named statement using the struct passed. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) ExecContext(ctx context.Context, arg interface{}) (sql.Result, error) { + args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) + if err != nil { + return *new(sql.Result), err + } + return n.Stmt.ExecContext(ctx, args...) +} + +// QueryContext executes a named statement using the struct argument, returning rows. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) QueryContext(ctx context.Context, arg interface{}) (*sql.Rows, error) { + args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) + if err != nil { + return nil, err + } + return n.Stmt.QueryContext(ctx, args...) +} + +// QueryRowContext executes a named statement against the database. Because sqlx cannot +// create a *sql.Row with an error condition pre-set for binding errors, sqlx +// returns a *sqlx.Row instead. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) QueryRowContext(ctx context.Context, arg interface{}) *Row { + args, err := bindAnyArgs(n.Params, arg, n.Stmt.Mapper) + if err != nil { + return &Row{err: err} + } + return n.Stmt.QueryRowxContext(ctx, args...) +} + +// MustExecContext execs a NamedStmt, panicing on error +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) MustExecContext(ctx context.Context, arg interface{}) sql.Result { + res, err := n.ExecContext(ctx, arg) + if err != nil { + panic(err) + } + return res +} + +// QueryxContext using this NamedStmt +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) QueryxContext(ctx context.Context, arg interface{}) (*Rows, error) { + r, err := n.QueryContext(ctx, arg) + if err != nil { + return nil, err + } + return &Rows{Rows: r, Mapper: n.Stmt.Mapper, unsafe: isUnsafe(n)}, err +} + +// QueryRowxContext this NamedStmt. Because of limitations with QueryRow, this is +// an alias for QueryRow. +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) QueryRowxContext(ctx context.Context, arg interface{}) *Row { + return n.QueryRowContext(ctx, arg) +} + +// SelectContext using this NamedStmt +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) SelectContext(ctx context.Context, dest interface{}, arg interface{}) error { + rows, err := n.QueryxContext(ctx, arg) + if err != nil { + return err + } + // if something happens here, we want to make sure the rows are Closed + defer rows.Close() + return scanAll(rows, dest, false) +} + +// GetContext using this NamedStmt +// Any named placeholder parameters are replaced with fields from arg. +func (n *NamedStmt) GetContext(ctx context.Context, dest interface{}, arg interface{}) error { + r := n.QueryRowxContext(ctx, arg) + return r.scanAny(dest, false) +} + +// NamedQueryContext binds a named query and then runs Query on the result using the +// provided Ext (sqlx.Tx, sqlx.Db). It works with both structs and with +// map[string]interface{} types. +func NamedQueryContext(ctx context.Context, e ExtContext, query string, arg interface{}) (*Rows, error) { + q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e)) + if err != nil { + return nil, err + } + return e.QueryxContext(ctx, q, args...) +} + +// NamedExecContext uses BindStruct to get a query executable by the driver and +// then runs Exec on the result. Returns an error from the binding +// or the query excution itself. +func NamedExecContext(ctx context.Context, e ExtContext, query string, arg interface{}) (sql.Result, error) { + q, args, err := bindNamedMapper(BindType(e.DriverName()), query, arg, mapperFor(e)) + if err != nil { + return nil, err + } + return e.ExecContext(ctx, q, args...) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go new file mode 100644 index 0000000..73c21eb --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/reflectx/reflect.go @@ -0,0 +1,441 @@ +// Package reflectx implements extensions to the standard reflect lib suitable +// for implementing marshalling and unmarshalling packages. The main Mapper type +// allows for Go-compatible named attribute access, including accessing embedded +// struct attributes and the ability to use functions and struct tags to +// customize field names. +// +package reflectx + +import ( + "reflect" + "runtime" + "strings" + "sync" +) + +// A FieldInfo is metadata for a struct field. +type FieldInfo struct { + Index []int + Path string + Field reflect.StructField + Zero reflect.Value + Name string + Options map[string]string + Embedded bool + Children []*FieldInfo + Parent *FieldInfo +} + +// A StructMap is an index of field metadata for a struct. +type StructMap struct { + Tree *FieldInfo + Index []*FieldInfo + Paths map[string]*FieldInfo + Names map[string]*FieldInfo +} + +// GetByPath returns a *FieldInfo for a given string path. +func (f StructMap) GetByPath(path string) *FieldInfo { + return f.Paths[path] +} + +// GetByTraversal returns a *FieldInfo for a given integer path. It is +// analogous to reflect.FieldByIndex, but using the cached traversal +// rather than re-executing the reflect machinery each time. +func (f StructMap) GetByTraversal(index []int) *FieldInfo { + if len(index) == 0 { + return nil + } + + tree := f.Tree + for _, i := range index { + if i >= len(tree.Children) || tree.Children[i] == nil { + return nil + } + tree = tree.Children[i] + } + return tree +} + +// Mapper is a general purpose mapper of names to struct fields. A Mapper +// behaves like most marshallers in the standard library, obeying a field tag +// for name mapping but also providing a basic transform function. +type Mapper struct { + cache map[reflect.Type]*StructMap + tagName string + tagMapFunc func(string) string + mapFunc func(string) string + mutex sync.Mutex +} + +// NewMapper returns a new mapper using the tagName as its struct field tag. +// If tagName is the empty string, it is ignored. +func NewMapper(tagName string) *Mapper { + return &Mapper{ + cache: make(map[reflect.Type]*StructMap), + tagName: tagName, + } +} + +// NewMapperTagFunc returns a new mapper which contains a mapper for field names +// AND a mapper for tag values. This is useful for tags like json which can +// have values like "name,omitempty". +func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper { + return &Mapper{ + cache: make(map[reflect.Type]*StructMap), + tagName: tagName, + mapFunc: mapFunc, + tagMapFunc: tagMapFunc, + } +} + +// NewMapperFunc returns a new mapper which optionally obeys a field tag and +// a struct field name mapper func given by f. Tags will take precedence, but +// for any other field, the mapped name will be f(field.Name) +func NewMapperFunc(tagName string, f func(string) string) *Mapper { + return &Mapper{ + cache: make(map[reflect.Type]*StructMap), + tagName: tagName, + mapFunc: f, + } +} + +// TypeMap returns a mapping of field strings to int slices representing +// the traversal down the struct to reach the field. +func (m *Mapper) TypeMap(t reflect.Type) *StructMap { + m.mutex.Lock() + mapping, ok := m.cache[t] + if !ok { + mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc) + m.cache[t] = mapping + } + m.mutex.Unlock() + return mapping +} + +// FieldMap returns the mapper's mapping of field names to reflect values. Panics +// if v's Kind is not Struct, or v is not Indirectable to a struct kind. +func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value { + v = reflect.Indirect(v) + mustBe(v, reflect.Struct) + + r := map[string]reflect.Value{} + tm := m.TypeMap(v.Type()) + for tagName, fi := range tm.Names { + r[tagName] = FieldByIndexes(v, fi.Index) + } + return r +} + +// FieldByName returns a field by its mapped name as a reflect.Value. +// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind. +// Returns zero Value if the name is not found. +func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value { + v = reflect.Indirect(v) + mustBe(v, reflect.Struct) + + tm := m.TypeMap(v.Type()) + fi, ok := tm.Names[name] + if !ok { + return v + } + return FieldByIndexes(v, fi.Index) +} + +// FieldsByName returns a slice of values corresponding to the slice of names +// for the value. Panics if v's Kind is not Struct or v is not Indirectable +// to a struct Kind. Returns zero Value for each name not found. +func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value { + v = reflect.Indirect(v) + mustBe(v, reflect.Struct) + + tm := m.TypeMap(v.Type()) + vals := make([]reflect.Value, 0, len(names)) + for _, name := range names { + fi, ok := tm.Names[name] + if !ok { + vals = append(vals, *new(reflect.Value)) + } else { + vals = append(vals, FieldByIndexes(v, fi.Index)) + } + } + return vals +} + +// TraversalsByName returns a slice of int slices which represent the struct +// traversals for each mapped name. Panics if t is not a struct or Indirectable +// to a struct. Returns empty int slice for each name not found. +func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int { + r := make([][]int, 0, len(names)) + m.TraversalsByNameFunc(t, names, func(_ int, i []int) error { + if i == nil { + r = append(r, []int{}) + } else { + r = append(r, i) + } + + return nil + }) + return r +} + +// TraversalsByNameFunc traverses the mapped names and calls fn with the index of +// each name and the struct traversal represented by that name. Panics if t is not +// a struct or Indirectable to a struct. Returns the first error returned by fn or nil. +func (m *Mapper) TraversalsByNameFunc(t reflect.Type, names []string, fn func(int, []int) error) error { + t = Deref(t) + mustBe(t, reflect.Struct) + tm := m.TypeMap(t) + for i, name := range names { + fi, ok := tm.Names[name] + if !ok { + if err := fn(i, nil); err != nil { + return err + } + } else { + if err := fn(i, fi.Index); err != nil { + return err + } + } + } + return nil +} + +// FieldByIndexes returns a value for the field given by the struct traversal +// for the given value. +func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value { + for _, i := range indexes { + v = reflect.Indirect(v).Field(i) + // if this is a pointer and it's nil, allocate a new value and set it + if v.Kind() == reflect.Ptr && v.IsNil() { + alloc := reflect.New(Deref(v.Type())) + v.Set(alloc) + } + if v.Kind() == reflect.Map && v.IsNil() { + v.Set(reflect.MakeMap(v.Type())) + } + } + return v +} + +// FieldByIndexesReadOnly returns a value for a particular struct traversal, +// but is not concerned with allocating nil pointers because the value is +// going to be used for reading and not setting. +func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value { + for _, i := range indexes { + v = reflect.Indirect(v).Field(i) + } + return v +} + +// Deref is Indirect for reflect.Types +func Deref(t reflect.Type) reflect.Type { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t +} + +// -- helpers & utilities -- + +type kinder interface { + Kind() reflect.Kind +} + +// mustBe checks a value against a kind, panicing with a reflect.ValueError +// if the kind isn't that which is required. +func mustBe(v kinder, expected reflect.Kind) { + if k := v.Kind(); k != expected { + panic(&reflect.ValueError{Method: methodName(), Kind: k}) + } +} + +// methodName returns the caller of the function calling methodName +func methodName() string { + pc, _, _, _ := runtime.Caller(2) + f := runtime.FuncForPC(pc) + if f == nil { + return "unknown method" + } + return f.Name() +} + +type typeQueue struct { + t reflect.Type + fi *FieldInfo + pp string // Parent path +} + +// A copying append that creates a new slice each time. +func apnd(is []int, i int) []int { + x := make([]int, len(is)+1) + for p, n := range is { + x[p] = n + } + x[len(x)-1] = i + return x +} + +type mapf func(string) string + +// parseName parses the tag and the target name for the given field using +// the tagName (eg 'json' for `json:"foo"` tags), mapFunc for mapping the +// field's name to a target name, and tagMapFunc for mapping the tag to +// a target name. +func parseName(field reflect.StructField, tagName string, mapFunc, tagMapFunc mapf) (tag, fieldName string) { + // first, set the fieldName to the field's name + fieldName = field.Name + // if a mapFunc is set, use that to override the fieldName + if mapFunc != nil { + fieldName = mapFunc(fieldName) + } + + // if there's no tag to look for, return the field name + if tagName == "" { + return "", fieldName + } + + // if this tag is not set using the normal convention in the tag, + // then return the fieldname.. this check is done because according + // to the reflect documentation: + // If the tag does not have the conventional format, + // the value returned by Get is unspecified. + // which doesn't sound great. + if !strings.Contains(string(field.Tag), tagName+":") { + return "", fieldName + } + + // at this point we're fairly sure that we have a tag, so lets pull it out + tag = field.Tag.Get(tagName) + + // if we have a mapper function, call it on the whole tag + // XXX: this is a change from the old version, which pulled out the name + // before the tagMapFunc could be run, but I think this is the right way + if tagMapFunc != nil { + tag = tagMapFunc(tag) + } + + // finally, split the options from the name + parts := strings.Split(tag, ",") + fieldName = parts[0] + + return tag, fieldName +} + +// parseOptions parses options out of a tag string, skipping the name +func parseOptions(tag string) map[string]string { + parts := strings.Split(tag, ",") + options := make(map[string]string, len(parts)) + if len(parts) > 1 { + for _, opt := range parts[1:] { + // short circuit potentially expensive split op + if strings.Contains(opt, "=") { + kv := strings.Split(opt, "=") + options[kv[0]] = kv[1] + continue + } + options[opt] = "" + } + } + return options +} + +// getMapping returns a mapping for the t type, using the tagName, mapFunc and +// tagMapFunc to determine the canonical names of fields. +func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc mapf) *StructMap { + m := []*FieldInfo{} + + root := &FieldInfo{} + queue := []typeQueue{} + queue = append(queue, typeQueue{Deref(t), root, ""}) + +QueueLoop: + for len(queue) != 0 { + // pop the first item off of the queue + tq := queue[0] + queue = queue[1:] + + // ignore recursive field + for p := tq.fi.Parent; p != nil; p = p.Parent { + if tq.fi.Field.Type == p.Field.Type { + continue QueueLoop + } + } + + nChildren := 0 + if tq.t.Kind() == reflect.Struct { + nChildren = tq.t.NumField() + } + tq.fi.Children = make([]*FieldInfo, nChildren) + + // iterate through all of its fields + for fieldPos := 0; fieldPos < nChildren; fieldPos++ { + + f := tq.t.Field(fieldPos) + + // parse the tag and the target name using the mapping options for this field + tag, name := parseName(f, tagName, mapFunc, tagMapFunc) + + // if the name is "-", disabled via a tag, skip it + if name == "-" { + continue + } + + fi := FieldInfo{ + Field: f, + Name: name, + Zero: reflect.New(f.Type).Elem(), + Options: parseOptions(tag), + } + + // if the path is empty this path is just the name + if tq.pp == "" { + fi.Path = fi.Name + } else { + fi.Path = tq.pp + "." + fi.Name + } + + // skip unexported fields + if len(f.PkgPath) != 0 && !f.Anonymous { + continue + } + + // bfs search of anonymous embedded structs + if f.Anonymous { + pp := tq.pp + if tag != "" { + pp = fi.Path + } + + fi.Embedded = true + fi.Index = apnd(tq.fi.Index, fieldPos) + nChildren := 0 + ft := Deref(f.Type) + if ft.Kind() == reflect.Struct { + nChildren = ft.NumField() + } + fi.Children = make([]*FieldInfo, nChildren) + queue = append(queue, typeQueue{Deref(f.Type), &fi, pp}) + } else if fi.Zero.Kind() == reflect.Struct || (fi.Zero.Kind() == reflect.Ptr && fi.Zero.Type().Elem().Kind() == reflect.Struct) { + fi.Index = apnd(tq.fi.Index, fieldPos) + fi.Children = make([]*FieldInfo, Deref(f.Type).NumField()) + queue = append(queue, typeQueue{Deref(f.Type), &fi, fi.Path}) + } + + fi.Index = apnd(tq.fi.Index, fieldPos) + fi.Parent = tq.fi + tq.fi.Children[fieldPos] = &fi + m = append(m, &fi) + } + } + + flds := &StructMap{Index: m, Tree: root, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}} + for _, fi := range flds.Index { + flds.Paths[fi.Path] = fi + if fi.Name != "" && !fi.Embedded { + flds.Names[fi.Path] = fi + } + } + + return flds +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx.go new file mode 100644 index 0000000..4385c3f --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx.go @@ -0,0 +1,1047 @@ +package sqlx + +import ( + "database/sql" + "database/sql/driver" + "errors" + "fmt" + + "io/ioutil" + "path/filepath" + "reflect" + "strings" + "sync" + + "github.com/jmoiron/sqlx/reflectx" +) + +// Although the NameMapper is convenient, in practice it should not +// be relied on except for application code. If you are writing a library +// that uses sqlx, you should be aware that the name mappings you expect +// can be overridden by your user's application. + +// NameMapper is used to map column names to struct field names. By default, +// it uses strings.ToLower to lowercase struct field names. It can be set +// to whatever you want, but it is encouraged to be set before sqlx is used +// as name-to-field mappings are cached after first use on a type. +var NameMapper = strings.ToLower +var origMapper = reflect.ValueOf(NameMapper) + +// Rather than creating on init, this is created when necessary so that +// importers have time to customize the NameMapper. +var mpr *reflectx.Mapper + +// mprMu protects mpr. +var mprMu sync.Mutex + +// mapper returns a valid mapper using the configured NameMapper func. +func mapper() *reflectx.Mapper { + mprMu.Lock() + defer mprMu.Unlock() + + if mpr == nil { + mpr = reflectx.NewMapperFunc("db", NameMapper) + } else if origMapper != reflect.ValueOf(NameMapper) { + // if NameMapper has changed, create a new mapper + mpr = reflectx.NewMapperFunc("db", NameMapper) + origMapper = reflect.ValueOf(NameMapper) + } + return mpr +} + +// isScannable takes the reflect.Type and the actual dest value and returns +// whether or not it's Scannable. Something is scannable if: +// * it is not a struct +// * it implements sql.Scanner +// * it has no exported fields +func isScannable(t reflect.Type) bool { + if reflect.PtrTo(t).Implements(_scannerInterface) { + return true + } + if t.Kind() != reflect.Struct { + return true + } + + // it's not important that we use the right mapper for this particular object, + // we're only concerned on how many exported fields this struct has + m := mapper() + if len(m.TypeMap(t).Index) == 0 { + return true + } + return false +} + +// ColScanner is an interface used by MapScan and SliceScan +type ColScanner interface { + Columns() ([]string, error) + Scan(dest ...interface{}) error + Err() error +} + +// Queryer is an interface used by Get and Select +type Queryer interface { + Query(query string, args ...interface{}) (*sql.Rows, error) + Queryx(query string, args ...interface{}) (*Rows, error) + QueryRowx(query string, args ...interface{}) *Row +} + +// Execer is an interface used by MustExec and LoadFile +type Execer interface { + Exec(query string, args ...interface{}) (sql.Result, error) +} + +// Binder is an interface for something which can bind queries (Tx, DB) +type binder interface { + DriverName() string + Rebind(string) string + BindNamed(string, interface{}) (string, []interface{}, error) +} + +// Ext is a union interface which can bind, query, and exec, used by +// NamedQuery and NamedExec. +type Ext interface { + binder + Queryer + Execer +} + +// Preparer is an interface used by Preparex. +type Preparer interface { + Prepare(query string) (*sql.Stmt, error) +} + +// determine if any of our extensions are unsafe +func isUnsafe(i interface{}) bool { + switch v := i.(type) { + case Row: + return v.unsafe + case *Row: + return v.unsafe + case Rows: + return v.unsafe + case *Rows: + return v.unsafe + case NamedStmt: + return v.Stmt.unsafe + case *NamedStmt: + return v.Stmt.unsafe + case Stmt: + return v.unsafe + case *Stmt: + return v.unsafe + case qStmt: + return v.unsafe + case *qStmt: + return v.unsafe + case DB: + return v.unsafe + case *DB: + return v.unsafe + case Tx: + return v.unsafe + case *Tx: + return v.unsafe + case sql.Rows, *sql.Rows: + return false + default: + return false + } +} + +func mapperFor(i interface{}) *reflectx.Mapper { + switch i.(type) { + case DB: + return i.(DB).Mapper + case *DB: + return i.(*DB).Mapper + case Tx: + return i.(Tx).Mapper + case *Tx: + return i.(*Tx).Mapper + default: + return mapper() + } +} + +var _scannerInterface = reflect.TypeOf((*sql.Scanner)(nil)).Elem() +var _valuerInterface = reflect.TypeOf((*driver.Valuer)(nil)).Elem() + +// Row is a reimplementation of sql.Row in order to gain access to the underlying +// sql.Rows.Columns() data, necessary for StructScan. +type Row struct { + err error + unsafe bool + rows *sql.Rows + Mapper *reflectx.Mapper +} + +// Scan is a fixed implementation of sql.Row.Scan, which does not discard the +// underlying error from the internal rows object if it exists. +func (r *Row) Scan(dest ...interface{}) error { + if r.err != nil { + return r.err + } + + // TODO(bradfitz): for now we need to defensively clone all + // []byte that the driver returned (not permitting + // *RawBytes in Rows.Scan), since we're about to close + // the Rows in our defer, when we return from this function. + // the contract with the driver.Next(...) interface is that it + // can return slices into read-only temporary memory that's + // only valid until the next Scan/Close. But the TODO is that + // for a lot of drivers, this copy will be unnecessary. We + // should provide an optional interface for drivers to + // implement to say, "don't worry, the []bytes that I return + // from Next will not be modified again." (for instance, if + // they were obtained from the network anyway) But for now we + // don't care. + defer r.rows.Close() + for _, dp := range dest { + if _, ok := dp.(*sql.RawBytes); ok { + return errors.New("sql: RawBytes isn't allowed on Row.Scan") + } + } + + if !r.rows.Next() { + if err := r.rows.Err(); err != nil { + return err + } + return sql.ErrNoRows + } + err := r.rows.Scan(dest...) + if err != nil { + return err + } + // Make sure the query can be processed to completion with no errors. + if err := r.rows.Close(); err != nil { + return err + } + return nil +} + +// Columns returns the underlying sql.Rows.Columns(), or the deferred error usually +// returned by Row.Scan() +func (r *Row) Columns() ([]string, error) { + if r.err != nil { + return []string{}, r.err + } + return r.rows.Columns() +} + +// ColumnTypes returns the underlying sql.Rows.ColumnTypes(), or the deferred error +func (r *Row) ColumnTypes() ([]*sql.ColumnType, error) { + if r.err != nil { + return []*sql.ColumnType{}, r.err + } + return r.rows.ColumnTypes() +} + +// Err returns the error encountered while scanning. +func (r *Row) Err() error { + return r.err +} + +// DB is a wrapper around sql.DB which keeps track of the driverName upon Open, +// used mostly to automatically bind named queries using the right bindvars. +type DB struct { + *sql.DB + driverName string + unsafe bool + Mapper *reflectx.Mapper +} + +// NewDb returns a new sqlx DB wrapper for a pre-existing *sql.DB. The +// driverName of the original database is required for named query support. +func NewDb(db *sql.DB, driverName string) *DB { + return &DB{DB: db, driverName: driverName, Mapper: mapper()} +} + +// DriverName returns the driverName passed to the Open function for this DB. +func (db *DB) DriverName() string { + return db.driverName +} + +// Open is the same as sql.Open, but returns an *sqlx.DB instead. +func Open(driverName, dataSourceName string) (*DB, error) { + db, err := sql.Open(driverName, dataSourceName) + if err != nil { + return nil, err + } + return &DB{DB: db, driverName: driverName, Mapper: mapper()}, err +} + +// MustOpen is the same as sql.Open, but returns an *sqlx.DB instead and panics on error. +func MustOpen(driverName, dataSourceName string) *DB { + db, err := Open(driverName, dataSourceName) + if err != nil { + panic(err) + } + return db +} + +// MapperFunc sets a new mapper for this db using the default sqlx struct tag +// and the provided mapper function. +func (db *DB) MapperFunc(mf func(string) string) { + db.Mapper = reflectx.NewMapperFunc("db", mf) +} + +// Rebind transforms a query from QUESTION to the DB driver's bindvar type. +func (db *DB) Rebind(query string) string { + return Rebind(BindType(db.driverName), query) +} + +// Unsafe returns a version of DB which will silently succeed to scan when +// columns in the SQL result have no fields in the destination struct. +// sqlx.Stmt and sqlx.Tx which are created from this DB will inherit its +// safety behavior. +func (db *DB) Unsafe() *DB { + return &DB{DB: db.DB, driverName: db.driverName, unsafe: true, Mapper: db.Mapper} +} + +// BindNamed binds a query using the DB driver's bindvar type. +func (db *DB) BindNamed(query string, arg interface{}) (string, []interface{}, error) { + return bindNamedMapper(BindType(db.driverName), query, arg, db.Mapper) +} + +// NamedQuery using this DB. +// Any named placeholder parameters are replaced with fields from arg. +func (db *DB) NamedQuery(query string, arg interface{}) (*Rows, error) { + return NamedQuery(db, query, arg) +} + +// NamedExec using this DB. +// Any named placeholder parameters are replaced with fields from arg. +func (db *DB) NamedExec(query string, arg interface{}) (sql.Result, error) { + return NamedExec(db, query, arg) +} + +// Select using this DB. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) Select(dest interface{}, query string, args ...interface{}) error { + return Select(db, dest, query, args...) +} + +// Get using this DB. +// Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func (db *DB) Get(dest interface{}, query string, args ...interface{}) error { + return Get(db, dest, query, args...) +} + +// MustBegin starts a transaction, and panics on error. Returns an *sqlx.Tx instead +// of an *sql.Tx. +func (db *DB) MustBegin() *Tx { + tx, err := db.Beginx() + if err != nil { + panic(err) + } + return tx +} + +// Beginx begins a transaction and returns an *sqlx.Tx instead of an *sql.Tx. +func (db *DB) Beginx() (*Tx, error) { + tx, err := db.DB.Begin() + if err != nil { + return nil, err + } + return &Tx{Tx: tx, driverName: db.driverName, unsafe: db.unsafe, Mapper: db.Mapper}, err +} + +// Queryx queries the database and returns an *sqlx.Rows. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) Queryx(query string, args ...interface{}) (*Rows, error) { + r, err := db.DB.Query(query, args...) + if err != nil { + return nil, err + } + return &Rows{Rows: r, unsafe: db.unsafe, Mapper: db.Mapper}, err +} + +// QueryRowx queries the database and returns an *sqlx.Row. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) QueryRowx(query string, args ...interface{}) *Row { + rows, err := db.DB.Query(query, args...) + return &Row{rows: rows, err: err, unsafe: db.unsafe, Mapper: db.Mapper} +} + +// MustExec (panic) runs MustExec using this database. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) MustExec(query string, args ...interface{}) sql.Result { + return MustExec(db, query, args...) +} + +// Preparex returns an sqlx.Stmt instead of a sql.Stmt +func (db *DB) Preparex(query string) (*Stmt, error) { + return Preparex(db, query) +} + +// PrepareNamed returns an sqlx.NamedStmt +func (db *DB) PrepareNamed(query string) (*NamedStmt, error) { + return prepareNamed(db, query) +} + +// Tx is an sqlx wrapper around sql.Tx with extra functionality +type Tx struct { + *sql.Tx + driverName string + unsafe bool + Mapper *reflectx.Mapper +} + +// DriverName returns the driverName used by the DB which began this transaction. +func (tx *Tx) DriverName() string { + return tx.driverName +} + +// Rebind a query within a transaction's bindvar type. +func (tx *Tx) Rebind(query string) string { + return Rebind(BindType(tx.driverName), query) +} + +// Unsafe returns a version of Tx which will silently succeed to scan when +// columns in the SQL result have no fields in the destination struct. +func (tx *Tx) Unsafe() *Tx { + return &Tx{Tx: tx.Tx, driverName: tx.driverName, unsafe: true, Mapper: tx.Mapper} +} + +// BindNamed binds a query within a transaction's bindvar type. +func (tx *Tx) BindNamed(query string, arg interface{}) (string, []interface{}, error) { + return bindNamedMapper(BindType(tx.driverName), query, arg, tx.Mapper) +} + +// NamedQuery within a transaction. +// Any named placeholder parameters are replaced with fields from arg. +func (tx *Tx) NamedQuery(query string, arg interface{}) (*Rows, error) { + return NamedQuery(tx, query, arg) +} + +// NamedExec a named query within a transaction. +// Any named placeholder parameters are replaced with fields from arg. +func (tx *Tx) NamedExec(query string, arg interface{}) (sql.Result, error) { + return NamedExec(tx, query, arg) +} + +// Select within a transaction. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) Select(dest interface{}, query string, args ...interface{}) error { + return Select(tx, dest, query, args...) +} + +// Queryx within a transaction. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) Queryx(query string, args ...interface{}) (*Rows, error) { + r, err := tx.Tx.Query(query, args...) + if err != nil { + return nil, err + } + return &Rows{Rows: r, unsafe: tx.unsafe, Mapper: tx.Mapper}, err +} + +// QueryRowx within a transaction. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) QueryRowx(query string, args ...interface{}) *Row { + rows, err := tx.Tx.Query(query, args...) + return &Row{rows: rows, err: err, unsafe: tx.unsafe, Mapper: tx.Mapper} +} + +// Get within a transaction. +// Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func (tx *Tx) Get(dest interface{}, query string, args ...interface{}) error { + return Get(tx, dest, query, args...) +} + +// MustExec runs MustExec within a transaction. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) MustExec(query string, args ...interface{}) sql.Result { + return MustExec(tx, query, args...) +} + +// Preparex a statement within a transaction. +func (tx *Tx) Preparex(query string) (*Stmt, error) { + return Preparex(tx, query) +} + +// Stmtx returns a version of the prepared statement which runs within a transaction. Provided +// stmt can be either *sql.Stmt or *sqlx.Stmt. +func (tx *Tx) Stmtx(stmt interface{}) *Stmt { + var s *sql.Stmt + switch v := stmt.(type) { + case Stmt: + s = v.Stmt + case *Stmt: + s = v.Stmt + case sql.Stmt: + s = &v + case *sql.Stmt: + s = v + default: + panic(fmt.Sprintf("non-statement type %v passed to Stmtx", reflect.ValueOf(stmt).Type())) + } + return &Stmt{Stmt: tx.Stmt(s), Mapper: tx.Mapper} +} + +// NamedStmt returns a version of the prepared statement which runs within a transaction. +func (tx *Tx) NamedStmt(stmt *NamedStmt) *NamedStmt { + return &NamedStmt{ + QueryString: stmt.QueryString, + Params: stmt.Params, + Stmt: tx.Stmtx(stmt.Stmt), + } +} + +// PrepareNamed returns an sqlx.NamedStmt +func (tx *Tx) PrepareNamed(query string) (*NamedStmt, error) { + return prepareNamed(tx, query) +} + +// Stmt is an sqlx wrapper around sql.Stmt with extra functionality +type Stmt struct { + *sql.Stmt + unsafe bool + Mapper *reflectx.Mapper +} + +// Unsafe returns a version of Stmt which will silently succeed to scan when +// columns in the SQL result have no fields in the destination struct. +func (s *Stmt) Unsafe() *Stmt { + return &Stmt{Stmt: s.Stmt, unsafe: true, Mapper: s.Mapper} +} + +// Select using the prepared statement. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) Select(dest interface{}, args ...interface{}) error { + return Select(&qStmt{s}, dest, "", args...) +} + +// Get using the prepared statement. +// Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func (s *Stmt) Get(dest interface{}, args ...interface{}) error { + return Get(&qStmt{s}, dest, "", args...) +} + +// MustExec (panic) using this statement. Note that the query portion of the error +// output will be blank, as Stmt does not expose its query. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) MustExec(args ...interface{}) sql.Result { + return MustExec(&qStmt{s}, "", args...) +} + +// QueryRowx using this statement. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) QueryRowx(args ...interface{}) *Row { + qs := &qStmt{s} + return qs.QueryRowx("", args...) +} + +// Queryx using this statement. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) Queryx(args ...interface{}) (*Rows, error) { + qs := &qStmt{s} + return qs.Queryx("", args...) +} + +// qStmt is an unexposed wrapper which lets you use a Stmt as a Queryer & Execer by +// implementing those interfaces and ignoring the `query` argument. +type qStmt struct{ *Stmt } + +func (q *qStmt) Query(query string, args ...interface{}) (*sql.Rows, error) { + return q.Stmt.Query(args...) +} + +func (q *qStmt) Queryx(query string, args ...interface{}) (*Rows, error) { + r, err := q.Stmt.Query(args...) + if err != nil { + return nil, err + } + return &Rows{Rows: r, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper}, err +} + +func (q *qStmt) QueryRowx(query string, args ...interface{}) *Row { + rows, err := q.Stmt.Query(args...) + return &Row{rows: rows, err: err, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper} +} + +func (q *qStmt) Exec(query string, args ...interface{}) (sql.Result, error) { + return q.Stmt.Exec(args...) +} + +// Rows is a wrapper around sql.Rows which caches costly reflect operations +// during a looped StructScan +type Rows struct { + *sql.Rows + unsafe bool + Mapper *reflectx.Mapper + // these fields cache memory use for a rows during iteration w/ structScan + started bool + fields [][]int + values []interface{} +} + +// SliceScan using this Rows. +func (r *Rows) SliceScan() ([]interface{}, error) { + return SliceScan(r) +} + +// MapScan using this Rows. +func (r *Rows) MapScan(dest map[string]interface{}) error { + return MapScan(r, dest) +} + +// StructScan is like sql.Rows.Scan, but scans a single Row into a single Struct. +// Use this and iterate over Rows manually when the memory load of Select() might be +// prohibitive. *Rows.StructScan caches the reflect work of matching up column +// positions to fields to avoid that overhead per scan, which means it is not safe +// to run StructScan on the same Rows instance with different struct types. +func (r *Rows) StructScan(dest interface{}) error { + v := reflect.ValueOf(dest) + + if v.Kind() != reflect.Ptr { + return errors.New("must pass a pointer, not a value, to StructScan destination") + } + + v = v.Elem() + + if !r.started { + columns, err := r.Columns() + if err != nil { + return err + } + m := r.Mapper + + r.fields = m.TraversalsByName(v.Type(), columns) + // if we are not unsafe and are missing fields, return an error + if f, err := missingFields(r.fields); err != nil && !r.unsafe { + return fmt.Errorf("missing destination name %s in %T", columns[f], dest) + } + r.values = make([]interface{}, len(columns)) + r.started = true + } + + err := fieldsByTraversal(v, r.fields, r.values, true) + if err != nil { + return err + } + // scan into the struct field pointers and append to our results + err = r.Scan(r.values...) + if err != nil { + return err + } + return r.Err() +} + +// Connect to a database and verify with a ping. +func Connect(driverName, dataSourceName string) (*DB, error) { + db, err := Open(driverName, dataSourceName) + if err != nil { + return nil, err + } + err = db.Ping() + if err != nil { + db.Close() + return nil, err + } + return db, nil +} + +// MustConnect connects to a database and panics on error. +func MustConnect(driverName, dataSourceName string) *DB { + db, err := Connect(driverName, dataSourceName) + if err != nil { + panic(err) + } + return db +} + +// Preparex prepares a statement. +func Preparex(p Preparer, query string) (*Stmt, error) { + s, err := p.Prepare(query) + if err != nil { + return nil, err + } + return &Stmt{Stmt: s, unsafe: isUnsafe(p), Mapper: mapperFor(p)}, err +} + +// Select executes a query using the provided Queryer, and StructScans each row +// into dest, which must be a slice. If the slice elements are scannable, then +// the result set must have only one column. Otherwise, StructScan is used. +// The *sql.Rows are closed automatically. +// Any placeholder parameters are replaced with supplied args. +func Select(q Queryer, dest interface{}, query string, args ...interface{}) error { + rows, err := q.Queryx(query, args...) + if err != nil { + return err + } + // if something happens here, we want to make sure the rows are Closed + defer rows.Close() + return scanAll(rows, dest, false) +} + +// Get does a QueryRow using the provided Queryer, and scans the resulting row +// to dest. If dest is scannable, the result must only have one column. Otherwise, +// StructScan is used. Get will return sql.ErrNoRows like row.Scan would. +// Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func Get(q Queryer, dest interface{}, query string, args ...interface{}) error { + r := q.QueryRowx(query, args...) + return r.scanAny(dest, false) +} + +// LoadFile exec's every statement in a file (as a single call to Exec). +// LoadFile may return a nil *sql.Result if errors are encountered locating or +// reading the file at path. LoadFile reads the entire file into memory, so it +// is not suitable for loading large data dumps, but can be useful for initializing +// schemas or loading indexes. +// +// FIXME: this does not really work with multi-statement files for mattn/go-sqlite3 +// or the go-mysql-driver/mysql drivers; pq seems to be an exception here. Detecting +// this by requiring something with DriverName() and then attempting to split the +// queries will be difficult to get right, and its current driver-specific behavior +// is deemed at least not complex in its incorrectness. +func LoadFile(e Execer, path string) (*sql.Result, error) { + realpath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + contents, err := ioutil.ReadFile(realpath) + if err != nil { + return nil, err + } + res, err := e.Exec(string(contents)) + return &res, err +} + +// MustExec execs the query using e and panics if there was an error. +// Any placeholder parameters are replaced with supplied args. +func MustExec(e Execer, query string, args ...interface{}) sql.Result { + res, err := e.Exec(query, args...) + if err != nil { + panic(err) + } + return res +} + +// SliceScan using this Rows. +func (r *Row) SliceScan() ([]interface{}, error) { + return SliceScan(r) +} + +// MapScan using this Rows. +func (r *Row) MapScan(dest map[string]interface{}) error { + return MapScan(r, dest) +} + +func (r *Row) scanAny(dest interface{}, structOnly bool) error { + if r.err != nil { + return r.err + } + if r.rows == nil { + r.err = sql.ErrNoRows + return r.err + } + defer r.rows.Close() + + v := reflect.ValueOf(dest) + if v.Kind() != reflect.Ptr { + return errors.New("must pass a pointer, not a value, to StructScan destination") + } + if v.IsNil() { + return errors.New("nil pointer passed to StructScan destination") + } + + base := reflectx.Deref(v.Type()) + scannable := isScannable(base) + + if structOnly && scannable { + return structOnlyError(base) + } + + columns, err := r.Columns() + if err != nil { + return err + } + + if scannable && len(columns) > 1 { + return fmt.Errorf("scannable dest type %s with >1 columns (%d) in result", base.Kind(), len(columns)) + } + + if scannable { + return r.Scan(dest) + } + + m := r.Mapper + + fields := m.TraversalsByName(v.Type(), columns) + // if we are not unsafe and are missing fields, return an error + if f, err := missingFields(fields); err != nil && !r.unsafe { + return fmt.Errorf("missing destination name %s in %T", columns[f], dest) + } + values := make([]interface{}, len(columns)) + + err = fieldsByTraversal(v, fields, values, true) + if err != nil { + return err + } + // scan into the struct field pointers and append to our results + return r.Scan(values...) +} + +// StructScan a single Row into dest. +func (r *Row) StructScan(dest interface{}) error { + return r.scanAny(dest, true) +} + +// SliceScan a row, returning a []interface{} with values similar to MapScan. +// This function is primarily intended for use where the number of columns +// is not known. Because you can pass an []interface{} directly to Scan, +// it's recommended that you do that as it will not have to allocate new +// slices per row. +func SliceScan(r ColScanner) ([]interface{}, error) { + // ignore r.started, since we needn't use reflect for anything. + columns, err := r.Columns() + if err != nil { + return []interface{}{}, err + } + + values := make([]interface{}, len(columns)) + for i := range values { + values[i] = new(interface{}) + } + + err = r.Scan(values...) + + if err != nil { + return values, err + } + + for i := range columns { + values[i] = *(values[i].(*interface{})) + } + + return values, r.Err() +} + +// MapScan scans a single Row into the dest map[string]interface{}. +// Use this to get results for SQL that might not be under your control +// (for instance, if you're building an interface for an SQL server that +// executes SQL from input). Please do not use this as a primary interface! +// This will modify the map sent to it in place, so reuse the same map with +// care. Columns which occur more than once in the result will overwrite +// each other! +func MapScan(r ColScanner, dest map[string]interface{}) error { + // ignore r.started, since we needn't use reflect for anything. + columns, err := r.Columns() + if err != nil { + return err + } + + values := make([]interface{}, len(columns)) + for i := range values { + values[i] = new(interface{}) + } + + err = r.Scan(values...) + if err != nil { + return err + } + + for i, column := range columns { + dest[column] = *(values[i].(*interface{})) + } + + return r.Err() +} + +type rowsi interface { + Close() error + Columns() ([]string, error) + Err() error + Next() bool + Scan(...interface{}) error +} + +// structOnlyError returns an error appropriate for type when a non-scannable +// struct is expected but something else is given +func structOnlyError(t reflect.Type) error { + isStruct := t.Kind() == reflect.Struct + isScanner := reflect.PtrTo(t).Implements(_scannerInterface) + if !isStruct { + return fmt.Errorf("expected %s but got %s", reflect.Struct, t.Kind()) + } + if isScanner { + return fmt.Errorf("structscan expects a struct dest but the provided struct type %s implements scanner", t.Name()) + } + return fmt.Errorf("expected a struct, but struct %s has no exported fields", t.Name()) +} + +// scanAll scans all rows into a destination, which must be a slice of any +// type. If the destination slice type is a Struct, then StructScan will be +// used on each row. If the destination is some other kind of base type, then +// each row must only have one column which can scan into that type. This +// allows you to do something like: +// +// rows, _ := db.Query("select id from people;") +// var ids []int +// scanAll(rows, &ids, false) +// +// and ids will be a list of the id results. I realize that this is a desirable +// interface to expose to users, but for now it will only be exposed via changes +// to `Get` and `Select`. The reason that this has been implemented like this is +// this is the only way to not duplicate reflect work in the new API while +// maintaining backwards compatibility. +func scanAll(rows rowsi, dest interface{}, structOnly bool) error { + var v, vp reflect.Value + + value := reflect.ValueOf(dest) + + // json.Unmarshal returns errors for these + if value.Kind() != reflect.Ptr { + return errors.New("must pass a pointer, not a value, to StructScan destination") + } + if value.IsNil() { + return errors.New("nil pointer passed to StructScan destination") + } + direct := reflect.Indirect(value) + + slice, err := baseType(value.Type(), reflect.Slice) + if err != nil { + return err + } + + isPtr := slice.Elem().Kind() == reflect.Ptr + base := reflectx.Deref(slice.Elem()) + scannable := isScannable(base) + + if structOnly && scannable { + return structOnlyError(base) + } + + columns, err := rows.Columns() + if err != nil { + return err + } + + // if it's a base type make sure it only has 1 column; if not return an error + if scannable && len(columns) > 1 { + return fmt.Errorf("non-struct dest type %s with >1 columns (%d)", base.Kind(), len(columns)) + } + + if !scannable { + var values []interface{} + var m *reflectx.Mapper + + switch rows.(type) { + case *Rows: + m = rows.(*Rows).Mapper + default: + m = mapper() + } + + fields := m.TraversalsByName(base, columns) + // if we are not unsafe and are missing fields, return an error + if f, err := missingFields(fields); err != nil && !isUnsafe(rows) { + return fmt.Errorf("missing destination name %s in %T", columns[f], dest) + } + values = make([]interface{}, len(columns)) + + for rows.Next() { + // create a new struct type (which returns PtrTo) and indirect it + vp = reflect.New(base) + v = reflect.Indirect(vp) + + err = fieldsByTraversal(v, fields, values, true) + if err != nil { + return err + } + + // scan into the struct field pointers and append to our results + err = rows.Scan(values...) + if err != nil { + return err + } + + if isPtr { + direct.Set(reflect.Append(direct, vp)) + } else { + direct.Set(reflect.Append(direct, v)) + } + } + } else { + for rows.Next() { + vp = reflect.New(base) + err = rows.Scan(vp.Interface()) + if err != nil { + return err + } + // append + if isPtr { + direct.Set(reflect.Append(direct, vp)) + } else { + direct.Set(reflect.Append(direct, reflect.Indirect(vp))) + } + } + } + + return rows.Err() +} + +// FIXME: StructScan was the very first bit of API in sqlx, and now unfortunately +// it doesn't really feel like it's named properly. There is an incongruency +// between this and the way that StructScan (which might better be ScanStruct +// anyway) works on a rows object. + +// StructScan all rows from an sql.Rows or an sqlx.Rows into the dest slice. +// StructScan will scan in the entire rows result, so if you do not want to +// allocate structs for the entire result, use Queryx and see sqlx.Rows.StructScan. +// If rows is sqlx.Rows, it will use its mapper, otherwise it will use the default. +func StructScan(rows rowsi, dest interface{}) error { + return scanAll(rows, dest, true) + +} + +// reflect helpers + +func baseType(t reflect.Type, expected reflect.Kind) (reflect.Type, error) { + t = reflectx.Deref(t) + if t.Kind() != expected { + return nil, fmt.Errorf("expected %s but got %s", expected, t.Kind()) + } + return t, nil +} + +// fieldsByName fills a values interface with fields from the passed value based +// on the traversals in int. If ptrs is true, return addresses instead of values. +// We write this instead of using FieldsByName to save allocations and map lookups +// when iterating over many rows. Empty traversals will get an interface pointer. +// Because of the necessity of requesting ptrs or values, it's considered a bit too +// specialized for inclusion in reflectx itself. +func fieldsByTraversal(v reflect.Value, traversals [][]int, values []interface{}, ptrs bool) error { + v = reflect.Indirect(v) + if v.Kind() != reflect.Struct { + return errors.New("argument not a struct") + } + + for i, traversal := range traversals { + if len(traversal) == 0 { + values[i] = new(interface{}) + continue + } + f := reflectx.FieldByIndexes(v, traversal) + if ptrs { + values[i] = f.Addr().Interface() + } else { + values[i] = f.Interface() + } + } + return nil +} + +func missingFields(transversals [][]int) (field int, err error) { + for i, t := range transversals { + if len(t) == 0 { + return i, errors.New("missing field") + } + } + return 0, nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx_context.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx_context.go new file mode 100644 index 0000000..d58ff33 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/jmoiron/sqlx/sqlx_context.go @@ -0,0 +1,348 @@ +// +build go1.8 + +package sqlx + +import ( + "context" + "database/sql" + "fmt" + "io/ioutil" + "path/filepath" + "reflect" +) + +// ConnectContext to a database and verify with a ping. +func ConnectContext(ctx context.Context, driverName, dataSourceName string) (*DB, error) { + db, err := Open(driverName, dataSourceName) + if err != nil { + return db, err + } + err = db.PingContext(ctx) + return db, err +} + +// QueryerContext is an interface used by GetContext and SelectContext +type QueryerContext interface { + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) + QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row +} + +// PreparerContext is an interface used by PreparexContext. +type PreparerContext interface { + PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) +} + +// ExecerContext is an interface used by MustExecContext and LoadFileContext +type ExecerContext interface { + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) +} + +// ExtContext is a union interface which can bind, query, and exec, with Context +// used by NamedQueryContext and NamedExecContext. +type ExtContext interface { + binder + QueryerContext + ExecerContext +} + +// SelectContext executes a query using the provided Queryer, and StructScans +// each row into dest, which must be a slice. If the slice elements are +// scannable, then the result set must have only one column. Otherwise, +// StructScan is used. The *sql.Rows are closed automatically. +// Any placeholder parameters are replaced with supplied args. +func SelectContext(ctx context.Context, q QueryerContext, dest interface{}, query string, args ...interface{}) error { + rows, err := q.QueryxContext(ctx, query, args...) + if err != nil { + return err + } + // if something happens here, we want to make sure the rows are Closed + defer rows.Close() + return scanAll(rows, dest, false) +} + +// PreparexContext prepares a statement. +// +// The provided context is used for the preparation of the statement, not for +// the execution of the statement. +func PreparexContext(ctx context.Context, p PreparerContext, query string) (*Stmt, error) { + s, err := p.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + return &Stmt{Stmt: s, unsafe: isUnsafe(p), Mapper: mapperFor(p)}, err +} + +// GetContext does a QueryRow using the provided Queryer, and scans the +// resulting row to dest. If dest is scannable, the result must only have one +// column. Otherwise, StructScan is used. Get will return sql.ErrNoRows like +// row.Scan would. Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func GetContext(ctx context.Context, q QueryerContext, dest interface{}, query string, args ...interface{}) error { + r := q.QueryRowxContext(ctx, query, args...) + return r.scanAny(dest, false) +} + +// LoadFileContext exec's every statement in a file (as a single call to Exec). +// LoadFileContext may return a nil *sql.Result if errors are encountered +// locating or reading the file at path. LoadFile reads the entire file into +// memory, so it is not suitable for loading large data dumps, but can be useful +// for initializing schemas or loading indexes. +// +// FIXME: this does not really work with multi-statement files for mattn/go-sqlite3 +// or the go-mysql-driver/mysql drivers; pq seems to be an exception here. Detecting +// this by requiring something with DriverName() and then attempting to split the +// queries will be difficult to get right, and its current driver-specific behavior +// is deemed at least not complex in its incorrectness. +func LoadFileContext(ctx context.Context, e ExecerContext, path string) (*sql.Result, error) { + realpath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + contents, err := ioutil.ReadFile(realpath) + if err != nil { + return nil, err + } + res, err := e.ExecContext(ctx, string(contents)) + return &res, err +} + +// MustExecContext execs the query using e and panics if there was an error. +// Any placeholder parameters are replaced with supplied args. +func MustExecContext(ctx context.Context, e ExecerContext, query string, args ...interface{}) sql.Result { + res, err := e.ExecContext(ctx, query, args...) + if err != nil { + panic(err) + } + return res +} + +// PrepareNamedContext returns an sqlx.NamedStmt +func (db *DB) PrepareNamedContext(ctx context.Context, query string) (*NamedStmt, error) { + return prepareNamedContext(ctx, db, query) +} + +// NamedQueryContext using this DB. +// Any named placeholder parameters are replaced with fields from arg. +func (db *DB) NamedQueryContext(ctx context.Context, query string, arg interface{}) (*Rows, error) { + return NamedQueryContext(ctx, db, query, arg) +} + +// NamedExecContext using this DB. +// Any named placeholder parameters are replaced with fields from arg. +func (db *DB) NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) { + return NamedExecContext(ctx, db, query, arg) +} + +// SelectContext using this DB. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return SelectContext(ctx, db, dest, query, args...) +} + +// GetContext using this DB. +// Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func (db *DB) GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return GetContext(ctx, db, dest, query, args...) +} + +// PreparexContext returns an sqlx.Stmt instead of a sql.Stmt. +// +// The provided context is used for the preparation of the statement, not for +// the execution of the statement. +func (db *DB) PreparexContext(ctx context.Context, query string) (*Stmt, error) { + return PreparexContext(ctx, db, query) +} + +// QueryxContext queries the database and returns an *sqlx.Rows. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) { + r, err := db.DB.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + return &Rows{Rows: r, unsafe: db.unsafe, Mapper: db.Mapper}, err +} + +// QueryRowxContext queries the database and returns an *sqlx.Row. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row { + rows, err := db.DB.QueryContext(ctx, query, args...) + return &Row{rows: rows, err: err, unsafe: db.unsafe, Mapper: db.Mapper} +} + +// MustBeginTx starts a transaction, and panics on error. Returns an *sqlx.Tx instead +// of an *sql.Tx. +// +// The provided context is used until the transaction is committed or rolled +// back. If the context is canceled, the sql package will roll back the +// transaction. Tx.Commit will return an error if the context provided to +// MustBeginContext is canceled. +func (db *DB) MustBeginTx(ctx context.Context, opts *sql.TxOptions) *Tx { + tx, err := db.BeginTxx(ctx, opts) + if err != nil { + panic(err) + } + return tx +} + +// MustExecContext (panic) runs MustExec using this database. +// Any placeholder parameters are replaced with supplied args. +func (db *DB) MustExecContext(ctx context.Context, query string, args ...interface{}) sql.Result { + return MustExecContext(ctx, db, query, args...) +} + +// BeginTxx begins a transaction and returns an *sqlx.Tx instead of an +// *sql.Tx. +// +// The provided context is used until the transaction is committed or rolled +// back. If the context is canceled, the sql package will roll back the +// transaction. Tx.Commit will return an error if the context provided to +// BeginxContext is canceled. +func (db *DB) BeginTxx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) { + tx, err := db.DB.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + return &Tx{Tx: tx, driverName: db.driverName, unsafe: db.unsafe, Mapper: db.Mapper}, err +} + +// StmtxContext returns a version of the prepared statement which runs within a +// transaction. Provided stmt can be either *sql.Stmt or *sqlx.Stmt. +func (tx *Tx) StmtxContext(ctx context.Context, stmt interface{}) *Stmt { + var s *sql.Stmt + switch v := stmt.(type) { + case Stmt: + s = v.Stmt + case *Stmt: + s = v.Stmt + case sql.Stmt: + s = &v + case *sql.Stmt: + s = v + default: + panic(fmt.Sprintf("non-statement type %v passed to Stmtx", reflect.ValueOf(stmt).Type())) + } + return &Stmt{Stmt: tx.StmtContext(ctx, s), Mapper: tx.Mapper} +} + +// NamedStmtContext returns a version of the prepared statement which runs +// within a transaction. +func (tx *Tx) NamedStmtContext(ctx context.Context, stmt *NamedStmt) *NamedStmt { + return &NamedStmt{ + QueryString: stmt.QueryString, + Params: stmt.Params, + Stmt: tx.StmtxContext(ctx, stmt.Stmt), + } +} + +// PreparexContext returns an sqlx.Stmt instead of a sql.Stmt. +// +// The provided context is used for the preparation of the statement, not for +// the execution of the statement. +func (tx *Tx) PreparexContext(ctx context.Context, query string) (*Stmt, error) { + return PreparexContext(ctx, tx, query) +} + +// PrepareNamedContext returns an sqlx.NamedStmt +func (tx *Tx) PrepareNamedContext(ctx context.Context, query string) (*NamedStmt, error) { + return prepareNamedContext(ctx, tx, query) +} + +// MustExecContext runs MustExecContext within a transaction. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) MustExecContext(ctx context.Context, query string, args ...interface{}) sql.Result { + return MustExecContext(ctx, tx, query, args...) +} + +// QueryxContext within a transaction and context. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) { + r, err := tx.Tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + return &Rows{Rows: r, unsafe: tx.unsafe, Mapper: tx.Mapper}, err +} + +// SelectContext within a transaction and context. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return SelectContext(ctx, tx, dest, query, args...) +} + +// GetContext within a transaction and context. +// Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func (tx *Tx) GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error { + return GetContext(ctx, tx, dest, query, args...) +} + +// QueryRowxContext within a transaction and context. +// Any placeholder parameters are replaced with supplied args. +func (tx *Tx) QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row { + rows, err := tx.Tx.QueryContext(ctx, query, args...) + return &Row{rows: rows, err: err, unsafe: tx.unsafe, Mapper: tx.Mapper} +} + +// NamedExecContext using this Tx. +// Any named placeholder parameters are replaced with fields from arg. +func (tx *Tx) NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) { + return NamedExecContext(ctx, tx, query, arg) +} + +// SelectContext using the prepared statement. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) SelectContext(ctx context.Context, dest interface{}, args ...interface{}) error { + return SelectContext(ctx, &qStmt{s}, dest, "", args...) +} + +// GetContext using the prepared statement. +// Any placeholder parameters are replaced with supplied args. +// An error is returned if the result set is empty. +func (s *Stmt) GetContext(ctx context.Context, dest interface{}, args ...interface{}) error { + return GetContext(ctx, &qStmt{s}, dest, "", args...) +} + +// MustExecContext (panic) using this statement. Note that the query portion of +// the error output will be blank, as Stmt does not expose its query. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) MustExecContext(ctx context.Context, args ...interface{}) sql.Result { + return MustExecContext(ctx, &qStmt{s}, "", args...) +} + +// QueryRowxContext using this statement. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) QueryRowxContext(ctx context.Context, args ...interface{}) *Row { + qs := &qStmt{s} + return qs.QueryRowxContext(ctx, "", args...) +} + +// QueryxContext using this statement. +// Any placeholder parameters are replaced with supplied args. +func (s *Stmt) QueryxContext(ctx context.Context, args ...interface{}) (*Rows, error) { + qs := &qStmt{s} + return qs.QueryxContext(ctx, "", args...) +} + +func (q *qStmt) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { + return q.Stmt.QueryContext(ctx, args...) +} + +func (q *qStmt) QueryxContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) { + r, err := q.Stmt.QueryContext(ctx, args...) + if err != nil { + return nil, err + } + return &Rows{Rows: r, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper}, err +} + +func (q *qStmt) QueryRowxContext(ctx context.Context, query string, args ...interface{}) *Row { + rows, err := q.Stmt.QueryContext(ctx, args...) + return &Row{rows: rows, err: err, unsafe: q.Stmt.unsafe, Mapper: q.Stmt.Mapper} +} + +func (q *qStmt) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { + return q.Stmt.ExecContext(ctx, args...) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/LICENSE new file mode 100644 index 0000000..ade9307 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/LICENSE @@ -0,0 +1,191 @@ +All files in this repository are licensed as follows. If you contribute +to this repository, it is assumed that you license your contribution +under the same license unless you state otherwise. + +All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file. + +This software is licensed under the LGPLv3, included below. + +As a special exception to the GNU Lesser General Public License version 3 +("LGPL3"), the copyright holders of this Library give you permission to +convey to a third party a Combined Work that links statically or dynamically +to this Library without providing any Minimal Corresponding Source or +Minimal Application Code as set out in 4d or providing the installation +information set out in section 4e, provided that you comply with the other +provisions of LGPL3 and provided that you meet, for the Application the +terms and conditions of the license(s) which apply to the Application. + +Except as stated in this special exception, the provisions of LGPL3 will +continue to comply in full to this Library. If you modify this Library, you +may apply this exception to your version of this Library, but you are not +obliged to do so. If you do not wish to do so, delete this exception +statement from your version. This exception does not (and cannot) modify any +license terms which apply to the Application, with which you must still +comply. + + + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/doc.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/doc.go new file mode 100644 index 0000000..35b119a --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/doc.go @@ -0,0 +1,81 @@ +// Copyright 2013, 2014 Canonical Ltd. +// Licensed under the LGPLv3, see LICENCE file for details. + +/* +[godoc-link-here] + +The juju/errors provides an easy way to annotate errors without losing the +orginal error context. + +The exported `New` and `Errorf` functions are designed to replace the +`errors.New` and `fmt.Errorf` functions respectively. The same underlying +error is there, but the package also records the location at which the error +was created. + +A primary use case for this library is to add extra context any time an +error is returned from a function. + + if err := SomeFunc(); err != nil { + return err + } + +This instead becomes: + + if err := SomeFunc(); err != nil { + return errors.Trace(err) + } + +which just records the file and line number of the Trace call, or + + if err := SomeFunc(); err != nil { + return errors.Annotate(err, "more context") + } + +which also adds an annotation to the error. + +When you want to check to see if an error is of a particular type, a helper +function is normally exported by the package that returned the error, like the +`os` package does. The underlying cause of the error is available using the +`Cause` function. + + os.IsNotExist(errors.Cause(err)) + +The result of the `Error()` call on an annotated error is the annotations joined +with colons, then the result of the `Error()` method for the underlying error +that was the cause. + + err := errors.Errorf("original") + err = errors.Annotatef(err, "context") + err = errors.Annotatef(err, "more context") + err.Error() -> "more context: context: original" + +Obviously recording the file, line and functions is not very useful if you +cannot get them back out again. + + errors.ErrorStack(err) + +will return something like: + + first error + github.com/juju/errors/annotation_test.go:193: + github.com/juju/errors/annotation_test.go:194: annotation + github.com/juju/errors/annotation_test.go:195: + github.com/juju/errors/annotation_test.go:196: more context + github.com/juju/errors/annotation_test.go:197: + +The first error was generated by an external system, so there was no location +associated. The second, fourth, and last lines were generated with Trace calls, +and the other two through Annotate. + +Sometimes when responding to an error you want to return a more specific error +for the situation. + + if err := FindField(field); err != nil { + return errors.Wrap(err, errors.NotFoundf(field)) + } + +This returns an error where the complete error stack is still available, and +`errors.Cause()` will return the `NotFound` error. + +*/ +package errors diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/error.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/error.go new file mode 100644 index 0000000..b7df735 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/error.go @@ -0,0 +1,172 @@ +// Copyright 2014 Canonical Ltd. +// Licensed under the LGPLv3, see LICENCE file for details. + +package errors + +import ( + "fmt" + "reflect" + "runtime" +) + +// Err holds a description of an error along with information about +// where the error was created. +// +// It may be embedded in custom error types to add extra information that +// this errors package can understand. +type Err struct { + // message holds an annotation of the error. + message string + + // cause holds the cause of the error as returned + // by the Cause method. + cause error + + // previous holds the previous error in the error stack, if any. + previous error + + // file and line hold the source code location where the error was + // created. + file string + line int +} + +// NewErr is used to return an Err for the purpose of embedding in other +// structures. The location is not specified, and needs to be set with a call +// to SetLocation. +// +// For example: +// type FooError struct { +// errors.Err +// code int +// } +// +// func NewFooError(code int) error { +// err := &FooError{errors.NewErr("foo"), code} +// err.SetLocation(1) +// return err +// } +func NewErr(format string, args ...interface{}) Err { + return Err{ + message: fmt.Sprintf(format, args...), + } +} + +// NewErrWithCause is used to return an Err with case by other error for the purpose of embedding in other +// structures. The location is not specified, and needs to be set with a call +// to SetLocation. +// +// For example: +// type FooError struct { +// errors.Err +// code int +// } +// +// func (e *FooError) Annotate(format string, args ...interface{}) error { +// err := &FooError{errors.NewErrWithCause(e.Err, format, args...), e.code} +// err.SetLocation(1) +// return err +// }) +func NewErrWithCause(other error, format string, args ...interface{}) Err { + return Err{ + message: fmt.Sprintf(format, args...), + cause: Cause(other), + previous: other, + } +} + +// Location is the file and line of where the error was most recently +// created or annotated. +func (e *Err) Location() (filename string, line int) { + return e.file, e.line +} + +// Underlying returns the previous error in the error stack, if any. A client +// should not ever really call this method. It is used to build the error +// stack and should not be introspected by client calls. Or more +// specifically, clients should not depend on anything but the `Cause` of an +// error. +func (e *Err) Underlying() error { + return e.previous +} + +// The Cause of an error is the most recent error in the error stack that +// meets one of these criteria: the original error that was raised; the new +// error that was passed into the Wrap function; the most recently masked +// error; or nil if the error itself is considered the Cause. Normally this +// method is not invoked directly, but instead through the Cause stand alone +// function. +func (e *Err) Cause() error { + return e.cause +} + +// Message returns the message stored with the most recent location. This is +// the empty string if the most recent call was Trace, or the message stored +// with Annotate or Mask. +func (e *Err) Message() string { + return e.message +} + +// Error implements error.Error. +func (e *Err) Error() string { + // We want to walk up the stack of errors showing the annotations + // as long as the cause is the same. + err := e.previous + if !sameError(Cause(err), e.cause) && e.cause != nil { + err = e.cause + } + switch { + case err == nil: + return e.message + case e.message == "": + return err.Error() + } + return fmt.Sprintf("%s: %v", e.message, err) +} + +// Format implements fmt.Formatter +// When printing errors with %+v it also prints the stack trace. +// %#v unsurprisingly will print the real underlying type. +func (e *Err) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + switch { + case s.Flag('+'): + fmt.Fprintf(s, "%s", ErrorStack(e)) + return + case s.Flag('#'): + // avoid infinite recursion by wrapping e into a type + // that doesn't implement Formatter. + fmt.Fprintf(s, "%#v", (*unformatter)(e)) + return + } + fallthrough + case 's': + fmt.Fprintf(s, "%s", e.Error()) + } +} + +// helper for Format +type unformatter Err + +func (unformatter) Format() { /* break the fmt.Formatter interface */ } + +// SetLocation records the source location of the error at callDepth stack +// frames above the call. +func (e *Err) SetLocation(callDepth int) { + _, file, line, _ := runtime.Caller(callDepth + 1) + e.file = trimGoPath(file) + e.line = line +} + +// StackTrace returns one string for each location recorded in the stack of +// errors. The first value is the originating error, with a line for each +// other annotation or tracing of the error. +func (e *Err) StackTrace() []string { + return errorStack(e) +} + +// Ideally we'd have a way to check identity, but deep equals will do. +func sameError(e1, e2 error) bool { + return reflect.DeepEqual(e1, e2) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/errortypes.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/errortypes.go new file mode 100644 index 0000000..9b731c4 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/errortypes.go @@ -0,0 +1,309 @@ +// Copyright 2014 Canonical Ltd. +// Licensed under the LGPLv3, see LICENCE file for details. + +package errors + +import ( + "fmt" +) + +// wrap is a helper to construct an *wrapper. +func wrap(err error, format, suffix string, args ...interface{}) Err { + newErr := Err{ + message: fmt.Sprintf(format+suffix, args...), + previous: err, + } + newErr.SetLocation(2) + return newErr +} + +// notFound represents an error when something has not been found. +type notFound struct { + Err +} + +// NotFoundf returns an error which satisfies IsNotFound(). +func NotFoundf(format string, args ...interface{}) error { + return ¬Found{wrap(nil, format, " not found", args...)} +} + +// NewNotFound returns an error which wraps err that satisfies +// IsNotFound(). +func NewNotFound(err error, msg string) error { + return ¬Found{wrap(err, msg, "")} +} + +// IsNotFound reports whether err was created with NotFoundf() or +// NewNotFound(). +func IsNotFound(err error) bool { + err = Cause(err) + _, ok := err.(*notFound) + return ok +} + +// userNotFound represents an error when an inexistent user is looked up. +type userNotFound struct { + Err +} + +// UserNotFoundf returns an error which satisfies IsUserNotFound(). +func UserNotFoundf(format string, args ...interface{}) error { + return &userNotFound{wrap(nil, format, " user not found", args...)} +} + +// NewUserNotFound returns an error which wraps err and satisfies +// IsUserNotFound(). +func NewUserNotFound(err error, msg string) error { + return &userNotFound{wrap(err, msg, "")} +} + +// IsUserNotFound reports whether err was created with UserNotFoundf() or +// NewUserNotFound(). +func IsUserNotFound(err error) bool { + err = Cause(err) + _, ok := err.(*userNotFound) + return ok +} + +// unauthorized represents an error when an operation is unauthorized. +type unauthorized struct { + Err +} + +// Unauthorizedf returns an error which satisfies IsUnauthorized(). +func Unauthorizedf(format string, args ...interface{}) error { + return &unauthorized{wrap(nil, format, "", args...)} +} + +// NewUnauthorized returns an error which wraps err and satisfies +// IsUnauthorized(). +func NewUnauthorized(err error, msg string) error { + return &unauthorized{wrap(err, msg, "")} +} + +// IsUnauthorized reports whether err was created with Unauthorizedf() or +// NewUnauthorized(). +func IsUnauthorized(err error) bool { + err = Cause(err) + _, ok := err.(*unauthorized) + return ok +} + +// notImplemented represents an error when something is not +// implemented. +type notImplemented struct { + Err +} + +// NotImplementedf returns an error which satisfies IsNotImplemented(). +func NotImplementedf(format string, args ...interface{}) error { + return ¬Implemented{wrap(nil, format, " not implemented", args...)} +} + +// NewNotImplemented returns an error which wraps err and satisfies +// IsNotImplemented(). +func NewNotImplemented(err error, msg string) error { + return ¬Implemented{wrap(err, msg, "")} +} + +// IsNotImplemented reports whether err was created with +// NotImplementedf() or NewNotImplemented(). +func IsNotImplemented(err error) bool { + err = Cause(err) + _, ok := err.(*notImplemented) + return ok +} + +// alreadyExists represents and error when something already exists. +type alreadyExists struct { + Err +} + +// AlreadyExistsf returns an error which satisfies IsAlreadyExists(). +func AlreadyExistsf(format string, args ...interface{}) error { + return &alreadyExists{wrap(nil, format, " already exists", args...)} +} + +// NewAlreadyExists returns an error which wraps err and satisfies +// IsAlreadyExists(). +func NewAlreadyExists(err error, msg string) error { + return &alreadyExists{wrap(err, msg, "")} +} + +// IsAlreadyExists reports whether the error was created with +// AlreadyExistsf() or NewAlreadyExists(). +func IsAlreadyExists(err error) bool { + err = Cause(err) + _, ok := err.(*alreadyExists) + return ok +} + +// notSupported represents an error when something is not supported. +type notSupported struct { + Err +} + +// NotSupportedf returns an error which satisfies IsNotSupported(). +func NotSupportedf(format string, args ...interface{}) error { + return ¬Supported{wrap(nil, format, " not supported", args...)} +} + +// NewNotSupported returns an error which wraps err and satisfies +// IsNotSupported(). +func NewNotSupported(err error, msg string) error { + return ¬Supported{wrap(err, msg, "")} +} + +// IsNotSupported reports whether the error was created with +// NotSupportedf() or NewNotSupported(). +func IsNotSupported(err error) bool { + err = Cause(err) + _, ok := err.(*notSupported) + return ok +} + +// notValid represents an error when something is not valid. +type notValid struct { + Err +} + +// NotValidf returns an error which satisfies IsNotValid(). +func NotValidf(format string, args ...interface{}) error { + return ¬Valid{wrap(nil, format, " not valid", args...)} +} + +// NewNotValid returns an error which wraps err and satisfies IsNotValid(). +func NewNotValid(err error, msg string) error { + return ¬Valid{wrap(err, msg, "")} +} + +// IsNotValid reports whether the error was created with NotValidf() or +// NewNotValid(). +func IsNotValid(err error) bool { + err = Cause(err) + _, ok := err.(*notValid) + return ok +} + +// notProvisioned represents an error when something is not yet provisioned. +type notProvisioned struct { + Err +} + +// NotProvisionedf returns an error which satisfies IsNotProvisioned(). +func NotProvisionedf(format string, args ...interface{}) error { + return ¬Provisioned{wrap(nil, format, " not provisioned", args...)} +} + +// NewNotProvisioned returns an error which wraps err that satisfies +// IsNotProvisioned(). +func NewNotProvisioned(err error, msg string) error { + return ¬Provisioned{wrap(err, msg, "")} +} + +// IsNotProvisioned reports whether err was created with NotProvisionedf() or +// NewNotProvisioned(). +func IsNotProvisioned(err error) bool { + err = Cause(err) + _, ok := err.(*notProvisioned) + return ok +} + +// notAssigned represents an error when something is not yet assigned to +// something else. +type notAssigned struct { + Err +} + +// NotAssignedf returns an error which satisfies IsNotAssigned(). +func NotAssignedf(format string, args ...interface{}) error { + return ¬Assigned{wrap(nil, format, " not assigned", args...)} +} + +// NewNotAssigned returns an error which wraps err that satisfies +// IsNotAssigned(). +func NewNotAssigned(err error, msg string) error { + return ¬Assigned{wrap(err, msg, "")} +} + +// IsNotAssigned reports whether err was created with NotAssignedf() or +// NewNotAssigned(). +func IsNotAssigned(err error) bool { + err = Cause(err) + _, ok := err.(*notAssigned) + return ok +} + +// badRequest represents an error when a request has bad parameters. +type badRequest struct { + Err +} + +// BadRequestf returns an error which satisfies IsBadRequest(). +func BadRequestf(format string, args ...interface{}) error { + return &badRequest{wrap(nil, format, "", args...)} +} + +// NewBadRequest returns an error which wraps err that satisfies +// IsBadRequest(). +func NewBadRequest(err error, msg string) error { + return &badRequest{wrap(err, msg, "")} +} + +// IsBadRequest reports whether err was created with BadRequestf() or +// NewBadRequest(). +func IsBadRequest(err error) bool { + err = Cause(err) + _, ok := err.(*badRequest) + return ok +} + +// methodNotAllowed represents an error when an HTTP request +// is made with an inappropriate method. +type methodNotAllowed struct { + Err +} + +// MethodNotAllowedf returns an error which satisfies IsMethodNotAllowed(). +func MethodNotAllowedf(format string, args ...interface{}) error { + return &methodNotAllowed{wrap(nil, format, "", args...)} +} + +// NewMethodNotAllowed returns an error which wraps err that satisfies +// IsMethodNotAllowed(). +func NewMethodNotAllowed(err error, msg string) error { + return &methodNotAllowed{wrap(err, msg, "")} +} + +// IsMethodNotAllowed reports whether err was created with MethodNotAllowedf() or +// NewMethodNotAllowed(). +func IsMethodNotAllowed(err error) bool { + err = Cause(err) + _, ok := err.(*methodNotAllowed) + return ok +} + +// forbidden represents an error when a request cannot be completed because of +// missing privileges +type forbidden struct { + Err +} + +// Forbiddenf returns an error which satistifes IsForbidden() +func Forbiddenf(format string, args ...interface{}) error { + return &forbidden{wrap(nil, format, "", args...)} +} + +// NewForbidden returns an error which wraps err that satisfies +// IsForbidden(). +func NewForbidden(err error, msg string) error { + return &forbidden{wrap(err, msg, "")} +} + +// IsForbidden reports whether err was created with Forbiddenf() or +// NewForbidden(). +func IsForbidden(err error) bool { + err = Cause(err) + _, ok := err.(*forbidden) + return ok +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/functions.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/functions.go new file mode 100644 index 0000000..f86b09b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/functions.go @@ -0,0 +1,330 @@ +// Copyright 2014 Canonical Ltd. +// Licensed under the LGPLv3, see LICENCE file for details. + +package errors + +import ( + "fmt" + "strings" +) + +// New is a drop in replacement for the standard library errors module that records +// the location that the error is created. +// +// For example: +// return errors.New("validation failed") +// +func New(message string) error { + err := &Err{message: message} + err.SetLocation(1) + return err +} + +// Errorf creates a new annotated error and records the location that the +// error is created. This should be a drop in replacement for fmt.Errorf. +// +// For example: +// return errors.Errorf("validation failed: %s", message) +// +func Errorf(format string, args ...interface{}) error { + err := &Err{message: fmt.Sprintf(format, args...)} + err.SetLocation(1) + return err +} + +// Trace adds the location of the Trace call to the stack. The Cause of the +// resulting error is the same as the error parameter. If the other error is +// nil, the result will be nil. +// +// For example: +// if err := SomeFunc(); err != nil { +// return errors.Trace(err) +// } +// +func Trace(other error) error { + if other == nil { + return nil + } + err := &Err{previous: other, cause: Cause(other)} + err.SetLocation(1) + return err +} + +// Annotate is used to add extra context to an existing error. The location of +// the Annotate call is recorded with the annotations. The file, line and +// function are also recorded. +// +// For example: +// if err := SomeFunc(); err != nil { +// return errors.Annotate(err, "failed to frombulate") +// } +// +func Annotate(other error, message string) error { + if other == nil { + return nil + } + err := &Err{ + previous: other, + cause: Cause(other), + message: message, + } + err.SetLocation(1) + return err +} + +// Annotatef is used to add extra context to an existing error. The location of +// the Annotate call is recorded with the annotations. The file, line and +// function are also recorded. +// +// For example: +// if err := SomeFunc(); err != nil { +// return errors.Annotatef(err, "failed to frombulate the %s", arg) +// } +// +func Annotatef(other error, format string, args ...interface{}) error { + if other == nil { + return nil + } + err := &Err{ + previous: other, + cause: Cause(other), + message: fmt.Sprintf(format, args...), + } + err.SetLocation(1) + return err +} + +// DeferredAnnotatef annotates the given error (when it is not nil) with the given +// format string and arguments (like fmt.Sprintf). If *err is nil, DeferredAnnotatef +// does nothing. This method is used in a defer statement in order to annotate any +// resulting error with the same message. +// +// For example: +// +// defer DeferredAnnotatef(&err, "failed to frombulate the %s", arg) +// +func DeferredAnnotatef(err *error, format string, args ...interface{}) { + if *err == nil { + return + } + newErr := &Err{ + message: fmt.Sprintf(format, args...), + cause: Cause(*err), + previous: *err, + } + newErr.SetLocation(1) + *err = newErr +} + +// Wrap changes the Cause of the error. The location of the Wrap call is also +// stored in the error stack. +// +// For example: +// if err := SomeFunc(); err != nil { +// newErr := &packageError{"more context", private_value} +// return errors.Wrap(err, newErr) +// } +// +func Wrap(other, newDescriptive error) error { + err := &Err{ + previous: other, + cause: newDescriptive, + } + err.SetLocation(1) + return err +} + +// Wrapf changes the Cause of the error, and adds an annotation. The location +// of the Wrap call is also stored in the error stack. +// +// For example: +// if err := SomeFunc(); err != nil { +// return errors.Wrapf(err, simpleErrorType, "invalid value %q", value) +// } +// +func Wrapf(other, newDescriptive error, format string, args ...interface{}) error { + err := &Err{ + message: fmt.Sprintf(format, args...), + previous: other, + cause: newDescriptive, + } + err.SetLocation(1) + return err +} + +// Mask masks the given error with the given format string and arguments (like +// fmt.Sprintf), returning a new error that maintains the error stack, but +// hides the underlying error type. The error string still contains the full +// annotations. If you want to hide the annotations, call Wrap. +func Maskf(other error, format string, args ...interface{}) error { + if other == nil { + return nil + } + err := &Err{ + message: fmt.Sprintf(format, args...), + previous: other, + } + err.SetLocation(1) + return err +} + +// Mask hides the underlying error type, and records the location of the masking. +func Mask(other error) error { + if other == nil { + return nil + } + err := &Err{ + previous: other, + } + err.SetLocation(1) + return err +} + +// Cause returns the cause of the given error. This will be either the +// original error, or the result of a Wrap or Mask call. +// +// Cause is the usual way to diagnose errors that may have been wrapped by +// the other errors functions. +func Cause(err error) error { + var diag error + if err, ok := err.(causer); ok { + diag = err.Cause() + } + if diag != nil { + return diag + } + return err +} + +type causer interface { + Cause() error +} + +type wrapper interface { + // Message returns the top level error message, + // not including the message from the Previous + // error. + Message() string + + // Underlying returns the Previous error, or nil + // if there is none. + Underlying() error +} + +type locationer interface { + Location() (string, int) +} + +var ( + _ wrapper = (*Err)(nil) + _ locationer = (*Err)(nil) + _ causer = (*Err)(nil) +) + +// Details returns information about the stack of errors wrapped by err, in +// the format: +// +// [{filename:99: error one} {otherfile:55: cause of error one}] +// +// This is a terse alternative to ErrorStack as it returns a single line. +func Details(err error) string { + if err == nil { + return "[]" + } + var s []byte + s = append(s, '[') + for { + s = append(s, '{') + if err, ok := err.(locationer); ok { + file, line := err.Location() + if file != "" { + s = append(s, fmt.Sprintf("%s:%d", file, line)...) + s = append(s, ": "...) + } + } + if cerr, ok := err.(wrapper); ok { + s = append(s, cerr.Message()...) + err = cerr.Underlying() + } else { + s = append(s, err.Error()...) + err = nil + } + s = append(s, '}') + if err == nil { + break + } + s = append(s, ' ') + } + s = append(s, ']') + return string(s) +} + +// ErrorStack returns a string representation of the annotated error. If the +// error passed as the parameter is not an annotated error, the result is +// simply the result of the Error() method on that error. +// +// If the error is an annotated error, a multi-line string is returned where +// each line represents one entry in the annotation stack. The full filename +// from the call stack is used in the output. +// +// first error +// github.com/juju/errors/annotation_test.go:193: +// github.com/juju/errors/annotation_test.go:194: annotation +// github.com/juju/errors/annotation_test.go:195: +// github.com/juju/errors/annotation_test.go:196: more context +// github.com/juju/errors/annotation_test.go:197: +func ErrorStack(err error) string { + return strings.Join(errorStack(err), "\n") +} + +func errorStack(err error) []string { + if err == nil { + return nil + } + + // We want the first error first + var lines []string + for { + var buff []byte + if err, ok := err.(locationer); ok { + file, line := err.Location() + // Strip off the leading GOPATH/src path elements. + file = trimGoPath(file) + if file != "" { + buff = append(buff, fmt.Sprintf("%s:%d", file, line)...) + buff = append(buff, ": "...) + } + } + if cerr, ok := err.(wrapper); ok { + message := cerr.Message() + buff = append(buff, message...) + // If there is a cause for this error, and it is different to the cause + // of the underlying error, then output the error string in the stack trace. + var cause error + if err1, ok := err.(causer); ok { + cause = err1.Cause() + } + err = cerr.Underlying() + if cause != nil && !sameError(Cause(err), cause) { + if message != "" { + buff = append(buff, ": "...) + } + buff = append(buff, cause.Error()...) + } + } else { + buff = append(buff, err.Error()...) + err = nil + } + lines = append(lines, string(buff)) + if err == nil { + break + } + } + // reverse the lines to get the original error, which was at the end of + // the list, back to the start. + var result []string + for i := len(lines); i > 0; i-- { + result = append(result, lines[i-1]) + } + return result +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/path.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/path.go new file mode 100644 index 0000000..a7b726a --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/juju/errors/path.go @@ -0,0 +1,38 @@ +// Copyright 2013, 2014 Canonical Ltd. +// Licensed under the LGPLv3, see LICENCE file for details. + +package errors + +import ( + "runtime" + "strings" +) + +// prefixSize is used internally to trim the user specific path from the +// front of the returned filenames from the runtime call stack. +var prefixSize int + +// goPath is the deduced path based on the location of this file as compiled. +var goPath string + +func init() { + _, file, _, ok := runtime.Caller(0) + if file == "?" { + return + } + if ok { + // We know that the end of the file should be: + // github.com/juju/errors/path.go + size := len(file) + suffix := len("github.com/juju/errors/path.go") + goPath = file[:size-suffix] + prefixSize = len(goPath) + } +} + +func trimGoPath(filename string) string { + if strings.HasPrefix(filename, goPath) { + return filename[prefixSize:] + } + return filename +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/benchmark.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/benchmark.go new file mode 100644 index 0000000..46ea9dc --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/benchmark.go @@ -0,0 +1,187 @@ +// Copyright (c) 2012 The Go Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package check + +import ( + "fmt" + "runtime" + "time" +) + +var memStats runtime.MemStats + +// testingB is a type passed to Benchmark functions to manage benchmark +// timing and to specify the number of iterations to run. +type timer struct { + start time.Time // Time test or benchmark started + duration time.Duration + N int + bytes int64 + timerOn bool + benchTime time.Duration + // The initial states of memStats.Mallocs and memStats.TotalAlloc. + startAllocs uint64 + startBytes uint64 + // The net total of this test after being run. + netAllocs uint64 + netBytes uint64 +} + +// StartTimer starts timing a test. This function is called automatically +// before a benchmark starts, but it can also used to resume timing after +// a call to StopTimer. +func (c *C) StartTimer() { + if !c.timerOn { + c.start = time.Now() + c.timerOn = true + + runtime.ReadMemStats(&memStats) + c.startAllocs = memStats.Mallocs + c.startBytes = memStats.TotalAlloc + } +} + +// StopTimer stops timing a test. This can be used to pause the timer +// while performing complex initialization that you don't +// want to measure. +func (c *C) StopTimer() { + if c.timerOn { + c.duration += time.Now().Sub(c.start) + c.timerOn = false + runtime.ReadMemStats(&memStats) + c.netAllocs += memStats.Mallocs - c.startAllocs + c.netBytes += memStats.TotalAlloc - c.startBytes + } +} + +// ResetTimer sets the elapsed benchmark time to zero. +// It does not affect whether the timer is running. +func (c *C) ResetTimer() { + if c.timerOn { + c.start = time.Now() + runtime.ReadMemStats(&memStats) + c.startAllocs = memStats.Mallocs + c.startBytes = memStats.TotalAlloc + } + c.duration = 0 + c.netAllocs = 0 + c.netBytes = 0 +} + +// SetBytes informs the number of bytes that the benchmark processes +// on each iteration. If this is called in a benchmark it will also +// report MB/s. +func (c *C) SetBytes(n int64) { + c.bytes = n +} + +func (c *C) nsPerOp() int64 { + if c.N <= 0 { + return 0 + } + return c.duration.Nanoseconds() / int64(c.N) +} + +func (c *C) mbPerSec() float64 { + if c.bytes <= 0 || c.duration <= 0 || c.N <= 0 { + return 0 + } + return (float64(c.bytes) * float64(c.N) / 1e6) / c.duration.Seconds() +} + +func (c *C) timerString() string { + if c.N <= 0 { + return fmt.Sprintf("%3.3fs", float64(c.duration.Nanoseconds())/1e9) + } + mbs := c.mbPerSec() + mb := "" + if mbs != 0 { + mb = fmt.Sprintf("\t%7.2f MB/s", mbs) + } + nsop := c.nsPerOp() + ns := fmt.Sprintf("%10d ns/op", nsop) + if c.N > 0 && nsop < 100 { + // The format specifiers here make sure that + // the ones digits line up for all three possible formats. + if nsop < 10 { + ns = fmt.Sprintf("%13.2f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) + } else { + ns = fmt.Sprintf("%12.1f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) + } + } + memStats := "" + if c.benchMem { + allocedBytes := fmt.Sprintf("%8d B/op", int64(c.netBytes)/int64(c.N)) + allocs := fmt.Sprintf("%8d allocs/op", int64(c.netAllocs)/int64(c.N)) + memStats = fmt.Sprintf("\t%s\t%s", allocedBytes, allocs) + } + return fmt.Sprintf("%8d\t%s%s%s", c.N, ns, mb, memStats) +} + +func min(x, y int) int { + if x > y { + return y + } + return x +} + +func max(x, y int) int { + if x < y { + return y + } + return x +} + +// roundDown10 rounds a number down to the nearest power of 10. +func roundDown10(n int) int { + var tens = 0 + // tens = floor(log_10(n)) + for n > 10 { + n = n / 10 + tens++ + } + // result = 10^tens + result := 1 + for i := 0; i < tens; i++ { + result *= 10 + } + return result +} + +// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. +func roundUp(n int) int { + base := roundDown10(n) + if n < (2 * base) { + return 2 * base + } + if n < (5 * base) { + return 5 * base + } + return 10 * base +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/check.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/check.go new file mode 100644 index 0000000..fc535bc --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/check.go @@ -0,0 +1,980 @@ +// Package check is a rich testing extension for Go's testing package. +// +// For details about the project, see: +// +// http://labix.org/gocheck +// +package check + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/rand" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// ----------------------------------------------------------------------- +// Internal type which deals with suite method calling. + +const ( + fixtureKd = iota + testKd +) + +type funcKind int + +const ( + succeededSt = iota + failedSt + skippedSt + panickedSt + fixturePanickedSt + missedSt +) + +type funcStatus uint32 + +// A method value can't reach its own Method structure. +type methodType struct { + reflect.Value + Info reflect.Method +} + +func newMethod(receiver reflect.Value, i int) *methodType { + return &methodType{receiver.Method(i), receiver.Type().Method(i)} +} + +func (method *methodType) PC() uintptr { + return method.Info.Func.Pointer() +} + +func (method *methodType) suiteName() string { + t := method.Info.Type.In(0) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t.Name() +} + +func (method *methodType) String() string { + return method.suiteName() + "." + method.Info.Name +} + +func (method *methodType) matches(re *regexp.Regexp) bool { + return (re.MatchString(method.Info.Name) || + re.MatchString(method.suiteName()) || + re.MatchString(method.String())) +} + +type C struct { + method *methodType + kind funcKind + testName string + _status funcStatus + logb *logger + logw io.Writer + done chan *C + parallel chan *C + reason string + mustFail bool + tempDir *tempDir + benchMem bool + startTime time.Time + timer +} + +func (c *C) status() funcStatus { + return funcStatus(atomic.LoadUint32((*uint32)(&c._status))) +} + +func (c *C) setStatus(s funcStatus) { + atomic.StoreUint32((*uint32)(&c._status), uint32(s)) +} + +func (c *C) stopNow() { + runtime.Goexit() +} + +// logger is a concurrency safe byte.Buffer +type logger struct { + sync.Mutex + writer bytes.Buffer +} + +func (l *logger) Write(buf []byte) (int, error) { + l.Lock() + defer l.Unlock() + return l.writer.Write(buf) +} + +func (l *logger) WriteTo(w io.Writer) (int64, error) { + l.Lock() + defer l.Unlock() + return l.writer.WriteTo(w) +} + +func (l *logger) String() string { + l.Lock() + defer l.Unlock() + return l.writer.String() +} + +// ----------------------------------------------------------------------- +// Handling of temporary files and directories. + +type tempDir struct { + sync.Mutex + path string + counter int +} + +func (td *tempDir) newPath() string { + td.Lock() + defer td.Unlock() + if td.path == "" { + var err error + for i := 0; i != 100; i++ { + path := fmt.Sprintf("%s%ccheck-%d", os.TempDir(), os.PathSeparator, rand.Int()) + if err = os.Mkdir(path, 0700); err == nil { + td.path = path + break + } + } + if td.path == "" { + panic("Couldn't create temporary directory: " + err.Error()) + } + } + result := filepath.Join(td.path, strconv.Itoa(td.counter)) + td.counter += 1 + return result +} + +func (td *tempDir) removeAll() { + td.Lock() + defer td.Unlock() + if td.path != "" { + err := os.RemoveAll(td.path) + if err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Error cleaning up temporaries: "+err.Error()) + } + } +} + +// Create a new temporary directory which is automatically removed after +// the suite finishes running. +func (c *C) MkDir() string { + path := c.tempDir.newPath() + if err := os.Mkdir(path, 0700); err != nil { + panic(fmt.Sprintf("Couldn't create temporary directory %s: %s", path, err.Error())) + } + return path +} + +// ----------------------------------------------------------------------- +// Low-level logging functions. + +func (c *C) log(args ...interface{}) { + c.writeLog([]byte(fmt.Sprint(args...) + "\n")) +} + +func (c *C) logf(format string, args ...interface{}) { + c.writeLog([]byte(fmt.Sprintf(format+"\n", args...))) +} + +func (c *C) logNewLine() { + c.writeLog([]byte{'\n'}) +} + +func (c *C) writeLog(buf []byte) { + c.logb.Write(buf) + if c.logw != nil { + c.logw.Write(buf) + } +} + +func hasStringOrError(x interface{}) (ok bool) { + _, ok = x.(fmt.Stringer) + if ok { + return + } + _, ok = x.(error) + return +} + +func (c *C) logValue(label string, value interface{}) { + if label == "" { + if hasStringOrError(value) { + c.logf("... %#v (%q)", value, value) + } else { + c.logf("... %#v", value) + } + } else if value == nil { + c.logf("... %s = nil", label) + } else { + if hasStringOrError(value) { + fv := fmt.Sprintf("%#v", value) + qv := fmt.Sprintf("%q", value) + if fv != qv { + c.logf("... %s %s = %s (%s)", label, reflect.TypeOf(value), fv, qv) + return + } + } + if s, ok := value.(string); ok && isMultiLine(s) { + c.logf(`... %s %s = "" +`, label, reflect.TypeOf(value)) + c.logMultiLine(s) + } else { + c.logf("... %s %s = %#v", label, reflect.TypeOf(value), value) + } + } +} + +func (c *C) logMultiLine(s string) { + b := make([]byte, 0, len(s)*2) + i := 0 + n := len(s) + for i < n { + j := i + 1 + for j < n && s[j-1] != '\n' { + j++ + } + b = append(b, "... "...) + b = strconv.AppendQuote(b, s[i:j]) + if j < n { + b = append(b, " +"...) + } + b = append(b, '\n') + i = j + } + c.writeLog(b) +} + +func isMultiLine(s string) bool { + for i := 0; i+1 < len(s); i++ { + if s[i] == '\n' { + return true + } + } + return false +} + +func (c *C) logString(issue string) { + c.log("... ", issue) +} + +func (c *C) logCaller(skip int) { + // This is a bit heavier than it ought to be. + skip += 1 // Our own frame. + pc, callerFile, callerLine, ok := runtime.Caller(skip) + if !ok { + return + } + var testFile string + var testLine int + testFunc := runtime.FuncForPC(c.method.PC()) + if runtime.FuncForPC(pc) != testFunc { + for { + skip += 1 + if pc, file, line, ok := runtime.Caller(skip); ok { + // Note that the test line may be different on + // distinct calls for the same test. Showing + // the "internal" line is helpful when debugging. + if runtime.FuncForPC(pc) == testFunc { + testFile, testLine = file, line + break + } + } else { + break + } + } + } + if testFile != "" && (testFile != callerFile || testLine != callerLine) { + c.logCode(testFile, testLine) + } + c.logCode(callerFile, callerLine) +} + +func (c *C) logCode(path string, line int) { + c.logf("%s:%d:", nicePath(path), line) + code, err := printLine(path, line) + if code == "" { + code = "..." // XXX Open the file and take the raw line. + if err != nil { + code += err.Error() + } + } + c.log(indent(code, " ")) +} + +var valueGo = filepath.Join("reflect", "value.go") +var asmGo = filepath.Join("runtime", "asm_") + +func (c *C) logPanic(skip int, value interface{}) { + skip++ // Our own frame. + initialSkip := skip + for ; ; skip++ { + if pc, file, line, ok := runtime.Caller(skip); ok { + if skip == initialSkip { + c.logf("... Panic: %s (PC=0x%X)\n", value, pc) + } + name := niceFuncName(pc) + path := nicePath(file) + if strings.Contains(path, "/gopkg.in/check.v") { + continue + } + if name == "Value.call" && strings.HasSuffix(path, valueGo) { + continue + } + if (name == "call16" || name == "call32") && strings.Contains(path, asmGo) { + continue + } + c.logf("%s:%d\n in %s", nicePath(file), line, name) + } else { + break + } + } +} + +func (c *C) logSoftPanic(issue string) { + c.log("... Panic: ", issue) +} + +func (c *C) logArgPanic(method *methodType, expectedType string) { + c.logf("... Panic: %s argument should be %s", + niceFuncName(method.PC()), expectedType) +} + +// ----------------------------------------------------------------------- +// Some simple formatting helpers. + +var initWD, initWDErr = os.Getwd() + +func init() { + if initWDErr == nil { + initWD = strings.Replace(initWD, "\\", "/", -1) + "/" + } +} + +func nicePath(path string) string { + if initWDErr == nil { + if strings.HasPrefix(path, initWD) { + return path[len(initWD):] + } + } + return path +} + +func niceFuncPath(pc uintptr) string { + function := runtime.FuncForPC(pc) + if function != nil { + filename, line := function.FileLine(pc) + return fmt.Sprintf("%s:%d", nicePath(filename), line) + } + return "" +} + +func niceFuncName(pc uintptr) string { + function := runtime.FuncForPC(pc) + if function != nil { + name := path.Base(function.Name()) + if i := strings.Index(name, "."); i > 0 { + name = name[i+1:] + } + if strings.HasPrefix(name, "(*") { + if i := strings.Index(name, ")"); i > 0 { + name = name[2:i] + name[i+1:] + } + } + if i := strings.LastIndex(name, ".*"); i != -1 { + name = name[:i] + "." + name[i+2:] + } + if i := strings.LastIndex(name, "·"); i != -1 { + name = name[:i] + "." + name[i+2:] + } + return name + } + return "" +} + +// ----------------------------------------------------------------------- +// Result tracker to aggregate call results. + +type Result struct { + Succeeded int + Failed int + Skipped int + Panicked int + FixturePanicked int + ExpectedFailures int + Missed int // Not even tried to run, related to a panic in the fixture. + RunError error // Houston, we've got a problem. + WorkDir string // If KeepWorkDir is true +} + +type resultTracker struct { + result Result + _lastWasProblem bool + _waiting int + _missed int + _expectChan chan *C + _doneChan chan *C + _stopChan chan bool +} + +func newResultTracker() *resultTracker { + return &resultTracker{_expectChan: make(chan *C), // Synchronous + _doneChan: make(chan *C, 32), // Asynchronous + _stopChan: make(chan bool)} // Synchronous +} + +func (tracker *resultTracker) start() { + go tracker._loopRoutine() +} + +func (tracker *resultTracker) waitAndStop() { + <-tracker._stopChan +} + +func (tracker *resultTracker) expectCall(c *C) { + tracker._expectChan <- c +} + +func (tracker *resultTracker) callDone(c *C) { + tracker._doneChan <- c +} + +func (tracker *resultTracker) _loopRoutine() { + for { + var c *C + if tracker._waiting > 0 { + // Calls still running. Can't stop. + select { + // XXX Reindent this (not now to make diff clear) + case c = <-tracker._expectChan: + tracker._waiting += 1 + case c = <-tracker._doneChan: + tracker._waiting -= 1 + switch c.status() { + case succeededSt: + if c.kind == testKd { + if c.mustFail { + tracker.result.ExpectedFailures++ + } else { + tracker.result.Succeeded++ + } + } + case failedSt: + tracker.result.Failed++ + case panickedSt: + if c.kind == fixtureKd { + tracker.result.FixturePanicked++ + } else { + tracker.result.Panicked++ + } + case fixturePanickedSt: + // Track it as missed, since the panic + // was on the fixture, not on the test. + tracker.result.Missed++ + case missedSt: + tracker.result.Missed++ + case skippedSt: + if c.kind == testKd { + tracker.result.Skipped++ + } + } + } + } else { + // No calls. Can stop, but no done calls here. + select { + case tracker._stopChan <- true: + return + case c = <-tracker._expectChan: + tracker._waiting += 1 + case c = <-tracker._doneChan: + panic("Tracker got an unexpected done call.") + } + } + } +} + +// ----------------------------------------------------------------------- +// The underlying suite runner. + +type suiteRunner struct { + suite interface{} + setUpSuite, tearDownSuite *methodType + setUpTest, tearDownTest *methodType + tests []*methodType + tracker *resultTracker + tempDir *tempDir + keepDir bool + output *outputWriter + reportedProblemLast bool + benchTime time.Duration + benchMem bool +} + +type RunConf struct { + Output io.Writer + Stream bool + Verbose bool + Filter string + Benchmark bool + BenchmarkTime time.Duration // Defaults to 1 second + BenchmarkMem bool + KeepWorkDir bool + Exclude string +} + +// Create a new suiteRunner able to run all methods in the given suite. +func newSuiteRunner(suite interface{}, runConf *RunConf) *suiteRunner { + var conf RunConf + if runConf != nil { + conf = *runConf + } + if conf.Output == nil { + conf.Output = os.Stdout + } + if conf.Benchmark { + conf.Verbose = true + } + + suiteType := reflect.TypeOf(suite) + suiteNumMethods := suiteType.NumMethod() + suiteValue := reflect.ValueOf(suite) + + runner := &suiteRunner{ + suite: suite, + output: newOutputWriter(conf.Output, conf.Stream, conf.Verbose), + tracker: newResultTracker(), + benchTime: conf.BenchmarkTime, + benchMem: conf.BenchmarkMem, + tempDir: &tempDir{}, + keepDir: conf.KeepWorkDir, + tests: make([]*methodType, 0, suiteNumMethods), + } + if runner.benchTime == 0 { + runner.benchTime = 1 * time.Second + } + + var filterRegexp *regexp.Regexp + if conf.Filter != "" { + if regexp, err := regexp.Compile(conf.Filter); err != nil { + msg := "Bad filter expression: " + err.Error() + runner.tracker.result.RunError = errors.New(msg) + return runner + } else { + filterRegexp = regexp + } + } + + var excludeRegexp *regexp.Regexp + if conf.Exclude != "" { + if regexp, err := regexp.Compile(conf.Exclude); err != nil { + msg := "Bad exclude expression: " + err.Error() + runner.tracker.result.RunError = errors.New(msg) + return runner + } else { + excludeRegexp = regexp + } + } + + for i := 0; i != suiteNumMethods; i++ { + method := newMethod(suiteValue, i) + switch method.Info.Name { + case "SetUpSuite": + runner.setUpSuite = method + case "TearDownSuite": + runner.tearDownSuite = method + case "SetUpTest": + runner.setUpTest = method + case "TearDownTest": + runner.tearDownTest = method + default: + prefix := "Test" + if conf.Benchmark { + prefix = "Benchmark" + } + if !strings.HasPrefix(method.Info.Name, prefix) { + continue + } + if filterRegexp == nil || method.matches(filterRegexp) { + if excludeRegexp == nil || !method.matches(excludeRegexp) { + runner.tests = append(runner.tests, method) + } + } + } + } + return runner +} + +// Run all methods in the given suite. +func (runner *suiteRunner) run() *Result { + if runner.tracker.result.RunError == nil && len(runner.tests) > 0 { + runner.tracker.start() + if runner.checkFixtureArgs() { + c := runner.runFixture(runner.setUpSuite, "", nil) + if c == nil || c.status() == succeededSt { + var delayedC []*C + for i := 0; i != len(runner.tests); i++ { + c := runner.forkTest(runner.tests[i]) + select { + case <-c.done: + case <-c.parallel: + delayedC = append(delayedC, c) + } + if c.status() == fixturePanickedSt { + runner.skipTests(missedSt, runner.tests[i+1:]) + break + } + } + // Wait those parallel tests finish. + for _, delayed := range delayedC { + <-delayed.done + } + } else if c != nil && c.status() == skippedSt { + runner.skipTests(skippedSt, runner.tests) + } else { + runner.skipTests(missedSt, runner.tests) + } + runner.runFixture(runner.tearDownSuite, "", nil) + } else { + runner.skipTests(missedSt, runner.tests) + } + runner.tracker.waitAndStop() + if runner.keepDir { + runner.tracker.result.WorkDir = runner.tempDir.path + } else { + runner.tempDir.removeAll() + } + } + return &runner.tracker.result +} + +// Create a call object with the given suite method, and fork a +// goroutine with the provided dispatcher for running it. +func (runner *suiteRunner) forkCall(method *methodType, kind funcKind, testName string, logb *logger, dispatcher func(c *C)) *C { + var logw io.Writer + if runner.output.Stream { + logw = runner.output + } + if logb == nil { + logb = new(logger) + } + c := &C{ + method: method, + kind: kind, + testName: testName, + logb: logb, + logw: logw, + tempDir: runner.tempDir, + done: make(chan *C, 1), + parallel: make(chan *C, 1), + timer: timer{benchTime: runner.benchTime}, + startTime: time.Now(), + benchMem: runner.benchMem, + } + runner.tracker.expectCall(c) + go (func() { + runner.reportCallStarted(c) + defer runner.callDone(c) + dispatcher(c) + })() + return c +} + +// Same as forkCall(), but wait for call to finish before returning. +func (runner *suiteRunner) runFunc(method *methodType, kind funcKind, testName string, logb *logger, dispatcher func(c *C)) *C { + c := runner.forkCall(method, kind, testName, logb, dispatcher) + <-c.done + return c +} + +// Handle a finished call. If there were any panics, update the call status +// accordingly. Then, mark the call as done and report to the tracker. +func (runner *suiteRunner) callDone(c *C) { + value := recover() + if value != nil { + switch v := value.(type) { + case *fixturePanic: + if v.status == skippedSt { + c.setStatus(skippedSt) + } else { + c.logSoftPanic("Fixture has panicked (see related PANIC)") + c.setStatus(fixturePanickedSt) + } + default: + c.logPanic(1, value) + c.setStatus(panickedSt) + } + } + if c.mustFail { + switch c.status() { + case failedSt: + c.setStatus(succeededSt) + case succeededSt: + c.setStatus(failedSt) + c.logString("Error: Test succeeded, but was expected to fail") + c.logString("Reason: " + c.reason) + } + } + + runner.reportCallDone(c) + c.done <- c +} + +// Runs a fixture call synchronously. The fixture will still be run in a +// goroutine like all suite methods, but this method will not return +// while the fixture goroutine is not done, because the fixture must be +// run in a desired order. +func (runner *suiteRunner) runFixture(method *methodType, testName string, logb *logger) *C { + if method != nil { + c := runner.runFunc(method, fixtureKd, testName, logb, func(c *C) { + c.ResetTimer() + c.StartTimer() + defer c.StopTimer() + c.method.Call([]reflect.Value{reflect.ValueOf(c)}) + }) + return c + } + return nil +} + +// Run the fixture method with runFixture(), but panic with a fixturePanic{} +// in case the fixture method panics. This makes it easier to track the +// fixture panic together with other call panics within forkTest(). +func (runner *suiteRunner) runFixtureWithPanic(method *methodType, testName string, logb *logger, skipped *bool) *C { + if skipped != nil && *skipped { + return nil + } + c := runner.runFixture(method, testName, logb) + if c != nil && c.status() != succeededSt { + if skipped != nil { + *skipped = c.status() == skippedSt + } + panic(&fixturePanic{c.status(), method}) + } + return c +} + +type fixturePanic struct { + status funcStatus + method *methodType +} + +// Run the suite test method, together with the test-specific fixture, +// asynchronously. +func (runner *suiteRunner) forkTest(method *methodType) *C { + testName := method.String() + return runner.forkCall(method, testKd, testName, nil, func(c *C) { + var skipped bool + defer runner.runFixtureWithPanic(runner.tearDownTest, testName, nil, &skipped) + defer c.StopTimer() + benchN := 1 + for { + runner.runFixtureWithPanic(runner.setUpTest, testName, c.logb, &skipped) + mt := c.method.Type() + if mt.NumIn() != 1 || mt.In(0) != reflect.TypeOf(c) { + // Rather than a plain panic, provide a more helpful message when + // the argument type is incorrect. + c.setStatus(panickedSt) + c.logArgPanic(c.method, "*check.C") + return + } + if strings.HasPrefix(c.method.Info.Name, "Test") { + c.ResetTimer() + c.StartTimer() + c.method.Call([]reflect.Value{reflect.ValueOf(c)}) + return + } + if !strings.HasPrefix(c.method.Info.Name, "Benchmark") { + panic("unexpected method prefix: " + c.method.Info.Name) + } + + runtime.GC() + c.N = benchN + c.ResetTimer() + c.StartTimer() + c.method.Call([]reflect.Value{reflect.ValueOf(c)}) + c.StopTimer() + if c.status() != succeededSt || c.duration >= c.benchTime || benchN >= 1e9 { + return + } + perOpN := int(1e9) + if c.nsPerOp() != 0 { + perOpN = int(c.benchTime.Nanoseconds() / c.nsPerOp()) + } + + // Logic taken from the stock testing package: + // - Run more iterations than we think we'll need for a second (1.5x). + // - Don't grow too fast in case we had timing errors previously. + // - Be sure to run at least one more than last time. + benchN = max(min(perOpN+perOpN/2, 100*benchN), benchN+1) + benchN = roundUp(benchN) + + skipped = true // Don't run the deferred one if this panics. + runner.runFixtureWithPanic(runner.tearDownTest, testName, nil, nil) + skipped = false + } + }) +} + +// Same as forkTest(), but wait for the test to finish before returning. +func (runner *suiteRunner) runTest(method *methodType) *C { + c := runner.forkTest(method) + <-c.done + return c +} + +// Helper to mark tests as skipped or missed. A bit heavy for what +// it does, but it enables homogeneous handling of tracking, including +// nice verbose output. +func (runner *suiteRunner) skipTests(status funcStatus, methods []*methodType) { + for _, method := range methods { + runner.runFunc(method, testKd, "", nil, func(c *C) { + c.setStatus(status) + }) + } +} + +// Verify if the fixture arguments are *check.C. In case of errors, +// log the error as a panic in the fixture method call, and return false. +func (runner *suiteRunner) checkFixtureArgs() bool { + succeeded := true + argType := reflect.TypeOf(&C{}) + for _, method := range []*methodType{runner.setUpSuite, runner.tearDownSuite, runner.setUpTest, runner.tearDownTest} { + if method != nil { + mt := method.Type() + if mt.NumIn() != 1 || mt.In(0) != argType { + succeeded = false + runner.runFunc(method, fixtureKd, "", nil, func(c *C) { + c.logArgPanic(method, "*check.C") + c.setStatus(panickedSt) + }) + } + } + } + return succeeded +} + +func (runner *suiteRunner) reportCallStarted(c *C) { + runner.output.WriteCallStarted("START", c) +} + +func (runner *suiteRunner) reportCallDone(c *C) { + runner.tracker.callDone(c) + switch c.status() { + case succeededSt: + if c.mustFail { + runner.output.WriteCallSuccess("FAIL EXPECTED", c) + } else { + runner.output.WriteCallSuccess("PASS", c) + } + case skippedSt: + runner.output.WriteCallSuccess("SKIP", c) + case failedSt: + runner.output.WriteCallProblem("FAIL", c) + case panickedSt: + runner.output.WriteCallProblem("PANIC", c) + case fixturePanickedSt: + // That's a testKd call reporting that its fixture + // has panicked. The fixture call which caused the + // panic itself was tracked above. We'll report to + // aid debugging. + runner.output.WriteCallProblem("PANIC", c) + case missedSt: + runner.output.WriteCallSuccess("MISS", c) + } +} + +// ----------------------------------------------------------------------- +// Output writer manages atomic output writing according to settings. + +type outputWriter struct { + m sync.Mutex + writer io.Writer + wroteCallProblemLast bool + Stream bool + Verbose bool +} + +func newOutputWriter(writer io.Writer, stream, verbose bool) *outputWriter { + return &outputWriter{writer: writer, Stream: stream, Verbose: verbose} +} + +func (ow *outputWriter) Write(content []byte) (n int, err error) { + ow.m.Lock() + n, err = ow.writer.Write(content) + ow.m.Unlock() + return +} + +func (ow *outputWriter) WriteCallStarted(label string, c *C) { + if ow.Stream { + header := renderCallHeader(label, c, "", "\n") + ow.m.Lock() + ow.writer.Write([]byte(header)) + ow.m.Unlock() + } +} + +func (ow *outputWriter) WriteCallProblem(label string, c *C) { + var prefix string + if !ow.Stream { + prefix = "\n-----------------------------------" + + "-----------------------------------\n" + } + header := renderCallHeader(label, c, prefix, "\n\n") + ow.m.Lock() + ow.wroteCallProblemLast = true + ow.writer.Write([]byte(header)) + if !ow.Stream { + c.logb.WriteTo(ow.writer) + } + ow.m.Unlock() +} + +func (ow *outputWriter) WriteCallSuccess(label string, c *C) { + if ow.Stream || (ow.Verbose && c.kind == testKd) { + // TODO Use a buffer here. + var suffix string + if c.reason != "" { + suffix = " (" + c.reason + ")" + } + if c.status() == succeededSt { + suffix += "\t" + c.timerString() + } + suffix += "\n" + if ow.Stream { + suffix += "\n" + } + header := renderCallHeader(label, c, "", suffix) + ow.m.Lock() + // Resist temptation of using line as prefix above due to race. + if !ow.Stream && ow.wroteCallProblemLast { + header = "\n-----------------------------------" + + "-----------------------------------\n" + + header + } + ow.wroteCallProblemLast = false + ow.writer.Write([]byte(header)) + ow.m.Unlock() + } +} + +func renderCallHeader(label string, c *C, prefix, suffix string) string { + pc := c.method.PC() + return fmt.Sprintf("%s%s: %s: %s%s", prefix, label, niceFuncPath(pc), + niceFuncName(pc), suffix) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers.go new file mode 100644 index 0000000..3749545 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers.go @@ -0,0 +1,458 @@ +package check + +import ( + "fmt" + "reflect" + "regexp" +) + +// ----------------------------------------------------------------------- +// CommentInterface and Commentf helper, to attach extra information to checks. + +type comment struct { + format string + args []interface{} +} + +// Commentf returns an infomational value to use with Assert or Check calls. +// If the checker test fails, the provided arguments will be passed to +// fmt.Sprintf, and will be presented next to the logged failure. +// +// For example: +// +// c.Assert(v, Equals, 42, Commentf("Iteration #%d failed.", i)) +// +// Note that if the comment is constant, a better option is to +// simply use a normal comment right above or next to the line, as +// it will also get printed with any errors: +// +// c.Assert(l, Equals, 8192) // Ensure buffer size is correct (bug #123) +// +func Commentf(format string, args ...interface{}) CommentInterface { + return &comment{format, args} +} + +// CommentInterface must be implemented by types that attach extra +// information to failed checks. See the Commentf function for details. +type CommentInterface interface { + CheckCommentString() string +} + +func (c *comment) CheckCommentString() string { + return fmt.Sprintf(c.format, c.args...) +} + +// ----------------------------------------------------------------------- +// The Checker interface. + +// The Checker interface must be provided by checkers used with +// the Assert and Check verification methods. +type Checker interface { + Info() *CheckerInfo + Check(params []interface{}, names []string) (result bool, error string) +} + +// See the Checker interface. +type CheckerInfo struct { + Name string + Params []string +} + +func (info *CheckerInfo) Info() *CheckerInfo { + return info +} + +// ----------------------------------------------------------------------- +// Not checker logic inverter. + +// The Not checker inverts the logic of the provided checker. The +// resulting checker will succeed where the original one failed, and +// vice-versa. +// +// For example: +// +// c.Assert(a, Not(Equals), b) +// +func Not(checker Checker) Checker { + return ¬Checker{checker} +} + +type notChecker struct { + sub Checker +} + +func (checker *notChecker) Info() *CheckerInfo { + info := *checker.sub.Info() + info.Name = "Not(" + info.Name + ")" + return &info +} + +func (checker *notChecker) Check(params []interface{}, names []string) (result bool, error string) { + result, error = checker.sub.Check(params, names) + result = !result + return +} + +// ----------------------------------------------------------------------- +// IsNil checker. + +type isNilChecker struct { + *CheckerInfo +} + +// The IsNil checker tests whether the obtained value is nil. +// +// For example: +// +// c.Assert(err, IsNil) +// +var IsNil Checker = &isNilChecker{ + &CheckerInfo{Name: "IsNil", Params: []string{"value"}}, +} + +func (checker *isNilChecker) Check(params []interface{}, names []string) (result bool, error string) { + return isNil(params[0]), "" +} + +func isNil(obtained interface{}) (result bool) { + if obtained == nil { + result = true + } else { + switch v := reflect.ValueOf(obtained); v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + } + } + return +} + +// ----------------------------------------------------------------------- +// NotNil checker. Alias for Not(IsNil), since it's so common. + +type notNilChecker struct { + *CheckerInfo +} + +// The NotNil checker verifies that the obtained value is not nil. +// +// For example: +// +// c.Assert(iface, NotNil) +// +// This is an alias for Not(IsNil), made available since it's a +// fairly common check. +// +var NotNil Checker = ¬NilChecker{ + &CheckerInfo{Name: "NotNil", Params: []string{"value"}}, +} + +func (checker *notNilChecker) Check(params []interface{}, names []string) (result bool, error string) { + return !isNil(params[0]), "" +} + +// ----------------------------------------------------------------------- +// Equals checker. + +type equalsChecker struct { + *CheckerInfo +} + +// The Equals checker verifies that the obtained value is equal to +// the expected value, according to usual Go semantics for ==. +// +// For example: +// +// c.Assert(value, Equals, 42) +// +var Equals Checker = &equalsChecker{ + &CheckerInfo{Name: "Equals", Params: []string{"obtained", "expected"}}, +} + +func (checker *equalsChecker) Check(params []interface{}, names []string) (result bool, error string) { + defer func() { + if v := recover(); v != nil { + result = false + error = fmt.Sprint(v) + } + }() + return params[0] == params[1], "" +} + +// ----------------------------------------------------------------------- +// DeepEquals checker. + +type deepEqualsChecker struct { + *CheckerInfo +} + +// The DeepEquals checker verifies that the obtained value is deep-equal to +// the expected value. The check will work correctly even when facing +// slices, interfaces, and values of different types (which always fail +// the test). +// +// For example: +// +// c.Assert(value, DeepEquals, 42) +// c.Assert(array, DeepEquals, []string{"hi", "there"}) +// +var DeepEquals Checker = &deepEqualsChecker{ + &CheckerInfo{Name: "DeepEquals", Params: []string{"obtained", "expected"}}, +} + +func (checker *deepEqualsChecker) Check(params []interface{}, names []string) (result bool, error string) { + return reflect.DeepEqual(params[0], params[1]), "" +} + +// ----------------------------------------------------------------------- +// HasLen checker. + +type hasLenChecker struct { + *CheckerInfo +} + +// The HasLen checker verifies that the obtained value has the +// provided length. In many cases this is superior to using Equals +// in conjunction with the len function because in case the check +// fails the value itself will be printed, instead of its length, +// providing more details for figuring the problem. +// +// For example: +// +// c.Assert(list, HasLen, 5) +// +var HasLen Checker = &hasLenChecker{ + &CheckerInfo{Name: "HasLen", Params: []string{"obtained", "n"}}, +} + +func (checker *hasLenChecker) Check(params []interface{}, names []string) (result bool, error string) { + n, ok := params[1].(int) + if !ok { + return false, "n must be an int" + } + value := reflect.ValueOf(params[0]) + switch value.Kind() { + case reflect.Map, reflect.Array, reflect.Slice, reflect.Chan, reflect.String: + default: + return false, "obtained value type has no length" + } + return value.Len() == n, "" +} + +// ----------------------------------------------------------------------- +// ErrorMatches checker. + +type errorMatchesChecker struct { + *CheckerInfo +} + +// The ErrorMatches checker verifies that the error value +// is non nil and matches the regular expression provided. +// +// For example: +// +// c.Assert(err, ErrorMatches, "perm.*denied") +// +var ErrorMatches Checker = errorMatchesChecker{ + &CheckerInfo{Name: "ErrorMatches", Params: []string{"value", "regex"}}, +} + +func (checker errorMatchesChecker) Check(params []interface{}, names []string) (result bool, errStr string) { + if params[0] == nil { + return false, "Error value is nil" + } + err, ok := params[0].(error) + if !ok { + return false, "Value is not an error" + } + params[0] = err.Error() + names[0] = "error" + return matches(params[0], params[1]) +} + +// ----------------------------------------------------------------------- +// Matches checker. + +type matchesChecker struct { + *CheckerInfo +} + +// The Matches checker verifies that the string provided as the obtained +// value (or the string resulting from obtained.String()) matches the +// regular expression provided. +// +// For example: +// +// c.Assert(err, Matches, "perm.*denied") +// +var Matches Checker = &matchesChecker{ + &CheckerInfo{Name: "Matches", Params: []string{"value", "regex"}}, +} + +func (checker *matchesChecker) Check(params []interface{}, names []string) (result bool, error string) { + return matches(params[0], params[1]) +} + +func matches(value, regex interface{}) (result bool, error string) { + reStr, ok := regex.(string) + if !ok { + return false, "Regex must be a string" + } + valueStr, valueIsStr := value.(string) + if !valueIsStr { + if valueWithStr, valueHasStr := value.(fmt.Stringer); valueHasStr { + valueStr, valueIsStr = valueWithStr.String(), true + } + } + if valueIsStr { + matches, err := regexp.MatchString("^"+reStr+"$", valueStr) + if err != nil { + return false, "Can't compile regex: " + err.Error() + } + return matches, "" + } + return false, "Obtained value is not a string and has no .String()" +} + +// ----------------------------------------------------------------------- +// Panics checker. + +type panicsChecker struct { + *CheckerInfo +} + +// The Panics checker verifies that calling the provided zero-argument +// function will cause a panic which is deep-equal to the provided value. +// +// For example: +// +// c.Assert(func() { f(1, 2) }, Panics, &SomeErrorType{"BOOM"}). +// +// +var Panics Checker = &panicsChecker{ + &CheckerInfo{Name: "Panics", Params: []string{"function", "expected"}}, +} + +func (checker *panicsChecker) Check(params []interface{}, names []string) (result bool, error string) { + f := reflect.ValueOf(params[0]) + if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { + return false, "Function must take zero arguments" + } + defer func() { + // If the function has not panicked, then don't do the check. + if error != "" { + return + } + params[0] = recover() + names[0] = "panic" + result = reflect.DeepEqual(params[0], params[1]) + }() + f.Call(nil) + return false, "Function has not panicked" +} + +type panicMatchesChecker struct { + *CheckerInfo +} + +// The PanicMatches checker verifies that calling the provided zero-argument +// function will cause a panic with an error value matching +// the regular expression provided. +// +// For example: +// +// c.Assert(func() { f(1, 2) }, PanicMatches, `open.*: no such file or directory`). +// +// +var PanicMatches Checker = &panicMatchesChecker{ + &CheckerInfo{Name: "PanicMatches", Params: []string{"function", "expected"}}, +} + +func (checker *panicMatchesChecker) Check(params []interface{}, names []string) (result bool, errmsg string) { + f := reflect.ValueOf(params[0]) + if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { + return false, "Function must take zero arguments" + } + defer func() { + // If the function has not panicked, then don't do the check. + if errmsg != "" { + return + } + obtained := recover() + names[0] = "panic" + if e, ok := obtained.(error); ok { + params[0] = e.Error() + } else if _, ok := obtained.(string); ok { + params[0] = obtained + } else { + errmsg = "Panic value is not a string or an error" + return + } + result, errmsg = matches(params[0], params[1]) + }() + f.Call(nil) + return false, "Function has not panicked" +} + +// ----------------------------------------------------------------------- +// FitsTypeOf checker. + +type fitsTypeChecker struct { + *CheckerInfo +} + +// The FitsTypeOf checker verifies that the obtained value is +// assignable to a variable with the same type as the provided +// sample value. +// +// For example: +// +// c.Assert(value, FitsTypeOf, int64(0)) +// c.Assert(value, FitsTypeOf, os.Error(nil)) +// +var FitsTypeOf Checker = &fitsTypeChecker{ + &CheckerInfo{Name: "FitsTypeOf", Params: []string{"obtained", "sample"}}, +} + +func (checker *fitsTypeChecker) Check(params []interface{}, names []string) (result bool, error string) { + obtained := reflect.ValueOf(params[0]) + sample := reflect.ValueOf(params[1]) + if !obtained.IsValid() { + return false, "" + } + if !sample.IsValid() { + return false, "Invalid sample value" + } + return obtained.Type().AssignableTo(sample.Type()), "" +} + +// ----------------------------------------------------------------------- +// Implements checker. + +type implementsChecker struct { + *CheckerInfo +} + +// The Implements checker verifies that the obtained value +// implements the interface specified via a pointer to an interface +// variable. +// +// For example: +// +// var e os.Error +// c.Assert(err, Implements, &e) +// +var Implements Checker = &implementsChecker{ + &CheckerInfo{Name: "Implements", Params: []string{"obtained", "ifaceptr"}}, +} + +func (checker *implementsChecker) Check(params []interface{}, names []string) (result bool, error string) { + obtained := reflect.ValueOf(params[0]) + ifaceptr := reflect.ValueOf(params[1]) + if !obtained.IsValid() { + return false, "" + } + if !ifaceptr.IsValid() || ifaceptr.Kind() != reflect.Ptr || ifaceptr.Elem().Kind() != reflect.Interface { + return false, "ifaceptr should be a pointer to an interface variable" + } + return obtained.Type().Implements(ifaceptr.Elem().Type()), "" +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers2.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers2.go new file mode 100644 index 0000000..c09bcdc --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/checkers2.go @@ -0,0 +1,131 @@ +// Extensions to the go-check unittest framework. +// +// NOTE: see https://github.com/go-check/check/pull/6 for reasons why these +// checkers live here. +package check + +import ( + "bytes" + "reflect" +) + +// ----------------------------------------------------------------------- +// IsTrue / IsFalse checker. + +type isBoolValueChecker struct { + *CheckerInfo + expected bool +} + +func (checker *isBoolValueChecker) Check( + params []interface{}, + names []string) ( + result bool, + error string) { + + obtained, ok := params[0].(bool) + if !ok { + return false, "Argument to " + checker.Name + " must be bool" + } + + return obtained == checker.expected, "" +} + +// The IsTrue checker verifies that the obtained value is true. +// +// For example: +// +// c.Assert(value, IsTrue) +// +var IsTrue Checker = &isBoolValueChecker{ + &CheckerInfo{Name: "IsTrue", Params: []string{"obtained"}}, + true, +} + +// The IsFalse checker verifies that the obtained value is false. +// +// For example: +// +// c.Assert(value, IsFalse) +// +var IsFalse Checker = &isBoolValueChecker{ + &CheckerInfo{Name: "IsFalse", Params: []string{"obtained"}}, + false, +} + +// ----------------------------------------------------------------------- +// BytesEquals checker. + +type bytesEquals struct{} + +func (b *bytesEquals) Check(params []interface{}, names []string) (bool, string) { + if len(params) != 2 { + return false, "BytesEqual takes 2 bytestring arguments" + } + b1, ok1 := params[0].([]byte) + b2, ok2 := params[1].([]byte) + + if !(ok1 && ok2) { + return false, "Arguments to BytesEqual must both be bytestrings" + } + + return bytes.Equal(b1, b2), "" +} + +func (b *bytesEquals) Info() *CheckerInfo { + return &CheckerInfo{ + Name: "BytesEquals", + Params: []string{"bytes_one", "bytes_two"}, + } +} + +// BytesEquals checker compares two bytes sequence using bytes.Equal. +// +// For example: +// +// c.Assert(b, BytesEquals, []byte("bar")) +// +// Main difference between DeepEquals and BytesEquals is that BytesEquals treats +// `nil` as empty byte sequence while DeepEquals doesn't. +// +// c.Assert(nil, BytesEquals, []byte("")) // succeeds +// c.Assert(nil, DeepEquals, []byte("")) // fails +var BytesEquals = &bytesEquals{} + +// ----------------------------------------------------------------------- +// HasKey checker. + +type hasKey struct{} + +func (h *hasKey) Check(params []interface{}, names []string) (bool, string) { + if len(params) != 2 { + return false, "HasKey takes 2 arguments: a map and a key" + } + + mapValue := reflect.ValueOf(params[0]) + if mapValue.Kind() != reflect.Map { + return false, "First argument to HasKey must be a map" + } + + keyValue := reflect.ValueOf(params[1]) + if !keyValue.Type().AssignableTo(mapValue.Type().Key()) { + return false, "Second argument must be assignable to the map key type" + } + + return mapValue.MapIndex(keyValue).IsValid(), "" +} + +func (h *hasKey) Info() *CheckerInfo { + return &CheckerInfo{ + Name: "HasKey", + Params: []string{"obtained", "key"}, + } +} + +// The HasKey checker verifies that the obtained map contains the given key. +// +// For example: +// +// c.Assert(myMap, HasKey, "foo") +// +var HasKey = &hasKey{} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/compare.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/compare.go new file mode 100644 index 0000000..7005cba --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/compare.go @@ -0,0 +1,161 @@ +package check + +import ( + "bytes" + "fmt" + "reflect" + "time" +) + +type compareFunc func(v1 interface{}, v2 interface{}) (bool, error) + +type valueCompare struct { + Name string + + Func compareFunc + + Operator string +} + +// v1 and v2 must have the same type +// return >0 if v1 > v2 +// return 0 if v1 = v2 +// return <0 if v1 < v2 +// now we only support int, uint, float64, string and []byte comparison +func compare(v1 interface{}, v2 interface{}) (int, error) { + value1 := reflect.ValueOf(v1) + value2 := reflect.ValueOf(v2) + + switch v1.(type) { + case int, int8, int16, int32, int64: + a1 := value1.Int() + a2 := value2.Int() + if a1 > a2 { + return 1, nil + } else if a1 == a2 { + return 0, nil + } + return -1, nil + case uint, uint8, uint16, uint32, uint64: + a1 := value1.Uint() + a2 := value2.Uint() + if a1 > a2 { + return 1, nil + } else if a1 == a2 { + return 0, nil + } + return -1, nil + case float32, float64: + a1 := value1.Float() + a2 := value2.Float() + if a1 > a2 { + return 1, nil + } else if a1 == a2 { + return 0, nil + } + return -1, nil + case string: + a1 := value1.String() + a2 := value2.String() + if a1 > a2 { + return 1, nil + } else if a1 == a2 { + return 0, nil + } + return -1, nil + case []byte: + a1 := value1.Bytes() + a2 := value2.Bytes() + return bytes.Compare(a1, a2), nil + case time.Time: + a1 := v1.(time.Time) + a2 := v2.(time.Time) + if a1.After(a2) { + return 1, nil + } else if a1.Equal(a2) { + return 0, nil + } + return -1, nil + case time.Duration: + a1 := v1.(time.Duration) + a2 := v2.(time.Duration) + if a1 > a2 { + return 1, nil + } else if a1 == a2 { + return 0, nil + } + return -1, nil + default: + return 0, fmt.Errorf("type %T is not supported now", v1) + } +} + +func less(v1 interface{}, v2 interface{}) (bool, error) { + n, err := compare(v1, v2) + if err != nil { + return false, err + } + + return n < 0, nil +} + +func lessEqual(v1 interface{}, v2 interface{}) (bool, error) { + n, err := compare(v1, v2) + if err != nil { + return false, err + } + + return n <= 0, nil +} + +func greater(v1 interface{}, v2 interface{}) (bool, error) { + n, err := compare(v1, v2) + if err != nil { + return false, err + } + + return n > 0, nil +} + +func greaterEqual(v1 interface{}, v2 interface{}) (bool, error) { + n, err := compare(v1, v2) + if err != nil { + return false, err + } + + return n >= 0, nil +} + +func (v *valueCompare) Check(params []interface{}, names []string) (bool, string) { + if len(params) != 2 { + return false, fmt.Sprintf("%s needs 2 arguments", v.Name) + } + + v1 := params[0] + v2 := params[1] + v1Type := reflect.TypeOf(v1) + v2Type := reflect.TypeOf(v2) + + if v1Type.Kind() != v2Type.Kind() { + return false, fmt.Sprintf("%s needs two same type, but %s != %s", v.Name, v1Type.Kind(), v2Type.Kind()) + } + + b, err := v.Func(v1, v2) + if err != nil { + return false, fmt.Sprintf("%s check err %v", v.Name, err) + } + + return b, "" +} + +func (v *valueCompare) Info() *CheckerInfo { + return &CheckerInfo{ + Name: v.Name, + Params: []string{"compare_one", "compare_two"}, + } +} + +var Less = &valueCompare{Name: "Less", Func: less, Operator: "<"} +var LessEqual = &valueCompare{Name: "LessEqual", Func: lessEqual, Operator: "<="} +var Greater = &valueCompare{Name: "Greater", Func: greater, Operator: ">"} +var GreaterEqual = &valueCompare{Name: "GreaterEqual", Func: greaterEqual, Operator: ">="} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/helpers.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/helpers.go new file mode 100644 index 0000000..68e861d --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/helpers.go @@ -0,0 +1,236 @@ +package check + +import ( + "fmt" + "strings" + "time" +) + +// TestName returns the current test name in the form "SuiteName.TestName" +func (c *C) TestName() string { + return c.testName +} + +// ----------------------------------------------------------------------- +// Basic succeeding/failing logic. + +// Failed returns whether the currently running test has already failed. +func (c *C) Failed() bool { + return c.status() == failedSt +} + +// Fail marks the currently running test as failed. +// +// Something ought to have been previously logged so the developer can tell +// what went wrong. The higher level helper functions will fail the test +// and do the logging properly. +func (c *C) Fail() { + c.setStatus(failedSt) +} + +// FailNow marks the currently running test as failed and stops running it. +// Something ought to have been previously logged so the developer can tell +// what went wrong. The higher level helper functions will fail the test +// and do the logging properly. +func (c *C) FailNow() { + c.Fail() + c.stopNow() +} + +// Succeed marks the currently running test as succeeded, undoing any +// previous failures. +func (c *C) Succeed() { + c.setStatus(succeededSt) +} + +// SucceedNow marks the currently running test as succeeded, undoing any +// previous failures, and stops running the test. +func (c *C) SucceedNow() { + c.Succeed() + c.stopNow() +} + +// ExpectFailure informs that the running test is knowingly broken for +// the provided reason. If the test does not fail, an error will be reported +// to raise attention to this fact. This method is useful to temporarily +// disable tests which cover well known problems until a better time to +// fix the problem is found, without forgetting about the fact that a +// failure still exists. +func (c *C) ExpectFailure(reason string) { + if reason == "" { + panic("Missing reason why the test is expected to fail") + } + c.mustFail = true + c.reason = reason +} + +// Skip skips the running test for the provided reason. If run from within +// SetUpTest, the individual test being set up will be skipped, and if run +// from within SetUpSuite, the whole suite is skipped. +func (c *C) Skip(reason string) { + if reason == "" { + panic("Missing reason why the test is being skipped") + } + c.reason = reason + c.setStatus(skippedSt) + c.stopNow() +} + +// Parallel will mark the test run parallel within a test suite. +func (c *C) Parallel() { + c.parallel <- c +} + +// ----------------------------------------------------------------------- +// Basic logging. + +// GetTestLog returns the current test error output. +func (c *C) GetTestLog() string { + return c.logb.String() +} + +// Log logs some information into the test error output. +// The provided arguments are assembled together into a string with fmt.Sprint. +func (c *C) Log(args ...interface{}) { + c.log(args...) +} + +// Log logs some information into the test error output. +// The provided arguments are assembled together into a string with fmt.Sprintf. +func (c *C) Logf(format string, args ...interface{}) { + c.logf(format, args...) +} + +// Output enables *C to be used as a logger in functions that require only +// the minimum interface of *log.Logger. +func (c *C) Output(calldepth int, s string) error { + d := time.Now().Sub(c.startTime) + msec := d / time.Millisecond + sec := d / time.Second + min := d / time.Minute + + c.Logf("[LOG] %d:%02d.%03d %s", min, sec%60, msec%1000, s) + return nil +} + +// Error logs an error into the test error output and marks the test as failed. +// The provided arguments are assembled together into a string with fmt.Sprint. +func (c *C) Error(args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) + c.logNewLine() + c.Fail() +} + +// Errorf logs an error into the test error output and marks the test as failed. +// The provided arguments are assembled together into a string with fmt.Sprintf. +func (c *C) Errorf(format string, args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprintf("Error: "+format, args...)) + c.logNewLine() + c.Fail() +} + +// Fatal logs an error into the test error output, marks the test as failed, and +// stops the test execution. The provided arguments are assembled together into +// a string with fmt.Sprint. +func (c *C) Fatal(args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) + c.logNewLine() + c.FailNow() +} + +// Fatlaf logs an error into the test error output, marks the test as failed, and +// stops the test execution. The provided arguments are assembled together into +// a string with fmt.Sprintf. +func (c *C) Fatalf(format string, args ...interface{}) { + c.logCaller(1) + c.logString(fmt.Sprint("Error: ", fmt.Sprintf(format, args...))) + c.logNewLine() + c.FailNow() +} + +// ----------------------------------------------------------------------- +// Generic checks and assertions based on checkers. + +// Check verifies if the first value matches the expected value according +// to the provided checker. If they do not match, an error is logged, the +// test is marked as failed, and the test execution continues. +// +// Some checkers may not need the expected argument (e.g. IsNil). +// +// Extra arguments provided to the function are logged next to the reported +// problem when the matching fails. +func (c *C) Check(obtained interface{}, checker Checker, args ...interface{}) bool { + return c.internalCheck("Check", obtained, checker, args...) +} + +// Assert ensures that the first value matches the expected value according +// to the provided checker. If they do not match, an error is logged, the +// test is marked as failed, and the test execution stops. +// +// Some checkers may not need the expected argument (e.g. IsNil). +// +// Extra arguments provided to the function are logged next to the reported +// problem when the matching fails. +func (c *C) Assert(obtained interface{}, checker Checker, args ...interface{}) { + if !c.internalCheck("Assert", obtained, checker, args...) { + c.stopNow() + } +} + +func (c *C) internalCheck(funcName string, obtained interface{}, checker Checker, args ...interface{}) bool { + if checker == nil { + c.logCaller(2) + c.logString(fmt.Sprintf("%s(obtained, nil!?, ...):", funcName)) + c.logString("Oops.. you've provided a nil checker!") + c.logNewLine() + c.Fail() + return false + } + + // If the last argument is a bug info, extract it out. + var comment CommentInterface + if len(args) > 0 { + if c, ok := args[len(args)-1].(CommentInterface); ok { + comment = c + args = args[:len(args)-1] + } + } + + params := append([]interface{}{obtained}, args...) + info := checker.Info() + + if len(params) != len(info.Params) { + names := append([]string{info.Params[0], info.Name}, info.Params[1:]...) + c.logCaller(2) + c.logString(fmt.Sprintf("%s(%s):", funcName, strings.Join(names, ", "))) + c.logString(fmt.Sprintf("Wrong number of parameters for %s: want %d, got %d", info.Name, len(names), len(params)+1)) + c.logNewLine() + c.Fail() + return false + } + + // Copy since it may be mutated by Check. + names := append([]string{}, info.Params...) + + // Do the actual check. + result, error := checker.Check(params, names) + if !result || error != "" { + c.logCaller(2) + for i := 0; i != len(params); i++ { + c.logValue(names[i], params[i]) + } + if comment != nil { + c.logString(comment.CheckCommentString()) + } + if error != "" { + c.logString(error) + } + c.logNewLine() + c.Fail() + return false + } + return true +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/printer.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/printer.go new file mode 100644 index 0000000..e0f7557 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/printer.go @@ -0,0 +1,168 @@ +package check + +import ( + "bytes" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "os" +) + +func indent(s, with string) (r string) { + eol := true + for i := 0; i != len(s); i++ { + c := s[i] + switch { + case eol && c == '\n' || c == '\r': + case c == '\n' || c == '\r': + eol = true + case eol: + eol = false + s = s[:i] + with + s[i:] + i += len(with) + } + } + return s +} + +func printLine(filename string, line int) (string, error) { + fset := token.NewFileSet() + file, err := os.Open(filename) + if err != nil { + return "", err + } + fnode, err := parser.ParseFile(fset, filename, file, parser.ParseComments) + if err != nil { + return "", err + } + config := &printer.Config{Mode: printer.UseSpaces, Tabwidth: 4} + lp := &linePrinter{fset: fset, fnode: fnode, line: line, config: config} + ast.Walk(lp, fnode) + result := lp.output.Bytes() + // Comments leave \n at the end. + n := len(result) + for n > 0 && result[n-1] == '\n' { + n-- + } + return string(result[:n]), nil +} + +type linePrinter struct { + config *printer.Config + fset *token.FileSet + fnode *ast.File + line int + output bytes.Buffer + stmt ast.Stmt +} + +func (lp *linePrinter) emit() bool { + if lp.stmt != nil { + lp.trim(lp.stmt) + lp.printWithComments(lp.stmt) + lp.stmt = nil + return true + } + return false +} + +func (lp *linePrinter) printWithComments(n ast.Node) { + nfirst := lp.fset.Position(n.Pos()).Line + nlast := lp.fset.Position(n.End()).Line + for _, g := range lp.fnode.Comments { + cfirst := lp.fset.Position(g.Pos()).Line + clast := lp.fset.Position(g.End()).Line + if clast == nfirst-1 && lp.fset.Position(n.Pos()).Column == lp.fset.Position(g.Pos()).Column { + for _, c := range g.List { + lp.output.WriteString(c.Text) + lp.output.WriteByte('\n') + } + } + if cfirst >= nfirst && cfirst <= nlast && n.End() <= g.List[0].Slash { + // The printer will not include the comment if it starts past + // the node itself. Trick it into printing by overlapping the + // slash with the end of the statement. + g.List[0].Slash = n.End() - 1 + } + } + node := &printer.CommentedNode{n, lp.fnode.Comments} + lp.config.Fprint(&lp.output, lp.fset, node) +} + +func (lp *linePrinter) Visit(n ast.Node) (w ast.Visitor) { + if n == nil { + if lp.output.Len() == 0 { + lp.emit() + } + return nil + } + first := lp.fset.Position(n.Pos()).Line + last := lp.fset.Position(n.End()).Line + if first <= lp.line && last >= lp.line { + // Print the innermost statement containing the line. + if stmt, ok := n.(ast.Stmt); ok { + if _, ok := n.(*ast.BlockStmt); !ok { + lp.stmt = stmt + } + } + if first == lp.line && lp.emit() { + return nil + } + return lp + } + return nil +} + +func (lp *linePrinter) trim(n ast.Node) bool { + stmt, ok := n.(ast.Stmt) + if !ok { + return true + } + line := lp.fset.Position(n.Pos()).Line + if line != lp.line { + return false + } + switch stmt := stmt.(type) { + case *ast.IfStmt: + stmt.Body = lp.trimBlock(stmt.Body) + case *ast.SwitchStmt: + stmt.Body = lp.trimBlock(stmt.Body) + case *ast.TypeSwitchStmt: + stmt.Body = lp.trimBlock(stmt.Body) + case *ast.CaseClause: + stmt.Body = lp.trimList(stmt.Body) + case *ast.CommClause: + stmt.Body = lp.trimList(stmt.Body) + case *ast.BlockStmt: + stmt.List = lp.trimList(stmt.List) + } + return true +} + +func (lp *linePrinter) trimBlock(stmt *ast.BlockStmt) *ast.BlockStmt { + if !lp.trim(stmt) { + return lp.emptyBlock(stmt) + } + stmt.Rbrace = stmt.Lbrace + return stmt +} + +func (lp *linePrinter) trimList(stmts []ast.Stmt) []ast.Stmt { + for i := 0; i != len(stmts); i++ { + if !lp.trim(stmts[i]) { + stmts[i] = lp.emptyStmt(stmts[i]) + break + } + } + return stmts +} + +func (lp *linePrinter) emptyStmt(n ast.Node) *ast.ExprStmt { + return &ast.ExprStmt{&ast.Ellipsis{n.Pos(), nil}} +} + +func (lp *linePrinter) emptyBlock(n ast.Node) *ast.BlockStmt { + p := n.Pos() + return &ast.BlockStmt{p, []ast.Stmt{lp.emptyStmt(n)}, p} +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/run.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/run.go new file mode 100644 index 0000000..afa631f --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/pingcap/check/run.go @@ -0,0 +1,179 @@ +package check + +import ( + "bufio" + "flag" + "fmt" + "os" + "testing" + "time" +) + +// ----------------------------------------------------------------------- +// Test suite registry. + +var allSuites []interface{} + +// Suite registers the given value as a test suite to be run. Any methods +// starting with the Test prefix in the given value will be considered as +// a test method. +func Suite(suite interface{}) interface{} { + allSuites = append(allSuites, suite) + return suite +} + +// ----------------------------------------------------------------------- +// Public running interface. + +var ( + oldFilterFlag = flag.String("gocheck.f", "", "Regular expression selecting which tests and/or suites to run") + oldVerboseFlag = flag.Bool("gocheck.v", false, "Verbose mode") + oldStreamFlag = flag.Bool("gocheck.vv", false, "Super verbose mode (disables output caching)") + oldBenchFlag = flag.Bool("gocheck.b", false, "Run benchmarks") + oldBenchTime = flag.Duration("gocheck.btime", 1*time.Second, "approximate run time for each benchmark") + oldListFlag = flag.Bool("gocheck.list", false, "List the names of all tests that will be run") + oldWorkFlag = flag.Bool("gocheck.work", false, "Display and do not remove the test working directory") + + newFilterFlag = flag.String("check.f", "", "Regular expression selecting which tests and/or suites to run") + newVerboseFlag = flag.Bool("check.v", false, "Verbose mode") + newStreamFlag = flag.Bool("check.vv", false, "Super verbose mode (disables output caching)") + newBenchFlag = flag.Bool("check.b", false, "Run benchmarks") + newBenchTime = flag.Duration("check.btime", 1*time.Second, "approximate run time for each benchmark") + newBenchMem = flag.Bool("check.bmem", false, "Report memory benchmarks") + newListFlag = flag.Bool("check.list", false, "List the names of all tests that will be run") + newWorkFlag = flag.Bool("check.work", false, "Display and do not remove the test working directory") + newExcludeFlag = flag.String("check.exclude", "", "Regular expression to exclude tests to run") +) + +var CustomVerboseFlag bool + +// TestingT runs all test suites registered with the Suite function, +// printing results to stdout, and reporting any failures back to +// the "testing" package. +func TestingT(testingT *testing.T) { + benchTime := *newBenchTime + if benchTime == 1*time.Second { + benchTime = *oldBenchTime + } + conf := &RunConf{ + Filter: *oldFilterFlag + *newFilterFlag, + Verbose: *oldVerboseFlag || *newVerboseFlag || CustomVerboseFlag, + Stream: *oldStreamFlag || *newStreamFlag, + Benchmark: *oldBenchFlag || *newBenchFlag, + BenchmarkTime: benchTime, + BenchmarkMem: *newBenchMem, + KeepWorkDir: *oldWorkFlag || *newWorkFlag, + Exclude: *newExcludeFlag, + } + if *oldListFlag || *newListFlag { + w := bufio.NewWriter(os.Stdout) + for _, name := range ListAll(conf) { + fmt.Fprintln(w, name) + } + w.Flush() + return + } + result := RunAll(conf) + println(result.String()) + if !result.Passed() { + testingT.Fail() + } +} + +// RunAll runs all test suites registered with the Suite function, using the +// provided run configuration. +func RunAll(runConf *RunConf) *Result { + result := Result{} + for _, suite := range allSuites { + result.Add(Run(suite, runConf)) + } + return &result +} + +// Run runs the provided test suite using the provided run configuration. +func Run(suite interface{}, runConf *RunConf) *Result { + runner := newSuiteRunner(suite, runConf) + return runner.run() +} + +// ListAll returns the names of all the test functions registered with the +// Suite function that will be run with the provided run configuration. +func ListAll(runConf *RunConf) []string { + var names []string + for _, suite := range allSuites { + names = append(names, List(suite, runConf)...) + } + return names +} + +// List returns the names of the test functions in the given +// suite that will be run with the provided run configuration. +func List(suite interface{}, runConf *RunConf) []string { + var names []string + runner := newSuiteRunner(suite, runConf) + for _, t := range runner.tests { + names = append(names, t.String()) + } + return names +} + +// ----------------------------------------------------------------------- +// Result methods. + +func (r *Result) Add(other *Result) { + r.Succeeded += other.Succeeded + r.Skipped += other.Skipped + r.Failed += other.Failed + r.Panicked += other.Panicked + r.FixturePanicked += other.FixturePanicked + r.ExpectedFailures += other.ExpectedFailures + r.Missed += other.Missed + if r.WorkDir != "" && other.WorkDir != "" { + r.WorkDir += ":" + other.WorkDir + } else if other.WorkDir != "" { + r.WorkDir = other.WorkDir + } +} + +func (r *Result) Passed() bool { + return (r.Failed == 0 && r.Panicked == 0 && + r.FixturePanicked == 0 && r.Missed == 0 && + r.RunError == nil) +} + +func (r *Result) String() string { + if r.RunError != nil { + return "ERROR: " + r.RunError.Error() + } + + var value string + if r.Failed == 0 && r.Panicked == 0 && r.FixturePanicked == 0 && + r.Missed == 0 { + value = "OK: " + } else { + value = "OOPS: " + } + value += fmt.Sprintf("%d passed", r.Succeeded) + if r.Skipped != 0 { + value += fmt.Sprintf(", %d skipped", r.Skipped) + } + if r.ExpectedFailures != 0 { + value += fmt.Sprintf(", %d expected failures", r.ExpectedFailures) + } + if r.Failed != 0 { + value += fmt.Sprintf(", %d FAILED", r.Failed) + } + if r.Panicked != 0 { + value += fmt.Sprintf(", %d PANICKED", r.Panicked) + } + if r.FixturePanicked != 0 { + value += fmt.Sprintf(", %d FIXTURE-PANICKED", r.FixturePanicked) + } + if r.Missed != 0 { + value += fmt.Sprintf(", %d MISSED", r.Missed) + } + if r.WorkDir != "" { + value += "\nWORK=" + r.WorkDir + } + return value +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/LICENSE new file mode 100644 index 0000000..926d549 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/LICENSE @@ -0,0 +1,20 @@ +Copyright (C) 2013-2018 by Maxim Bublis + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/codec.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/codec.go new file mode 100644 index 0000000..656892c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/codec.go @@ -0,0 +1,206 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package uuid + +import ( + "bytes" + "encoding/hex" + "fmt" +) + +// FromBytes returns UUID converted from raw byte slice input. +// It will return error if the slice isn't 16 bytes long. +func FromBytes(input []byte) (u UUID, err error) { + err = u.UnmarshalBinary(input) + return +} + +// FromBytesOrNil returns UUID converted from raw byte slice input. +// Same behavior as FromBytes, but returns a Nil UUID on error. +func FromBytesOrNil(input []byte) UUID { + uuid, err := FromBytes(input) + if err != nil { + return Nil + } + return uuid +} + +// FromString returns UUID parsed from string input. +// Input is expected in a form accepted by UnmarshalText. +func FromString(input string) (u UUID, err error) { + err = u.UnmarshalText([]byte(input)) + return +} + +// FromStringOrNil returns UUID parsed from string input. +// Same behavior as FromString, but returns a Nil UUID on error. +func FromStringOrNil(input string) UUID { + uuid, err := FromString(input) + if err != nil { + return Nil + } + return uuid +} + +// MarshalText implements the encoding.TextMarshaler interface. +// The encoding is the same as returned by String. +func (u UUID) MarshalText() (text []byte, err error) { + text = []byte(u.String()) + return +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface. +// Following formats are supported: +// "6ba7b810-9dad-11d1-80b4-00c04fd430c8", +// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}", +// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" +// "6ba7b8109dad11d180b400c04fd430c8" +// ABNF for supported UUID text representation follows: +// uuid := canonical | hashlike | braced | urn +// plain := canonical | hashlike +// canonical := 4hexoct '-' 2hexoct '-' 2hexoct '-' 6hexoct +// hashlike := 12hexoct +// braced := '{' plain '}' +// urn := URN ':' UUID-NID ':' plain +// URN := 'urn' +// UUID-NID := 'uuid' +// 12hexoct := 6hexoct 6hexoct +// 6hexoct := 4hexoct 2hexoct +// 4hexoct := 2hexoct 2hexoct +// 2hexoct := hexoct hexoct +// hexoct := hexdig hexdig +// hexdig := '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | +// 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | +// 'A' | 'B' | 'C' | 'D' | 'E' | 'F' +func (u *UUID) UnmarshalText(text []byte) (err error) { + switch len(text) { + case 32: + return u.decodeHashLike(text) + case 36: + return u.decodeCanonical(text) + case 38: + return u.decodeBraced(text) + case 41: + fallthrough + case 45: + return u.decodeURN(text) + default: + return fmt.Errorf("uuid: incorrect UUID length: %s", text) + } +} + +// decodeCanonical decodes UUID string in format +// "6ba7b810-9dad-11d1-80b4-00c04fd430c8". +func (u *UUID) decodeCanonical(t []byte) (err error) { + if t[8] != '-' || t[13] != '-' || t[18] != '-' || t[23] != '-' { + return fmt.Errorf("uuid: incorrect UUID format %s", t) + } + + src := t[:] + dst := u[:] + + for i, byteGroup := range byteGroups { + if i > 0 { + src = src[1:] // skip dash + } + _, err = hex.Decode(dst[:byteGroup/2], src[:byteGroup]) + if err != nil { + return + } + src = src[byteGroup:] + dst = dst[byteGroup/2:] + } + + return +} + +// decodeHashLike decodes UUID string in format +// "6ba7b8109dad11d180b400c04fd430c8". +func (u *UUID) decodeHashLike(t []byte) (err error) { + src := t[:] + dst := u[:] + + if _, err = hex.Decode(dst, src); err != nil { + return err + } + return +} + +// decodeBraced decodes UUID string in format +// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}" or in format +// "{6ba7b8109dad11d180b400c04fd430c8}". +func (u *UUID) decodeBraced(t []byte) (err error) { + l := len(t) + + if t[0] != '{' || t[l-1] != '}' { + return fmt.Errorf("uuid: incorrect UUID format %s", t) + } + + return u.decodePlain(t[1 : l-1]) +} + +// decodeURN decodes UUID string in format +// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in format +// "urn:uuid:6ba7b8109dad11d180b400c04fd430c8". +func (u *UUID) decodeURN(t []byte) (err error) { + total := len(t) + + urn_uuid_prefix := t[:9] + + if !bytes.Equal(urn_uuid_prefix, urnPrefix) { + return fmt.Errorf("uuid: incorrect UUID format: %s", t) + } + + return u.decodePlain(t[9:total]) +} + +// decodePlain decodes UUID string in canonical format +// "6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in hash-like format +// "6ba7b8109dad11d180b400c04fd430c8". +func (u *UUID) decodePlain(t []byte) (err error) { + switch len(t) { + case 32: + return u.decodeHashLike(t) + case 36: + return u.decodeCanonical(t) + default: + return fmt.Errorf("uuid: incorrrect UUID length: %s", t) + } +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +func (u UUID) MarshalBinary() (data []byte, err error) { + data = u.Bytes() + return +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// It will return error if the slice isn't 16 bytes long. +func (u *UUID) UnmarshalBinary(data []byte) (err error) { + if len(data) != Size { + err = fmt.Errorf("uuid: UUID must be exactly 16 bytes long, got %d bytes", len(data)) + return + } + copy(u[:], data) + + return +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/generator.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/generator.go new file mode 100644 index 0000000..3f2f1da --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/generator.go @@ -0,0 +1,239 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package uuid + +import ( + "crypto/md5" + "crypto/rand" + "crypto/sha1" + "encoding/binary" + "hash" + "net" + "os" + "sync" + "time" +) + +// Difference in 100-nanosecond intervals between +// UUID epoch (October 15, 1582) and Unix epoch (January 1, 1970). +const epochStart = 122192928000000000 + +var ( + global = newDefaultGenerator() + + epochFunc = unixTimeFunc + posixUID = uint32(os.Getuid()) + posixGID = uint32(os.Getgid()) +) + +// NewV1 returns UUID based on current timestamp and MAC address. +func NewV1() UUID { + return global.NewV1() +} + +// NewV2 returns DCE Security UUID based on POSIX UID/GID. +func NewV2(domain byte) UUID { + return global.NewV2(domain) +} + +// NewV3 returns UUID based on MD5 hash of namespace UUID and name. +func NewV3(ns UUID, name string) UUID { + return global.NewV3(ns, name) +} + +// NewV4 returns random generated UUID. +func NewV4() UUID { + return global.NewV4() +} + +// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. +func NewV5(ns UUID, name string) UUID { + return global.NewV5(ns, name) +} + +// Generator provides interface for generating UUIDs. +type Generator interface { + NewV1() UUID + NewV2(domain byte) UUID + NewV3(ns UUID, name string) UUID + NewV4() UUID + NewV5(ns UUID, name string) UUID +} + +// Default generator implementation. +type generator struct { + storageOnce sync.Once + storageMutex sync.Mutex + + lastTime uint64 + clockSequence uint16 + hardwareAddr [6]byte +} + +func newDefaultGenerator() Generator { + return &generator{} +} + +// NewV1 returns UUID based on current timestamp and MAC address. +func (g *generator) NewV1() UUID { + u := UUID{} + + timeNow, clockSeq, hardwareAddr := g.getStorage() + + binary.BigEndian.PutUint32(u[0:], uint32(timeNow)) + binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) + binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) + binary.BigEndian.PutUint16(u[8:], clockSeq) + + copy(u[10:], hardwareAddr) + + u.SetVersion(V1) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV2 returns DCE Security UUID based on POSIX UID/GID. +func (g *generator) NewV2(domain byte) UUID { + u := UUID{} + + timeNow, clockSeq, hardwareAddr := g.getStorage() + + switch domain { + case DomainPerson: + binary.BigEndian.PutUint32(u[0:], posixUID) + case DomainGroup: + binary.BigEndian.PutUint32(u[0:], posixGID) + } + + binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) + binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) + binary.BigEndian.PutUint16(u[8:], clockSeq) + u[9] = domain + + copy(u[10:], hardwareAddr) + + u.SetVersion(V2) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV3 returns UUID based on MD5 hash of namespace UUID and name. +func (g *generator) NewV3(ns UUID, name string) UUID { + u := newFromHash(md5.New(), ns, name) + u.SetVersion(V3) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV4 returns random generated UUID. +func (g *generator) NewV4() UUID { + u := UUID{} + g.safeRandom(u[:]) + u.SetVersion(V4) + u.SetVariant(VariantRFC4122) + + return u +} + +// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. +func (g *generator) NewV5(ns UUID, name string) UUID { + u := newFromHash(sha1.New(), ns, name) + u.SetVersion(V5) + u.SetVariant(VariantRFC4122) + + return u +} + +func (g *generator) initStorage() { + g.initClockSequence() + g.initHardwareAddr() +} + +func (g *generator) initClockSequence() { + buf := make([]byte, 2) + g.safeRandom(buf) + g.clockSequence = binary.BigEndian.Uint16(buf) +} + +func (g *generator) initHardwareAddr() { + interfaces, err := net.Interfaces() + if err == nil { + for _, iface := range interfaces { + if len(iface.HardwareAddr) >= 6 { + copy(g.hardwareAddr[:], iface.HardwareAddr) + return + } + } + } + + // Initialize hardwareAddr randomly in case + // of real network interfaces absence + g.safeRandom(g.hardwareAddr[:]) + + // Set multicast bit as recommended in RFC 4122 + g.hardwareAddr[0] |= 0x01 +} + +func (g *generator) safeRandom(dest []byte) { + if _, err := rand.Read(dest); err != nil { + panic(err) + } +} + +// Returns UUID v1/v2 storage state. +// Returns epoch timestamp, clock sequence, and hardware address. +func (g *generator) getStorage() (uint64, uint16, []byte) { + g.storageOnce.Do(g.initStorage) + + g.storageMutex.Lock() + defer g.storageMutex.Unlock() + + timeNow := epochFunc() + // Clock changed backwards since last UUID generation. + // Should increase clock sequence. + if timeNow <= g.lastTime { + g.clockSequence++ + } + g.lastTime = timeNow + + return timeNow, g.clockSequence, g.hardwareAddr[:] +} + +// Returns difference in 100-nanosecond intervals between +// UUID epoch (October 15, 1582) and current time. +// This is default epoch calculation function. +func unixTimeFunc() uint64 { + return epochStart + uint64(time.Now().UnixNano()/100) +} + +// Returns UUID based on hashing of namespace UUID and name. +func newFromHash(h hash.Hash, ns UUID, name string) UUID { + u := UUID{} + h.Write(ns[:]) + h.Write([]byte(name)) + copy(u[:], h.Sum(nil)) + + return u +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/sql.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/sql.go new file mode 100644 index 0000000..56759d3 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/sql.go @@ -0,0 +1,78 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package uuid + +import ( + "database/sql/driver" + "fmt" +) + +// Value implements the driver.Valuer interface. +func (u UUID) Value() (driver.Value, error) { + return u.String(), nil +} + +// Scan implements the sql.Scanner interface. +// A 16-byte slice is handled by UnmarshalBinary, while +// a longer byte slice or a string is handled by UnmarshalText. +func (u *UUID) Scan(src interface{}) error { + switch src := src.(type) { + case []byte: + if len(src) == Size { + return u.UnmarshalBinary(src) + } + return u.UnmarshalText(src) + + case string: + return u.UnmarshalText([]byte(src)) + } + + return fmt.Errorf("uuid: cannot convert %T to UUID", src) +} + +// NullUUID can be used with the standard sql package to represent a +// UUID value that can be NULL in the database +type NullUUID struct { + UUID UUID + Valid bool +} + +// Value implements the driver.Valuer interface. +func (u NullUUID) Value() (driver.Value, error) { + if !u.Valid { + return nil, nil + } + // Delegate to UUID Value function + return u.UUID.Value() +} + +// Scan implements the sql.Scanner interface. +func (u *NullUUID) Scan(src interface{}) error { + if src == nil { + u.UUID, u.Valid = Nil, false + return nil + } + + // Delegate to UUID Scan function + u.Valid = true + return u.UUID.Scan(src) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/uuid.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/uuid.go new file mode 100644 index 0000000..a2b8e2c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/satori/go.uuid/uuid.go @@ -0,0 +1,161 @@ +// Copyright (C) 2013-2018 by Maxim Bublis +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// Package uuid provides implementation of Universally Unique Identifier (UUID). +// Supported versions are 1, 3, 4 and 5 (as specified in RFC 4122) and +// version 2 (as specified in DCE 1.1). +package uuid + +import ( + "bytes" + "encoding/hex" +) + +// Size of a UUID in bytes. +const Size = 16 + +// UUID representation compliant with specification +// described in RFC 4122. +type UUID [Size]byte + +// UUID versions +const ( + _ byte = iota + V1 + V2 + V3 + V4 + V5 +) + +// UUID layout variants. +const ( + VariantNCS byte = iota + VariantRFC4122 + VariantMicrosoft + VariantFuture +) + +// UUID DCE domains. +const ( + DomainPerson = iota + DomainGroup + DomainOrg +) + +// String parse helpers. +var ( + urnPrefix = []byte("urn:uuid:") + byteGroups = []int{8, 4, 4, 4, 12} +) + +// Nil is special form of UUID that is specified to have all +// 128 bits set to zero. +var Nil = UUID{} + +// Predefined namespace UUIDs. +var ( + NamespaceDNS = Must(FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8")) + NamespaceURL = Must(FromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) + NamespaceOID = Must(FromString("6ba7b812-9dad-11d1-80b4-00c04fd430c8")) + NamespaceX500 = Must(FromString("6ba7b814-9dad-11d1-80b4-00c04fd430c8")) +) + +// Equal returns true if u1 and u2 equals, otherwise returns false. +func Equal(u1 UUID, u2 UUID) bool { + return bytes.Equal(u1[:], u2[:]) +} + +// Version returns algorithm version used to generate UUID. +func (u UUID) Version() byte { + return u[6] >> 4 +} + +// Variant returns UUID layout variant. +func (u UUID) Variant() byte { + switch { + case (u[8] >> 7) == 0x00: + return VariantNCS + case (u[8] >> 6) == 0x02: + return VariantRFC4122 + case (u[8] >> 5) == 0x06: + return VariantMicrosoft + case (u[8] >> 5) == 0x07: + fallthrough + default: + return VariantFuture + } +} + +// Bytes returns bytes slice representation of UUID. +func (u UUID) Bytes() []byte { + return u[:] +} + +// Returns canonical string representation of UUID: +// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. +func (u UUID) String() string { + buf := make([]byte, 36) + + hex.Encode(buf[0:8], u[0:4]) + buf[8] = '-' + hex.Encode(buf[9:13], u[4:6]) + buf[13] = '-' + hex.Encode(buf[14:18], u[6:8]) + buf[18] = '-' + hex.Encode(buf[19:23], u[8:10]) + buf[23] = '-' + hex.Encode(buf[24:], u[10:]) + + return string(buf) +} + +// SetVersion sets version bits. +func (u *UUID) SetVersion(v byte) { + u[6] = (u[6] & 0x0f) | (v << 4) +} + +// SetVariant sets variant bits. +func (u *UUID) SetVariant(v byte) { + switch v { + case VariantNCS: + u[8] = (u[8]&(0xff>>1) | (0x00 << 7)) + case VariantRFC4122: + u[8] = (u[8]&(0xff>>2) | (0x02 << 6)) + case VariantMicrosoft: + u[8] = (u[8]&(0xff>>3) | (0x06 << 5)) + case VariantFuture: + fallthrough + default: + u[8] = (u[8]&(0xff>>3) | (0x07 << 5)) + } +} + +// Must is a helper that wraps a call to a function returning (UUID, error) +// and panics if the error is non-nil. It is intended for use in variable +// initializations such as +// var packageUUID = uuid.Must(uuid.FromString("123e4567-e89b-12d3-a456-426655440000")); +func Must(u UUID, err error) UUID { + if err != nil { + panic(err) + } + return u +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/LICENSE new file mode 100644 index 0000000..ad2148a --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/LICENSE @@ -0,0 +1,45 @@ +The MIT License (MIT) + +Copyright (c) 2015 Spring, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +- Based on https://github.com/oguzbilgic/fpd, which has the following license: +""" +The MIT License (MIT) + +Copyright (c) 2013 Oguz Bilgic + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal-go.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal-go.go new file mode 100644 index 0000000..e08a15c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal-go.go @@ -0,0 +1,414 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Multiprecision decimal numbers. +// For floating-point formatting only; not general purpose. +// Only operations are assign and (binary) left/right shift. +// Can do binary floating point in multiprecision decimal precisely +// because 2 divides 10; cannot do decimal floating point +// in multiprecision binary precisely. +package decimal + +type decimal struct { + d [800]byte // digits, big-endian representation + nd int // number of digits used + dp int // decimal point + neg bool // negative flag + trunc bool // discarded nonzero digits beyond d[:nd] +} + +func (a *decimal) String() string { + n := 10 + a.nd + if a.dp > 0 { + n += a.dp + } + if a.dp < 0 { + n += -a.dp + } + + buf := make([]byte, n) + w := 0 + switch { + case a.nd == 0: + return "0" + + case a.dp <= 0: + // zeros fill space between decimal point and digits + buf[w] = '0' + w++ + buf[w] = '.' + w++ + w += digitZero(buf[w : w+-a.dp]) + w += copy(buf[w:], a.d[0:a.nd]) + + case a.dp < a.nd: + // decimal point in middle of digits + w += copy(buf[w:], a.d[0:a.dp]) + buf[w] = '.' + w++ + w += copy(buf[w:], a.d[a.dp:a.nd]) + + default: + // zeros fill space between digits and decimal point + w += copy(buf[w:], a.d[0:a.nd]) + w += digitZero(buf[w : w+a.dp-a.nd]) + } + return string(buf[0:w]) +} + +func digitZero(dst []byte) int { + for i := range dst { + dst[i] = '0' + } + return len(dst) +} + +// trim trailing zeros from number. +// (They are meaningless; the decimal point is tracked +// independent of the number of digits.) +func trim(a *decimal) { + for a.nd > 0 && a.d[a.nd-1] == '0' { + a.nd-- + } + if a.nd == 0 { + a.dp = 0 + } +} + +// Assign v to a. +func (a *decimal) Assign(v uint64) { + var buf [24]byte + + // Write reversed decimal in buf. + n := 0 + for v > 0 { + v1 := v / 10 + v -= 10 * v1 + buf[n] = byte(v + '0') + n++ + v = v1 + } + + // Reverse again to produce forward decimal in a.d. + a.nd = 0 + for n--; n >= 0; n-- { + a.d[a.nd] = buf[n] + a.nd++ + } + a.dp = a.nd + trim(a) +} + +// Maximum shift that we can do in one pass without overflow. +// A uint has 32 or 64 bits, and we have to be able to accommodate 9<> 63) +const maxShift = uintSize - 4 + +// Binary shift right (/ 2) by k bits. k <= maxShift to avoid overflow. +func rightShift(a *decimal, k uint) { + r := 0 // read pointer + w := 0 // write pointer + + // Pick up enough leading digits to cover first shift. + var n uint + for ; n>>k == 0; r++ { + if r >= a.nd { + if n == 0 { + // a == 0; shouldn't get here, but handle anyway. + a.nd = 0 + return + } + for n>>k == 0 { + n = n * 10 + r++ + } + break + } + c := uint(a.d[r]) + n = n*10 + c - '0' + } + a.dp -= r - 1 + + var mask uint = (1 << k) - 1 + + // Pick up a digit, put down a digit. + for ; r < a.nd; r++ { + c := uint(a.d[r]) + dig := n >> k + n &= mask + a.d[w] = byte(dig + '0') + w++ + n = n*10 + c - '0' + } + + // Put down extra digits. + for n > 0 { + dig := n >> k + n &= mask + if w < len(a.d) { + a.d[w] = byte(dig + '0') + w++ + } else if dig > 0 { + a.trunc = true + } + n = n * 10 + } + + a.nd = w + trim(a) +} + +// Cheat sheet for left shift: table indexed by shift count giving +// number of new digits that will be introduced by that shift. +// +// For example, leftcheats[4] = {2, "625"}. That means that +// if we are shifting by 4 (multiplying by 16), it will add 2 digits +// when the string prefix is "625" through "999", and one fewer digit +// if the string prefix is "000" through "624". +// +// Credit for this trick goes to Ken. + +type leftCheat struct { + delta int // number of new digits + cutoff string // minus one digit if original < a. +} + +var leftcheats = []leftCheat{ + // Leading digits of 1/2^i = 5^i. + // 5^23 is not an exact 64-bit floating point number, + // so have to use bc for the math. + // Go up to 60 to be large enough for 32bit and 64bit platforms. + /* + seq 60 | sed 's/^/5^/' | bc | + awk 'BEGIN{ print "\t{ 0, \"\" }," } + { + log2 = log(2)/log(10) + printf("\t{ %d, \"%s\" },\t// * %d\n", + int(log2*NR+1), $0, 2**NR) + }' + */ + {0, ""}, + {1, "5"}, // * 2 + {1, "25"}, // * 4 + {1, "125"}, // * 8 + {2, "625"}, // * 16 + {2, "3125"}, // * 32 + {2, "15625"}, // * 64 + {3, "78125"}, // * 128 + {3, "390625"}, // * 256 + {3, "1953125"}, // * 512 + {4, "9765625"}, // * 1024 + {4, "48828125"}, // * 2048 + {4, "244140625"}, // * 4096 + {4, "1220703125"}, // * 8192 + {5, "6103515625"}, // * 16384 + {5, "30517578125"}, // * 32768 + {5, "152587890625"}, // * 65536 + {6, "762939453125"}, // * 131072 + {6, "3814697265625"}, // * 262144 + {6, "19073486328125"}, // * 524288 + {7, "95367431640625"}, // * 1048576 + {7, "476837158203125"}, // * 2097152 + {7, "2384185791015625"}, // * 4194304 + {7, "11920928955078125"}, // * 8388608 + {8, "59604644775390625"}, // * 16777216 + {8, "298023223876953125"}, // * 33554432 + {8, "1490116119384765625"}, // * 67108864 + {9, "7450580596923828125"}, // * 134217728 + {9, "37252902984619140625"}, // * 268435456 + {9, "186264514923095703125"}, // * 536870912 + {10, "931322574615478515625"}, // * 1073741824 + {10, "4656612873077392578125"}, // * 2147483648 + {10, "23283064365386962890625"}, // * 4294967296 + {10, "116415321826934814453125"}, // * 8589934592 + {11, "582076609134674072265625"}, // * 17179869184 + {11, "2910383045673370361328125"}, // * 34359738368 + {11, "14551915228366851806640625"}, // * 68719476736 + {12, "72759576141834259033203125"}, // * 137438953472 + {12, "363797880709171295166015625"}, // * 274877906944 + {12, "1818989403545856475830078125"}, // * 549755813888 + {13, "9094947017729282379150390625"}, // * 1099511627776 + {13, "45474735088646411895751953125"}, // * 2199023255552 + {13, "227373675443232059478759765625"}, // * 4398046511104 + {13, "1136868377216160297393798828125"}, // * 8796093022208 + {14, "5684341886080801486968994140625"}, // * 17592186044416 + {14, "28421709430404007434844970703125"}, // * 35184372088832 + {14, "142108547152020037174224853515625"}, // * 70368744177664 + {15, "710542735760100185871124267578125"}, // * 140737488355328 + {15, "3552713678800500929355621337890625"}, // * 281474976710656 + {15, "17763568394002504646778106689453125"}, // * 562949953421312 + {16, "88817841970012523233890533447265625"}, // * 1125899906842624 + {16, "444089209850062616169452667236328125"}, // * 2251799813685248 + {16, "2220446049250313080847263336181640625"}, // * 4503599627370496 + {16, "11102230246251565404236316680908203125"}, // * 9007199254740992 + {17, "55511151231257827021181583404541015625"}, // * 18014398509481984 + {17, "277555756156289135105907917022705078125"}, // * 36028797018963968 + {17, "1387778780781445675529539585113525390625"}, // * 72057594037927936 + {18, "6938893903907228377647697925567626953125"}, // * 144115188075855872 + {18, "34694469519536141888238489627838134765625"}, // * 288230376151711744 + {18, "173472347597680709441192448139190673828125"}, // * 576460752303423488 + {19, "867361737988403547205962240695953369140625"}, // * 1152921504606846976 +} + +// Is the leading prefix of b lexicographically less than s? +func prefixIsLessThan(b []byte, s string) bool { + for i := 0; i < len(s); i++ { + if i >= len(b) { + return true + } + if b[i] != s[i] { + return b[i] < s[i] + } + } + return false +} + +// Binary shift left (* 2) by k bits. k <= maxShift to avoid overflow. +func leftShift(a *decimal, k uint) { + delta := leftcheats[k].delta + if prefixIsLessThan(a.d[0:a.nd], leftcheats[k].cutoff) { + delta-- + } + + r := a.nd // read index + w := a.nd + delta // write index + + // Pick up a digit, put down a digit. + var n uint + for r--; r >= 0; r-- { + n += (uint(a.d[r]) - '0') << k + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + // Put down extra digits. + for n > 0 { + quo := n / 10 + rem := n - 10*quo + w-- + if w < len(a.d) { + a.d[w] = byte(rem + '0') + } else if rem != 0 { + a.trunc = true + } + n = quo + } + + a.nd += delta + if a.nd >= len(a.d) { + a.nd = len(a.d) + } + a.dp += delta + trim(a) +} + +// Binary shift left (k > 0) or right (k < 0). +func (a *decimal) Shift(k int) { + switch { + case a.nd == 0: + // nothing to do: a == 0 + case k > 0: + for k > maxShift { + leftShift(a, maxShift) + k -= maxShift + } + leftShift(a, uint(k)) + case k < 0: + for k < -maxShift { + rightShift(a, maxShift) + k += maxShift + } + rightShift(a, uint(-k)) + } +} + +// If we chop a at nd digits, should we round up? +func shouldRoundUp(a *decimal, nd int) bool { + if nd < 0 || nd >= a.nd { + return false + } + if a.d[nd] == '5' && nd+1 == a.nd { // exactly halfway - round to even + // if we truncated, a little higher than what's recorded - always round up + if a.trunc { + return true + } + return nd > 0 && (a.d[nd-1]-'0')%2 != 0 + } + // not halfway - digit tells all + return a.d[nd] >= '5' +} + +// Round a to nd digits (or fewer). +// If nd is zero, it means we're rounding +// just to the left of the digits, as in +// 0.09 -> 0.1. +func (a *decimal) Round(nd int) { + if nd < 0 || nd >= a.nd { + return + } + if shouldRoundUp(a, nd) { + a.RoundUp(nd) + } else { + a.RoundDown(nd) + } +} + +// Round a down to nd digits (or fewer). +func (a *decimal) RoundDown(nd int) { + if nd < 0 || nd >= a.nd { + return + } + a.nd = nd + trim(a) +} + +// Round a up to nd digits (or fewer). +func (a *decimal) RoundUp(nd int) { + if nd < 0 || nd >= a.nd { + return + } + + // round up + for i := nd - 1; i >= 0; i-- { + c := a.d[i] + if c < '9' { // can stop after this digit + a.d[i]++ + a.nd = i + 1 + return + } + } + + // Number is all 9s. + // Change to single 1 with adjusted decimal point. + a.d[0] = '1' + a.nd = 1 + a.dp++ +} + +// Extract integer part, rounded appropriately. +// No guarantees about overflow. +func (a *decimal) RoundedInteger() uint64 { + if a.dp > 20 { + return 0xFFFFFFFFFFFFFFFF + } + var i int + n := uint64(0) + for i = 0; i < a.dp && i < a.nd; i++ { + n = n*10 + uint64(a.d[i]-'0') + } + for ; i < a.dp; i++ { + n *= 10 + } + if shouldRoundUp(a, a.dp) { + n++ + } + return n +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal.go new file mode 100644 index 0000000..134ece2 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/decimal.go @@ -0,0 +1,1434 @@ +// Package decimal implements an arbitrary precision fixed-point decimal. +// +// To use as part of a struct: +// +// type Struct struct { +// Number Decimal +// } +// +// The zero-value of a Decimal is 0, as you would expect. +// +// The best way to create a new Decimal is to use decimal.NewFromString, ex: +// +// n, err := decimal.NewFromString("-123.4567") +// n.String() // output: "-123.4567" +// +// NOTE: This can "only" represent numbers with a maximum of 2^31 digits +// after the decimal point. +package decimal + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "math/big" + "strconv" + "strings" +) + +// DivisionPrecision is the number of decimal places in the result when it +// doesn't divide exactly. +// +// Example: +// +// d1 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(3) +// d1.String() // output: "0.6666666666666667" +// d2 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(30000) +// d2.String() // output: "0.0000666666666667" +// d3 := decimal.NewFromFloat(20000).Div(decimal.NewFromFloat(3) +// d3.String() // output: "6666.6666666666666667" +// decimal.DivisionPrecision = 3 +// d4 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(3) +// d4.String() // output: "0.667" +// +var DivisionPrecision = 16 + +// MarshalJSONWithoutQuotes should be set to true if you want the decimal to +// be JSON marshaled as a number, instead of as a string. +// WARNING: this is dangerous for decimals with many digits, since many JSON +// unmarshallers (ex: Javascript's) will unmarshal JSON numbers to IEEE 754 +// double-precision floating point numbers, which means you can potentially +// silently lose precision. +var MarshalJSONWithoutQuotes = false + +// Zero constant, to make computations faster. +var Zero = New(0, 1) + +// fiveDec used in Cash Rounding +var fiveDec = New(5, 0) + +var zeroInt = big.NewInt(0) +var oneInt = big.NewInt(1) +var twoInt = big.NewInt(2) +var fourInt = big.NewInt(4) +var fiveInt = big.NewInt(5) +var tenInt = big.NewInt(10) +var twentyInt = big.NewInt(20) + +// Decimal represents a fixed-point decimal. It is immutable. +// number = value * 10 ^ exp +type Decimal struct { + value *big.Int + + // NOTE(vadim): this must be an int32, because we cast it to float64 during + // calculations. If exp is 64 bit, we might lose precision. + // If we cared about being able to represent every possible decimal, we + // could make exp a *big.Int but it would hurt performance and numbers + // like that are unrealistic. + exp int32 +} + +// New returns a new fixed-point decimal, value * 10 ^ exp. +func New(value int64, exp int32) Decimal { + return Decimal{ + value: big.NewInt(value), + exp: exp, + } +} + +// NewFromBigInt returns a new Decimal from a big.Int, value * 10 ^ exp +func NewFromBigInt(value *big.Int, exp int32) Decimal { + return Decimal{ + value: big.NewInt(0).Set(value), + exp: exp, + } +} + +// NewFromString returns a new Decimal from a string representation. +// +// Example: +// +// d, err := NewFromString("-123.45") +// d2, err := NewFromString(".0001") +// +func NewFromString(value string) (Decimal, error) { + originalInput := value + var intString string + var exp int64 + + // Check if number is using scientific notation + eIndex := strings.IndexAny(value, "Ee") + if eIndex != -1 { + expInt, err := strconv.ParseInt(value[eIndex+1:], 10, 32) + if err != nil { + if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { + return Decimal{}, fmt.Errorf("can't convert %s to decimal: fractional part too long", value) + } + return Decimal{}, fmt.Errorf("can't convert %s to decimal: exponent is not numeric", value) + } + value = value[:eIndex] + exp = expInt + } + + parts := strings.Split(value, ".") + if len(parts) == 1 { + // There is no decimal point, we can just parse the original string as + // an int + intString = value + } else if len(parts) == 2 { + // strip the insignificant digits for more accurate comparisons. + decimalPart := strings.TrimRight(parts[1], "0") + intString = parts[0] + decimalPart + expInt := -len(decimalPart) + exp += int64(expInt) + } else { + return Decimal{}, fmt.Errorf("can't convert %s to decimal: too many .s", value) + } + + dValue := new(big.Int) + _, ok := dValue.SetString(intString, 10) + if !ok { + return Decimal{}, fmt.Errorf("can't convert %s to decimal", value) + } + + if exp < math.MinInt32 || exp > math.MaxInt32 { + // NOTE(vadim): I doubt a string could realistically be this long + return Decimal{}, fmt.Errorf("can't convert %s to decimal: fractional part too long", originalInput) + } + + return Decimal{ + value: dValue, + exp: int32(exp), + }, nil +} + +// RequireFromString returns a new Decimal from a string representation +// or panics if NewFromString would have returned an error. +// +// Example: +// +// d := RequireFromString("-123.45") +// d2 := RequireFromString(".0001") +// +func RequireFromString(value string) Decimal { + dec, err := NewFromString(value) + if err != nil { + panic(err) + } + return dec +} + +// NewFromFloat converts a float64 to Decimal. +// +// The converted number will contain the number of significant digits that can be +// represented in a float with reliable roundtrip. +// This is typically 15 digits, but may be more in some cases. +// See https://www.exploringbinary.com/decimal-precision-of-binary-floating-point-numbers/ for more information. +// +// For slightly faster conversion, use NewFromFloatWithExponent where you can specify the precision in absolute terms. +// +// NOTE: this will panic on NaN, +/-inf +func NewFromFloat(value float64) Decimal { + if value == 0 { + return New(0, 0) + } + return newFromFloat(value, math.Float64bits(value), &float64info) +} + +// NewFromFloat converts a float32 to Decimal. +// +// The converted number will contain the number of significant digits that can be +// represented in a float with reliable roundtrip. +// This is typically 6-8 digits depending on the input. +// See https://www.exploringbinary.com/decimal-precision-of-binary-floating-point-numbers/ for more information. +// +// For slightly faster conversion, use NewFromFloatWithExponent where you can specify the precision in absolute terms. +// +// NOTE: this will panic on NaN, +/-inf +func NewFromFloat32(value float32) Decimal { + if value == 0 { + return New(0, 0) + } + // XOR is workaround for https://github.com/golang/go/issues/26285 + a := math.Float32bits(value) ^ 0x80808080 + return newFromFloat(float64(value), uint64(a)^0x80808080, &float32info) +} + +func newFromFloat(val float64, bits uint64, flt *floatInfo) Decimal { + if math.IsNaN(val) || math.IsInf(val, 0) { + panic(fmt.Sprintf("Cannot create a Decimal from %v", val)) + } + exp := int(bits>>flt.mantbits) & (1<>(flt.expbits+flt.mantbits) != 0 + + roundShortest(&d, mant, exp, flt) + // If less than 19 digits, we can do calculation in an int64. + if d.nd < 19 { + tmp := int64(0) + m := int64(1) + for i := d.nd - 1; i >= 0; i-- { + tmp += m * int64(d.d[i]-'0') + m *= 10 + } + if d.neg { + tmp *= -1 + } + return Decimal{value: big.NewInt(tmp), exp: int32(d.dp) - int32(d.nd)} + } + dValue := new(big.Int) + dValue, ok := dValue.SetString(string(d.d[:d.nd]), 10) + if ok { + return Decimal{value: dValue, exp: int32(d.dp) - int32(d.nd)} + } + + return NewFromFloatWithExponent(val, int32(d.dp)-int32(d.nd)) +} + +// NewFromFloatWithExponent converts a float64 to Decimal, with an arbitrary +// number of fractional digits. +// +// Example: +// +// NewFromFloatWithExponent(123.456, -2).String() // output: "123.46" +// +func NewFromFloatWithExponent(value float64, exp int32) Decimal { + if math.IsNaN(value) || math.IsInf(value, 0) { + panic(fmt.Sprintf("Cannot create a Decimal from %v", value)) + } + + bits := math.Float64bits(value) + mant := bits & (1<<52 - 1) + exp2 := int32((bits >> 52) & (1<<11 - 1)) + sign := bits >> 63 + + if exp2 == 0 { + // specials + if mant == 0 { + return Decimal{} + } else { + // subnormal + exp2++ + } + } else { + // normal + mant |= 1 << 52 + } + + exp2 -= 1023 + 52 + + // normalizing base-2 values + for mant&1 == 0 { + mant = mant >> 1 + exp2++ + } + + // maximum number of fractional base-10 digits to represent 2^N exactly cannot be more than -N if N<0 + if exp < 0 && exp < exp2 { + if exp2 < 0 { + exp = exp2 + } else { + exp = 0 + } + } + + // representing 10^M * 2^N as 5^M * 2^(M+N) + exp2 -= exp + + temp := big.NewInt(1) + dMant := big.NewInt(int64(mant)) + + // applying 5^M + if exp > 0 { + temp = temp.SetInt64(int64(exp)) + temp = temp.Exp(fiveInt, temp, nil) + } else if exp < 0 { + temp = temp.SetInt64(-int64(exp)) + temp = temp.Exp(fiveInt, temp, nil) + dMant = dMant.Mul(dMant, temp) + temp = temp.SetUint64(1) + } + + // applying 2^(M+N) + if exp2 > 0 { + dMant = dMant.Lsh(dMant, uint(exp2)) + } else if exp2 < 0 { + temp = temp.Lsh(temp, uint(-exp2)) + } + + // rounding and downscaling + if exp > 0 || exp2 < 0 { + halfDown := new(big.Int).Rsh(temp, 1) + dMant = dMant.Add(dMant, halfDown) + dMant = dMant.Quo(dMant, temp) + } + + if sign == 1 { + dMant = dMant.Neg(dMant) + } + + return Decimal{ + value: dMant, + exp: exp, + } +} + +// rescale returns a rescaled version of the decimal. Returned +// decimal may be less precise if the given exponent is bigger +// than the initial exponent of the Decimal. +// NOTE: this will truncate, NOT round +// +// Example: +// +// d := New(12345, -4) +// d2 := d.rescale(-1) +// d3 := d2.rescale(-4) +// println(d1) +// println(d2) +// println(d3) +// +// Output: +// +// 1.2345 +// 1.2 +// 1.2000 +// +func (d Decimal) rescale(exp int32) Decimal { + d.ensureInitialized() + // NOTE(vadim): must convert exps to float64 before - to prevent overflow + diff := math.Abs(float64(exp) - float64(d.exp)) + value := new(big.Int).Set(d.value) + + expScale := new(big.Int).Exp(tenInt, big.NewInt(int64(diff)), nil) + if exp > d.exp { + value = value.Quo(value, expScale) + } else if exp < d.exp { + value = value.Mul(value, expScale) + } + + return Decimal{ + value: value, + exp: exp, + } +} + +// Abs returns the absolute value of the decimal. +func (d Decimal) Abs() Decimal { + d.ensureInitialized() + d2Value := new(big.Int).Abs(d.value) + return Decimal{ + value: d2Value, + exp: d.exp, + } +} + +// Add returns d + d2. +func (d Decimal) Add(d2 Decimal) Decimal { + baseScale := min(d.exp, d2.exp) + rd := d.rescale(baseScale) + rd2 := d2.rescale(baseScale) + + d3Value := new(big.Int).Add(rd.value, rd2.value) + return Decimal{ + value: d3Value, + exp: baseScale, + } +} + +// Sub returns d - d2. +func (d Decimal) Sub(d2 Decimal) Decimal { + baseScale := min(d.exp, d2.exp) + rd := d.rescale(baseScale) + rd2 := d2.rescale(baseScale) + + d3Value := new(big.Int).Sub(rd.value, rd2.value) + return Decimal{ + value: d3Value, + exp: baseScale, + } +} + +// Neg returns -d. +func (d Decimal) Neg() Decimal { + d.ensureInitialized() + val := new(big.Int).Neg(d.value) + return Decimal{ + value: val, + exp: d.exp, + } +} + +// Mul returns d * d2. +func (d Decimal) Mul(d2 Decimal) Decimal { + d.ensureInitialized() + d2.ensureInitialized() + + expInt64 := int64(d.exp) + int64(d2.exp) + if expInt64 > math.MaxInt32 || expInt64 < math.MinInt32 { + // NOTE(vadim): better to panic than give incorrect results, as + // Decimals are usually used for money + panic(fmt.Sprintf("exponent %v overflows an int32!", expInt64)) + } + + d3Value := new(big.Int).Mul(d.value, d2.value) + return Decimal{ + value: d3Value, + exp: int32(expInt64), + } +} + +// Shift shifts the decimal in base 10. +// It shifts left when shift is positive and right if shift is negative. +// In simpler terms, the given value for shift is added to the exponent +// of the decimal. +func (d Decimal) Shift(shift int32) Decimal { + d.ensureInitialized() + return Decimal{ + value: new(big.Int).Set(d.value), + exp: d.exp + shift, + } +} + +// Div returns d / d2. If it doesn't divide exactly, the result will have +// DivisionPrecision digits after the decimal point. +func (d Decimal) Div(d2 Decimal) Decimal { + return d.DivRound(d2, int32(DivisionPrecision)) +} + +// QuoRem does divsion with remainder +// d.QuoRem(d2,precision) returns quotient q and remainder r such that +// d = d2 * q + r, q an integer multiple of 10^(-precision) +// 0 <= r < abs(d2) * 10 ^(-precision) if d>=0 +// 0 >= r > -abs(d2) * 10 ^(-precision) if d<0 +// Note that precision<0 is allowed as input. +func (d Decimal) QuoRem(d2 Decimal, precision int32) (Decimal, Decimal) { + d.ensureInitialized() + d2.ensureInitialized() + if d2.value.Sign() == 0 { + panic("decimal division by 0") + } + scale := -precision + e := int64(d.exp - d2.exp - scale) + if e > math.MaxInt32 || e < math.MinInt32 { + panic("overflow in decimal QuoRem") + } + var aa, bb, expo big.Int + var scalerest int32 + // d = a 10^ea + // d2 = b 10^eb + if e < 0 { + aa = *d.value + expo.SetInt64(-e) + bb.Exp(tenInt, &expo, nil) + bb.Mul(d2.value, &bb) + scalerest = d.exp + // now aa = a + // bb = b 10^(scale + eb - ea) + } else { + expo.SetInt64(e) + aa.Exp(tenInt, &expo, nil) + aa.Mul(d.value, &aa) + bb = *d2.value + scalerest = scale + d2.exp + // now aa = a ^ (ea - eb - scale) + // bb = b + } + var q, r big.Int + q.QuoRem(&aa, &bb, &r) + dq := Decimal{value: &q, exp: scale} + dr := Decimal{value: &r, exp: scalerest} + return dq, dr +} + +// DivRound divides and rounds to a given precision +// i.e. to an integer multiple of 10^(-precision) +// for a positive quotient digit 5 is rounded up, away from 0 +// if the quotient is negative then digit 5 is rounded down, away from 0 +// Note that precision<0 is allowed as input. +func (d Decimal) DivRound(d2 Decimal, precision int32) Decimal { + // QuoRem already checks initialization + q, r := d.QuoRem(d2, precision) + // the actual rounding decision is based on comparing r*10^precision and d2/2 + // instead compare 2 r 10 ^precision and d2 + var rv2 big.Int + rv2.Abs(r.value) + rv2.Lsh(&rv2, 1) + // now rv2 = abs(r.value) * 2 + r2 := Decimal{value: &rv2, exp: r.exp + precision} + // r2 is now 2 * r * 10 ^ precision + var c = r2.Cmp(d2.Abs()) + + if c < 0 { + return q + } + + if d.value.Sign()*d2.value.Sign() < 0 { + return q.Sub(New(1, -precision)) + } + + return q.Add(New(1, -precision)) +} + +// Mod returns d % d2. +func (d Decimal) Mod(d2 Decimal) Decimal { + quo := d.Div(d2).Truncate(0) + return d.Sub(d2.Mul(quo)) +} + +// Pow returns d to the power d2 +func (d Decimal) Pow(d2 Decimal) Decimal { + var temp Decimal + if d2.IntPart() == 0 { + return NewFromFloat(1) + } + temp = d.Pow(d2.Div(NewFromFloat(2))) + if d2.IntPart()%2 == 0 { + return temp.Mul(temp) + } + if d2.IntPart() > 0 { + return temp.Mul(temp).Mul(d) + } + return temp.Mul(temp).Div(d) +} + +// Cmp compares the numbers represented by d and d2 and returns: +// +// -1 if d < d2 +// 0 if d == d2 +// +1 if d > d2 +// +func (d Decimal) Cmp(d2 Decimal) int { + d.ensureInitialized() + d2.ensureInitialized() + + if d.exp == d2.exp { + return d.value.Cmp(d2.value) + } + + baseExp := min(d.exp, d2.exp) + rd := d.rescale(baseExp) + rd2 := d2.rescale(baseExp) + + return rd.value.Cmp(rd2.value) +} + +// Equal returns whether the numbers represented by d and d2 are equal. +func (d Decimal) Equal(d2 Decimal) bool { + return d.Cmp(d2) == 0 +} + +// Equals is deprecated, please use Equal method instead +func (d Decimal) Equals(d2 Decimal) bool { + return d.Equal(d2) +} + +// GreaterThan (GT) returns true when d is greater than d2. +func (d Decimal) GreaterThan(d2 Decimal) bool { + return d.Cmp(d2) == 1 +} + +// GreaterThanOrEqual (GTE) returns true when d is greater than or equal to d2. +func (d Decimal) GreaterThanOrEqual(d2 Decimal) bool { + cmp := d.Cmp(d2) + return cmp == 1 || cmp == 0 +} + +// LessThan (LT) returns true when d is less than d2. +func (d Decimal) LessThan(d2 Decimal) bool { + return d.Cmp(d2) == -1 +} + +// LessThanOrEqual (LTE) returns true when d is less than or equal to d2. +func (d Decimal) LessThanOrEqual(d2 Decimal) bool { + cmp := d.Cmp(d2) + return cmp == -1 || cmp == 0 +} + +// Sign returns: +// +// -1 if d < 0 +// 0 if d == 0 +// +1 if d > 0 +// +func (d Decimal) Sign() int { + if d.value == nil { + return 0 + } + return d.value.Sign() +} + +// IsPositive return +// +// true if d > 0 +// false if d == 0 +// false if d < 0 +func (d Decimal) IsPositive() bool { + return d.Sign() == 1 +} + +// IsNegative return +// +// true if d < 0 +// false if d == 0 +// false if d > 0 +func (d Decimal) IsNegative() bool { + return d.Sign() == -1 +} + +// IsZero return +// +// true if d == 0 +// false if d > 0 +// false if d < 0 +func (d Decimal) IsZero() bool { + return d.Sign() == 0 +} + +// Exponent returns the exponent, or scale component of the decimal. +func (d Decimal) Exponent() int32 { + return d.exp +} + +// Coefficient returns the coefficient of the decimal. It is scaled by 10^Exponent() +func (d Decimal) Coefficient() *big.Int { + // we copy the coefficient so that mutating the result does not mutate the + // Decimal. + return big.NewInt(0).Set(d.value) +} + +// IntPart returns the integer component of the decimal. +func (d Decimal) IntPart() int64 { + scaledD := d.rescale(0) + return scaledD.value.Int64() +} + +// Rat returns a rational number representation of the decimal. +func (d Decimal) Rat() *big.Rat { + d.ensureInitialized() + if d.exp <= 0 { + // NOTE(vadim): must negate after casting to prevent int32 overflow + denom := new(big.Int).Exp(tenInt, big.NewInt(-int64(d.exp)), nil) + return new(big.Rat).SetFrac(d.value, denom) + } + + mul := new(big.Int).Exp(tenInt, big.NewInt(int64(d.exp)), nil) + num := new(big.Int).Mul(d.value, mul) + return new(big.Rat).SetFrac(num, oneInt) +} + +// Float64 returns the nearest float64 value for d and a bool indicating +// whether f represents d exactly. +// For more details, see the documentation for big.Rat.Float64 +func (d Decimal) Float64() (f float64, exact bool) { + return d.Rat().Float64() +} + +// String returns the string representation of the decimal +// with the fixed point. +// +// Example: +// +// d := New(-12345, -3) +// println(d.String()) +// +// Output: +// +// -12.345 +// +func (d Decimal) String() string { + return d.string(true) +} + +// StringFixed returns a rounded fixed-point string with places digits after +// the decimal point. +// +// Example: +// +// NewFromFloat(0).StringFixed(2) // output: "0.00" +// NewFromFloat(0).StringFixed(0) // output: "0" +// NewFromFloat(5.45).StringFixed(0) // output: "5" +// NewFromFloat(5.45).StringFixed(1) // output: "5.5" +// NewFromFloat(5.45).StringFixed(2) // output: "5.45" +// NewFromFloat(5.45).StringFixed(3) // output: "5.450" +// NewFromFloat(545).StringFixed(-1) // output: "550" +// +func (d Decimal) StringFixed(places int32) string { + rounded := d.Round(places) + return rounded.string(false) +} + +// StringFixedBank returns a banker rounded fixed-point string with places digits +// after the decimal point. +// +// Example: +// +// NewFromFloat(0).StringFixed(2) // output: "0.00" +// NewFromFloat(0).StringFixed(0) // output: "0" +// NewFromFloat(5.45).StringFixed(0) // output: "5" +// NewFromFloat(5.45).StringFixed(1) // output: "5.4" +// NewFromFloat(5.45).StringFixed(2) // output: "5.45" +// NewFromFloat(5.45).StringFixed(3) // output: "5.450" +// NewFromFloat(545).StringFixed(-1) // output: "550" +// +func (d Decimal) StringFixedBank(places int32) string { + rounded := d.RoundBank(places) + return rounded.string(false) +} + +// StringFixedCash returns a Swedish/Cash rounded fixed-point string. For +// more details see the documentation at function RoundCash. +func (d Decimal) StringFixedCash(interval uint8) string { + rounded := d.RoundCash(interval) + return rounded.string(false) +} + +// Round rounds the decimal to places decimal places. +// If places < 0, it will round the integer part to the nearest 10^(-places). +// +// Example: +// +// NewFromFloat(5.45).Round(1).String() // output: "5.5" +// NewFromFloat(545).Round(-1).String() // output: "550" +// +func (d Decimal) Round(places int32) Decimal { + // truncate to places + 1 + ret := d.rescale(-places - 1) + + // add sign(d) * 0.5 + if ret.value.Sign() < 0 { + ret.value.Sub(ret.value, fiveInt) + } else { + ret.value.Add(ret.value, fiveInt) + } + + // floor for positive numbers, ceil for negative numbers + _, m := ret.value.DivMod(ret.value, tenInt, new(big.Int)) + ret.exp++ + if ret.value.Sign() < 0 && m.Cmp(zeroInt) != 0 { + ret.value.Add(ret.value, oneInt) + } + + return ret +} + +// RoundBank rounds the decimal to places decimal places. +// If the final digit to round is equidistant from the nearest two integers the +// rounded value is taken as the even number +// +// If places < 0, it will round the integer part to the nearest 10^(-places). +// +// Examples: +// +// NewFromFloat(5.45).Round(1).String() // output: "5.4" +// NewFromFloat(545).Round(-1).String() // output: "540" +// NewFromFloat(5.46).Round(1).String() // output: "5.5" +// NewFromFloat(546).Round(-1).String() // output: "550" +// NewFromFloat(5.55).Round(1).String() // output: "5.6" +// NewFromFloat(555).Round(-1).String() // output: "560" +// +func (d Decimal) RoundBank(places int32) Decimal { + + round := d.Round(places) + remainder := d.Sub(round).Abs() + + half := New(5, -places-1) + if remainder.Cmp(half) == 0 && round.value.Bit(0) != 0 { + if round.value.Sign() < 0 { + round.value.Add(round.value, oneInt) + } else { + round.value.Sub(round.value, oneInt) + } + } + + return round +} + +// RoundCash aka Cash/Penny/öre rounding rounds decimal to a specific +// interval. The amount payable for a cash transaction is rounded to the nearest +// multiple of the minimum currency unit available. The following intervals are +// available: 5, 10, 15, 25, 50 and 100; any other number throws a panic. +// 5: 5 cent rounding 3.43 => 3.45 +// 10: 10 cent rounding 3.45 => 3.50 (5 gets rounded up) +// 15: 10 cent rounding 3.45 => 3.40 (5 gets rounded down) +// 25: 25 cent rounding 3.41 => 3.50 +// 50: 50 cent rounding 3.75 => 4.00 +// 100: 100 cent rounding 3.50 => 4.00 +// For more details: https://en.wikipedia.org/wiki/Cash_rounding +func (d Decimal) RoundCash(interval uint8) Decimal { + var iVal *big.Int + switch interval { + case 5: + iVal = twentyInt + case 10: + iVal = tenInt + case 15: + if d.exp < 0 { + // TODO: optimize and reduce allocations + orgExp := d.exp + dOne := New(10^-int64(orgExp), orgExp) + d2 := d + d2.exp = 0 + if d2.Mod(fiveDec).Equal(Zero) { + d2.exp = orgExp + d2 = d2.Sub(dOne) + d = d2 + } + } + iVal = tenInt + case 25: + iVal = fourInt + case 50: + iVal = twoInt + case 100: + iVal = oneInt + default: + panic(fmt.Sprintf("Decimal does not support this Cash rounding interval `%d`. Supported: 5, 10, 15, 25, 50, 100", interval)) + } + dVal := Decimal{ + value: iVal, + } + // TODO: optimize those calculations to reduce the high allocations (~29 allocs). + return d.Mul(dVal).Round(0).Div(dVal).Truncate(2) +} + +// Floor returns the nearest integer value less than or equal to d. +func (d Decimal) Floor() Decimal { + d.ensureInitialized() + + if d.exp >= 0 { + return d + } + + exp := big.NewInt(10) + + // NOTE(vadim): must negate after casting to prevent int32 overflow + exp.Exp(exp, big.NewInt(-int64(d.exp)), nil) + + z := new(big.Int).Div(d.value, exp) + return Decimal{value: z, exp: 0} +} + +// Ceil returns the nearest integer value greater than or equal to d. +func (d Decimal) Ceil() Decimal { + d.ensureInitialized() + + if d.exp >= 0 { + return d + } + + exp := big.NewInt(10) + + // NOTE(vadim): must negate after casting to prevent int32 overflow + exp.Exp(exp, big.NewInt(-int64(d.exp)), nil) + + z, m := new(big.Int).DivMod(d.value, exp, new(big.Int)) + if m.Cmp(zeroInt) != 0 { + z.Add(z, oneInt) + } + return Decimal{value: z, exp: 0} +} + +// Truncate truncates off digits from the number, without rounding. +// +// NOTE: precision is the last digit that will not be truncated (must be >= 0). +// +// Example: +// +// decimal.NewFromString("123.456").Truncate(2).String() // "123.45" +// +func (d Decimal) Truncate(precision int32) Decimal { + d.ensureInitialized() + if precision >= 0 && -precision > d.exp { + return d.rescale(-precision) + } + return d +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (d *Decimal) UnmarshalJSON(decimalBytes []byte) error { + if string(decimalBytes) == "null" { + return nil + } + + str, err := unquoteIfQuoted(decimalBytes) + if err != nil { + return fmt.Errorf("Error decoding string '%s': %s", decimalBytes, err) + } + + decimal, err := NewFromString(str) + *d = decimal + if err != nil { + return fmt.Errorf("Error decoding string '%s': %s", str, err) + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (d Decimal) MarshalJSON() ([]byte, error) { + var str string + if MarshalJSONWithoutQuotes { + str = d.String() + } else { + str = "\"" + d.String() + "\"" + } + return []byte(str), nil +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. As a string representation +// is already used when encoding to text, this method stores that string as []byte +func (d *Decimal) UnmarshalBinary(data []byte) error { + // Extract the exponent + d.exp = int32(binary.BigEndian.Uint32(data[:4])) + + // Extract the value + d.value = new(big.Int) + return d.value.GobDecode(data[4:]) +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +func (d Decimal) MarshalBinary() (data []byte, err error) { + // Write the exponent first since it's a fixed size + v1 := make([]byte, 4) + binary.BigEndian.PutUint32(v1, uint32(d.exp)) + + // Add the value + var v2 []byte + if v2, err = d.value.GobEncode(); err != nil { + return + } + + // Return the byte array + data = append(v1, v2...) + return +} + +// Scan implements the sql.Scanner interface for database deserialization. +func (d *Decimal) Scan(value interface{}) error { + // first try to see if the data is stored in database as a Numeric datatype + switch v := value.(type) { + + case float32: + *d = NewFromFloat(float64(v)) + return nil + + case float64: + // numeric in sqlite3 sends us float64 + *d = NewFromFloat(v) + return nil + + case int64: + // at least in sqlite3 when the value is 0 in db, the data is sent + // to us as an int64 instead of a float64 ... + *d = New(v, 0) + return nil + + default: + // default is trying to interpret value stored as string + str, err := unquoteIfQuoted(v) + if err != nil { + return err + } + *d, err = NewFromString(str) + return err + } +} + +// Value implements the driver.Valuer interface for database serialization. +func (d Decimal) Value() (driver.Value, error) { + return d.String(), nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface for XML +// deserialization. +func (d *Decimal) UnmarshalText(text []byte) error { + str := string(text) + + dec, err := NewFromString(str) + *d = dec + if err != nil { + return fmt.Errorf("Error decoding string '%s': %s", str, err) + } + + return nil +} + +// MarshalText implements the encoding.TextMarshaler interface for XML +// serialization. +func (d Decimal) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +// GobEncode implements the gob.GobEncoder interface for gob serialization. +func (d Decimal) GobEncode() ([]byte, error) { + return d.MarshalBinary() +} + +// GobDecode implements the gob.GobDecoder interface for gob serialization. +func (d *Decimal) GobDecode(data []byte) error { + return d.UnmarshalBinary(data) +} + +// StringScaled first scales the decimal then calls .String() on it. +// NOTE: buggy, unintuitive, and DEPRECATED! Use StringFixed instead. +func (d Decimal) StringScaled(exp int32) string { + return d.rescale(exp).String() +} + +func (d Decimal) string(trimTrailingZeros bool) string { + if d.exp >= 0 { + return d.rescale(0).value.String() + } + + abs := new(big.Int).Abs(d.value) + str := abs.String() + + var intPart, fractionalPart string + + // NOTE(vadim): this cast to int will cause bugs if d.exp == INT_MIN + // and you are on a 32-bit machine. Won't fix this super-edge case. + dExpInt := int(d.exp) + if len(str) > -dExpInt { + intPart = str[:len(str)+dExpInt] + fractionalPart = str[len(str)+dExpInt:] + } else { + intPart = "0" + + num0s := -dExpInt - len(str) + fractionalPart = strings.Repeat("0", num0s) + str + } + + if trimTrailingZeros { + i := len(fractionalPart) - 1 + for ; i >= 0; i-- { + if fractionalPart[i] != '0' { + break + } + } + fractionalPart = fractionalPart[:i+1] + } + + number := intPart + if len(fractionalPart) > 0 { + number += "." + fractionalPart + } + + if d.value.Sign() < 0 { + return "-" + number + } + + return number +} + +func (d *Decimal) ensureInitialized() { + if d.value == nil { + d.value = new(big.Int) + } +} + +// Min returns the smallest Decimal that was passed in the arguments. +// +// To call this function with an array, you must do: +// +// Min(arr[0], arr[1:]...) +// +// This makes it harder to accidentally call Min with 0 arguments. +func Min(first Decimal, rest ...Decimal) Decimal { + ans := first + for _, item := range rest { + if item.Cmp(ans) < 0 { + ans = item + } + } + return ans +} + +// Max returns the largest Decimal that was passed in the arguments. +// +// To call this function with an array, you must do: +// +// Max(arr[0], arr[1:]...) +// +// This makes it harder to accidentally call Max with 0 arguments. +func Max(first Decimal, rest ...Decimal) Decimal { + ans := first + for _, item := range rest { + if item.Cmp(ans) > 0 { + ans = item + } + } + return ans +} + +// Sum returns the combined total of the provided first and rest Decimals +func Sum(first Decimal, rest ...Decimal) Decimal { + total := first + for _, item := range rest { + total = total.Add(item) + } + + return total +} + +// Avg returns the average value of the provided first and rest Decimals +func Avg(first Decimal, rest ...Decimal) Decimal { + count := New(int64(len(rest)+1), 0) + sum := Sum(first, rest...) + return sum.Div(count) +} + +func min(x, y int32) int32 { + if x >= y { + return y + } + return x +} + +func unquoteIfQuoted(value interface{}) (string, error) { + var bytes []byte + + switch v := value.(type) { + case string: + bytes = []byte(v) + case []byte: + bytes = v + default: + return "", fmt.Errorf("Could not convert value '%+v' to byte array of type '%T'", + value, value) + } + + // If the amount is quoted, strip the quotes + if len(bytes) > 2 && bytes[0] == '"' && bytes[len(bytes)-1] == '"' { + bytes = bytes[1 : len(bytes)-1] + } + return string(bytes), nil +} + +// NullDecimal represents a nullable decimal with compatibility for +// scanning null values from the database. +type NullDecimal struct { + Decimal Decimal + Valid bool +} + +// Scan implements the sql.Scanner interface for database deserialization. +func (d *NullDecimal) Scan(value interface{}) error { + if value == nil { + d.Valid = false + return nil + } + d.Valid = true + return d.Decimal.Scan(value) +} + +// Value implements the driver.Valuer interface for database serialization. +func (d NullDecimal) Value() (driver.Value, error) { + if !d.Valid { + return nil, nil + } + return d.Decimal.Value() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (d *NullDecimal) UnmarshalJSON(decimalBytes []byte) error { + if string(decimalBytes) == "null" { + d.Valid = false + return nil + } + d.Valid = true + return d.Decimal.UnmarshalJSON(decimalBytes) +} + +// MarshalJSON implements the json.Marshaler interface. +func (d NullDecimal) MarshalJSON() ([]byte, error) { + if !d.Valid { + return []byte("null"), nil + } + return d.Decimal.MarshalJSON() +} + +// Trig functions + +// Atan returns the arctangent, in radians, of x. +func (x Decimal) Atan() Decimal { + if x.Equal(NewFromFloat(0.0)) { + return x + } + if x.GreaterThan(NewFromFloat(0.0)) { + return x.satan() + } + return x.Neg().satan().Neg() +} + +func (d Decimal) xatan() Decimal { + P0 := NewFromFloat(-8.750608600031904122785e-01) + P1 := NewFromFloat(-1.615753718733365076637e+01) + P2 := NewFromFloat(-7.500855792314704667340e+01) + P3 := NewFromFloat(-1.228866684490136173410e+02) + P4 := NewFromFloat(-6.485021904942025371773e+01) + Q0 := NewFromFloat(2.485846490142306297962e+01) + Q1 := NewFromFloat(1.650270098316988542046e+02) + Q2 := NewFromFloat(4.328810604912902668951e+02) + Q3 := NewFromFloat(4.853903996359136964868e+02) + Q4 := NewFromFloat(1.945506571482613964425e+02) + z := d.Mul(d) + b1 := P0.Mul(z).Add(P1).Mul(z).Add(P2).Mul(z).Add(P3).Mul(z).Add(P4).Mul(z) + b2 := z.Add(Q0).Mul(z).Add(Q1).Mul(z).Add(Q2).Mul(z).Add(Q3).Mul(z).Add(Q4) + z = b1.Div(b2) + z = d.Mul(z).Add(d) + return z +} + +// satan reduces its argument (known to be positive) +// to the range [0, 0.66] and calls xatan. +func (d Decimal) satan() Decimal { + Morebits := NewFromFloat(6.123233995736765886130e-17) // pi/2 = PIO2 + Morebits + Tan3pio8 := NewFromFloat(2.41421356237309504880) // tan(3*pi/8) + pi := NewFromFloat(3.14159265358979323846264338327950288419716939937510582097494459) + + if d.LessThanOrEqual(NewFromFloat(0.66)) { + return d.xatan() + } + if d.GreaterThan(Tan3pio8) { + return pi.Div(NewFromFloat(2.0)).Sub(NewFromFloat(1.0).Div(d).xatan()).Add(Morebits) + } + return pi.Div(NewFromFloat(4.0)).Add((d.Sub(NewFromFloat(1.0)).Div(d.Add(NewFromFloat(1.0)))).xatan()).Add(NewFromFloat(0.5).Mul(Morebits)) +} + +// sin coefficients + var _sin = [...]Decimal{ + NewFromFloat(1.58962301576546568060E-10), // 0x3de5d8fd1fd19ccd + NewFromFloat(-2.50507477628578072866E-8), // 0xbe5ae5e5a9291f5d + NewFromFloat(2.75573136213857245213E-6), // 0x3ec71de3567d48a1 + NewFromFloat(-1.98412698295895385996E-4), // 0xbf2a01a019bfdf03 + NewFromFloat(8.33333333332211858878E-3), // 0x3f8111111110f7d0 + NewFromFloat(-1.66666666666666307295E-1), // 0xbfc5555555555548 + } + +// Sin returns the sine of the radian argument x. + func (d Decimal) Sin() Decimal { + PI4A := NewFromFloat(7.85398125648498535156E-1) // 0x3fe921fb40000000, Pi/4 split into three parts + PI4B := NewFromFloat(3.77489470793079817668E-8) // 0x3e64442d00000000, + PI4C := NewFromFloat(2.69515142907905952645E-15) // 0x3ce8469898cc5170, + M4PI := NewFromFloat(1.273239544735162542821171882678754627704620361328125) // 4/pi + + if d.Equal(NewFromFloat(0.0)) { + return d + } + // make argument positive but save the sign + sign := false + if d.LessThan(NewFromFloat(0.0)) { + d = d.Neg() + sign = true + } + + j := d.Mul(M4PI).IntPart() // integer part of x/(Pi/4), as integer for tests on the phase angle + y := NewFromFloat(float64(j)) // integer part of x/(Pi/4), as float + + // map zeros to origin + if j&1 == 1 { + j++ + y = y.Add(NewFromFloat(1.0)) + } + j &= 7 // octant modulo 2Pi radians (360 degrees) + // reflect in x axis + if j > 3 { + sign = !sign + j -= 4 + } + z := d.Sub(y.Mul(PI4A)).Sub(y.Mul(PI4B)).Sub(y.Mul(PI4C)) // Extended precision modular arithmetic + zz := z.Mul(z) + + if j == 1 || j == 2 { + w := zz.Mul(zz).Mul(_cos[0].Mul(zz).Add(_cos[1]).Mul(zz).Add(_cos[2]).Mul(zz).Add(_cos[3]).Mul(zz).Add(_cos[4]).Mul(zz).Add(_cos[5])) + y = NewFromFloat(1.0).Sub(NewFromFloat(0.5).Mul(zz)).Add(w) + } else { + y = z.Add(z.Mul(zz).Mul(_sin[0].Mul(zz).Add(_sin[1]).Mul(zz).Add(_sin[2]).Mul(zz).Add(_sin[3]).Mul(zz).Add(_sin[4]).Mul(zz).Add(_sin[5]))) + } + if sign { + y = y.Neg() + } + return y + } + + // cos coefficients + var _cos = [...]Decimal{ + NewFromFloat(-1.13585365213876817300E-11), // 0xbda8fa49a0861a9b + NewFromFloat(2.08757008419747316778E-9), // 0x3e21ee9d7b4e3f05 + NewFromFloat(-2.75573141792967388112E-7), // 0xbe927e4f7eac4bc6 + NewFromFloat(2.48015872888517045348E-5), // 0x3efa01a019c844f5 + NewFromFloat(-1.38888888888730564116E-3), // 0xbf56c16c16c14f91 + NewFromFloat(4.16666666666665929218E-2), // 0x3fa555555555554b + } + + // Cos returns the cosine of the radian argument x. + func (d Decimal) Cos() Decimal { + + PI4A := NewFromFloat(7.85398125648498535156E-1) // 0x3fe921fb40000000, Pi/4 split into three parts + PI4B := NewFromFloat(3.77489470793079817668E-8) // 0x3e64442d00000000, + PI4C := NewFromFloat(2.69515142907905952645E-15) // 0x3ce8469898cc5170, + M4PI := NewFromFloat(1.273239544735162542821171882678754627704620361328125) // 4/pi + + // make argument positive + sign := false + if d.LessThan(NewFromFloat(0.0)) { + d = d.Neg() + } + + j := d.Mul(M4PI).IntPart() // integer part of x/(Pi/4), as integer for tests on the phase angle + y := NewFromFloat(float64(j)) // integer part of x/(Pi/4), as float + + // map zeros to origin + if j&1 == 1 { + j++ + y = y.Add(NewFromFloat(1.0)) + } + j &= 7 // octant modulo 2Pi radians (360 degrees) + // reflect in x axis + if j > 3 { + sign = !sign + j -= 4 + } + if j > 1 { + sign = !sign + } + + z := d.Sub(y.Mul(PI4A)).Sub(y.Mul(PI4B)).Sub(y.Mul(PI4C)) // Extended precision modular arithmetic + zz := z.Mul(z) + + if j == 1 || j == 2 { + y = z.Add(z.Mul(zz).Mul(_sin[0].Mul(zz).Add(_sin[1]).Mul(zz).Add(_sin[2]).Mul(zz).Add(_sin[3]).Mul(zz).Add(_sin[4]).Mul(zz).Add(_sin[5]))) + } else { + w := zz.Mul(zz).Mul(_cos[0].Mul(zz).Add(_cos[1]).Mul(zz).Add(_cos[2]).Mul(zz).Add(_cos[3]).Mul(zz).Add(_cos[4]).Mul(zz).Add(_cos[5])) + y = NewFromFloat(1.0).Sub(NewFromFloat(0.5).Mul(zz)).Add(w) + } + if sign { + y = y.Neg() + } + return y + } + + var _tanP = [...]Decimal{ + NewFromFloat(-1.30936939181383777646E+4), // 0xc0c992d8d24f3f38 + NewFromFloat(1.15351664838587416140E+6), // 0x413199eca5fc9ddd + NewFromFloat(-1.79565251976484877988E+7), // 0xc1711fead3299176 + } + var _tanQ = [...]Decimal{ + NewFromFloat(1.00000000000000000000E+0), + NewFromFloat(1.36812963470692954678E+4), //0x40cab8a5eeb36572 + NewFromFloat(-1.32089234440210967447E+6), //0xc13427bc582abc96 + NewFromFloat(2.50083801823357915839E+7), //0x4177d98fc2ead8ef + NewFromFloat(-5.38695755929454629881E+7), //0xc189afe03cbe5a31 + } + + // Tan returns the tangent of the radian argument x. + func (d Decimal) Tan() Decimal { + + PI4A := NewFromFloat(7.85398125648498535156E-1) // 0x3fe921fb40000000, Pi/4 split into three parts + PI4B := NewFromFloat(3.77489470793079817668E-8) // 0x3e64442d00000000, + PI4C := NewFromFloat(2.69515142907905952645E-15) // 0x3ce8469898cc5170, + M4PI := NewFromFloat(1.273239544735162542821171882678754627704620361328125) // 4/pi + + if d.Equal(NewFromFloat(0.0)) { + return d + } + + // make argument positive but save the sign + sign := false + if d.LessThan(NewFromFloat(0.0)) { + d = d.Neg() + sign = true + } + + j := d.Mul(M4PI).IntPart() // integer part of x/(Pi/4), as integer for tests on the phase angle + y := NewFromFloat(float64(j)) // integer part of x/(Pi/4), as float + + // map zeros to origin + if j&1 == 1 { + j++ + y = y.Add(NewFromFloat(1.0)) + } + + z := d.Sub(y.Mul(PI4A)).Sub(y.Mul(PI4B)).Sub(y.Mul(PI4C)) // Extended precision modular arithmetic + zz := z.Mul(z) + + if zz.GreaterThan(NewFromFloat(1e-14)) { + w := zz.Mul(_tanP[0].Mul(zz).Add(_tanP[1]).Mul(zz).Add(_tanP[2])) + x := zz.Add(_tanQ[1]).Mul(zz).Add(_tanQ[2]).Mul(zz).Add(_tanQ[3]).Mul(zz).Add(_tanQ[4]) + y = z.Add(z.Mul(w.Div(x))) + } else { + y = z + } + if j&2 == 2 { + y = NewFromFloat(-1.0).Div(y) + } + if sign { + y = y.Neg() + } + return y + } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/rounding.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/rounding.go new file mode 100644 index 0000000..fdd74ea --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/shopspring/decimal/rounding.go @@ -0,0 +1,118 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Multiprecision decimal numbers. +// For floating-point formatting only; not general purpose. +// Only operations are assign and (binary) left/right shift. +// Can do binary floating point in multiprecision decimal precisely +// because 2 divides 10; cannot do decimal floating point +// in multiprecision binary precisely. +package decimal + +type floatInfo struct { + mantbits uint + expbits uint + bias int +} + +var float32info = floatInfo{23, 8, -127} +var float64info = floatInfo{52, 11, -1023} + +// roundShortest rounds d (= mant * 2^exp) to the shortest number of digits +// that will let the original floating point value be precisely reconstructed. +func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) { + // If mantissa is zero, the number is zero; stop now. + if mant == 0 { + d.nd = 0 + return + } + + // Compute upper and lower such that any decimal number + // between upper and lower (possibly inclusive) + // will round to the original floating point number. + + // We may see at once that the number is already shortest. + // + // Suppose d is not denormal, so that 2^exp <= d < 10^dp. + // The closest shorter number is at least 10^(dp-nd) away. + // The lower/upper bounds computed below are at distance + // at most 2^(exp-mantbits). + // + // So the number is already shortest if 10^(dp-nd) > 2^(exp-mantbits), + // or equivalently log2(10)*(dp-nd) > exp-mantbits. + // It is true if 332/100*(dp-nd) >= exp-mantbits (log2(10) > 3.32). + minexp := flt.bias + 1 // minimum possible exponent + if exp > minexp && 332*(d.dp-d.nd) >= 100*(exp-int(flt.mantbits)) { + // The number is already shortest. + return + } + + // d = mant << (exp - mantbits) + // Next highest floating point number is mant+1 << exp-mantbits. + // Our upper bound is halfway between, mant*2+1 << exp-mantbits-1. + upper := new(decimal) + upper.Assign(mant*2 + 1) + upper.Shift(exp - int(flt.mantbits) - 1) + + // d = mant << (exp - mantbits) + // Next lowest floating point number is mant-1 << exp-mantbits, + // unless mant-1 drops the significant bit and exp is not the minimum exp, + // in which case the next lowest is mant*2-1 << exp-mantbits-1. + // Either way, call it mantlo << explo-mantbits. + // Our lower bound is halfway between, mantlo*2+1 << explo-mantbits-1. + var mantlo uint64 + var explo int + if mant > 1< 0 { + h.fd.Close() + + for i := h.backupCount - 1; i > 0; i-- { + sfn := fmt.Sprintf("%s.%d", h.fileName, i) + dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) + + os.Rename(sfn, dfn) + } + + dfn := fmt.Sprintf("%s.1", h.fileName) + os.Rename(h.fileName, dfn) + + h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + h.curBytes = 0 + f, err := h.fd.Stat() + if err != nil { + return + } + h.curBytes = int(f.Size()) + } +} + +// TimeRotatingFileHandler writes log to a file, +// it will backup current and open a new one, with a period time you sepecified. +// +// refer: http://docs.python.org/2/library/logging.handlers.html. +// same like python TimedRotatingFileHandler. +type TimeRotatingFileHandler struct { + fd *os.File + + baseName string + interval int64 + suffix string + rolloverAt int64 +} + +// TimeRotating way +const ( + WhenSecond = iota + WhenMinute + WhenHour + WhenDay +) + +// NewTimeRotatingFileHandler creates a TimeRotatingFileHandler +func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { + dir := path.Dir(baseName) + os.MkdirAll(dir, 0777) + + h := new(TimeRotatingFileHandler) + + h.baseName = baseName + + switch when { + case WhenSecond: + h.interval = 1 + h.suffix = "2006-01-02_15-04-05" + case WhenMinute: + h.interval = 60 + h.suffix = "2006-01-02_15-04" + case WhenHour: + h.interval = 3600 + h.suffix = "2006-01-02_15" + case WhenDay: + h.interval = 3600 * 24 + h.suffix = "2006-01-02" + default: + return nil, fmt.Errorf("invalid when_rotate: %d", when) + } + + h.interval = h.interval * int64(interval) + + var err error + h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + fInfo, _ := h.fd.Stat() + h.rolloverAt = fInfo.ModTime().Unix() + h.interval + + return h, nil +} + +func (h *TimeRotatingFileHandler) doRollover() { + //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py + now := time.Now() + + if h.rolloverAt <= now.Unix() { + fName := h.baseName + now.Format(h.suffix) + h.fd.Close() + e := os.Rename(h.baseName, fName) + if e != nil { + panic(e) + } + + h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + + h.rolloverAt = time.Now().Unix() + h.interval + } +} + +// Write implements Handler interface +func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { + h.doRollover() + return h.fd.Write(b) +} + +// Close implements Handler interface +func (h *TimeRotatingFileHandler) Close() error { + return h.fd.Close() +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/handler.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/handler.go new file mode 100644 index 0000000..5460f06 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/handler.go @@ -0,0 +1,54 @@ +package log + +import ( + "io" +) + +//Handler writes logs to somewhere +type Handler interface { + Write(p []byte) (n int, err error) + Close() error +} + +// StreamHandler writes logs to a specified io Writer, maybe stdout, stderr, etc... +type StreamHandler struct { + w io.Writer +} + +// NewStreamHandler creates a StreamHandler +func NewStreamHandler(w io.Writer) (*StreamHandler, error) { + h := new(StreamHandler) + + h.w = w + + return h, nil +} + +// Write implements Handler interface +func (h *StreamHandler) Write(b []byte) (n int, err error) { + return h.w.Write(b) +} + +// Close implements Handler interface +func (h *StreamHandler) Close() error { + return nil +} + +// NullHandler does nothing, it discards anything. +type NullHandler struct { +} + +// NewNullHandler creates a NullHandler +func NewNullHandler() (*NullHandler, error) { + return new(NullHandler), nil +} + +// // Write implements Handler interface +func (h *NullHandler) Write(b []byte) (n int, err error) { + return len(b), nil +} + +// Close implements Handler interface +func (h *NullHandler) Close() { + +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/log.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/log.go new file mode 100644 index 0000000..956186d --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/log.go @@ -0,0 +1,137 @@ +package log + +import ( + "fmt" + "os" +) + +var logger = NewDefault(newStdHandler()) + +// SetDefaultLogger changes the global logger +func SetDefaultLogger(l *Logger) { + logger = l +} + +// SetLevel changes the logger level +func SetLevel(level Level) { + logger.SetLevel(level) +} + +// SetLevelByName changes the logger level by name +func SetLevelByName(name string) { + logger.SetLevelByName(name) +} + +// Fatal records the log with fatal level and exits +func Fatal(args ...interface{}) { + logger.Output(2, LevelFatal, fmt.Sprint(args...)) + os.Exit(1) +} + +// Fatalf records the log with fatal level and exits +func Fatalf(format string, args ...interface{}) { + logger.Output(2, LevelFatal, fmt.Sprintf(format, args...)) + os.Exit(1) +} + +// Fatalln records the log with fatal level and exits +func Fatalln(args ...interface{}) { + logger.Output(2, LevelFatal, fmt.Sprintln(args...)) + os.Exit(1) +} + +// Panic records the log with fatal level and panics +func Panic(args ...interface{}) { + msg := fmt.Sprint(args...) + logger.Output(2, LevelError, msg) + panic(msg) +} + +// Panicf records the log with fatal level and panics +func Panicf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + logger.Output(2, LevelError, msg) + panic(msg) +} + +// Panicln records the log with fatal level and panics +func Panicln(args ...interface{}) { + msg := fmt.Sprintln(args...) + logger.Output(2, LevelError, msg) + panic(msg) +} + +// Print records the log with trace level +func Print(args ...interface{}) { + logger.Output(2, LevelTrace, fmt.Sprint(args...)) +} + +// Printf records the log with trace level +func Printf(format string, args ...interface{}) { + logger.Output(2, LevelTrace, fmt.Sprintf(format, args...)) +} + +// Println records the log with trace level +func Println(args ...interface{}) { + logger.Output(2, LevelTrace, fmt.Sprintln(args...)) +} + +// Debug records the log with debug level +func Debug(args ...interface{}) { + logger.Output(2, LevelDebug, fmt.Sprint(args...)) +} + +// Debugf records the log with debug level +func Debugf(format string, args ...interface{}) { + logger.Output(2, LevelDebug, fmt.Sprintf(format, args...)) +} + +// Debugln records the log with debug level +func Debugln(args ...interface{}) { + logger.Output(2, LevelDebug, fmt.Sprintln(args...)) +} + +// Error records the log with error level +func Error(args ...interface{}) { + logger.Output(2, LevelError, fmt.Sprint(args...)) +} + +// Errorf records the log with error level +func Errorf(format string, args ...interface{}) { + logger.Output(2, LevelError, fmt.Sprintf(format, args...)) +} + +// Errorln records the log with error level +func Errorln(args ...interface{}) { + logger.Output(2, LevelError, fmt.Sprintln(args...)) +} + +// Info records the log with info level +func Info(args ...interface{}) { + logger.Output(2, LevelInfo, fmt.Sprint(args...)) +} + +// Infof records the log with info level +func Infof(format string, args ...interface{}) { + logger.Output(2, LevelInfo, fmt.Sprintf(format, args...)) +} + +// Infoln records the log with info level +func Infoln(args ...interface{}) { + logger.Output(2, LevelInfo, fmt.Sprintln(args...)) +} + +// Warn records the log with warn level +func Warn(args ...interface{}) { + logger.Output(2, LevelWarn, fmt.Sprint(args...)) +} + +// Warnf records the log with warn level +func Warnf(format string, args ...interface{}) { + logger.Output(2, LevelWarn, fmt.Sprintf(format, args...)) +} + +// Warnln records the log with warn level +func Warnln(args ...interface{}) { + logger.Output(2, LevelWarn, fmt.Sprintln(args...)) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/logger.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/logger.go new file mode 100644 index 0000000..b2f7ed2 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/log/logger.go @@ -0,0 +1,340 @@ +package log + +import ( + "fmt" + "os" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/siddontang/go-log/loggers" +) + +const ( + timeFormat = "2006/01/02 15:04:05" + maxBufPoolSize = 16 +) + +// Logger flag +const ( + Ltime = 1 << iota // time format "2006/01/02 15:04:05" + Lfile // file.go:123 + Llevel // [Trace|Debug|Info...] +) + +// Level type +type Level int + +// Log level, from low to high, more high means more serious +const ( + LevelTrace Level = iota + LevelDebug + LevelInfo + LevelWarn + LevelError + LevelFatal +) + +// String returns level String +func (l Level) String() string { + switch l { + case LevelTrace: + return "trace" + case LevelDebug: + return "debug" + case LevelInfo: + return "info" + case LevelWarn: + return "warn" + case LevelError: + return "error" + case LevelFatal: + return "fatal" + } + // return default info + return "info" +} + +// Logger is the logger to record log +type Logger struct { + // TODO: support logger.Contextual + loggers.Advanced + + sync.Mutex + + level Level + flag int + + handler Handler + + quit chan struct{} + msg chan []byte + + bufs [][]byte +} + +// New creates a logger with specified handler and flag +func New(handler Handler, flag int) *Logger { + var l = new(Logger) + + l.level = LevelInfo + l.handler = handler + + l.flag = flag + + l.quit = make(chan struct{}) + + l.msg = make(chan []byte, 1024) + + l.bufs = make([][]byte, 0, 16) + + go l.run() + + return l +} + +// NewDefault creates default logger with specified handler and flag: Ltime|Lfile|Llevel +func NewDefault(handler Handler) *Logger { + return New(handler, Ltime|Lfile|Llevel) +} + +func newStdHandler() *StreamHandler { + h, _ := NewStreamHandler(os.Stdout) + return h +} + +func (l *Logger) run() { + for { + select { + case msg := <-l.msg: + l.handler.Write(msg) + l.putBuf(msg) + case <-l.quit: + l.handler.Close() + } + } +} + +func (l *Logger) popBuf() []byte { + l.Lock() + var buf []byte + if len(l.bufs) == 0 { + buf = make([]byte, 0, 1024) + } else { + buf = l.bufs[len(l.bufs)-1] + l.bufs = l.bufs[0 : len(l.bufs)-1] + } + l.Unlock() + + return buf +} + +func (l *Logger) putBuf(buf []byte) { + l.Lock() + if len(l.bufs) < maxBufPoolSize { + buf = buf[0:0] + l.bufs = append(l.bufs, buf) + } + l.Unlock() +} + +// Close closes the logger +func (l *Logger) Close() { + if l.quit == nil { + return + } + + close(l.quit) + l.quit = nil +} + +// SetLevel sets log level, any log level less than it will not log +func (l *Logger) SetLevel(level Level) { + l.level = level +} + +// SetLevelByName sets log level by name +func (l *Logger) SetLevelByName(name string) { + level := LevelInfo + switch strings.ToLower(name) { + case "trace": + level = LevelTrace + case "debug": + level = LevelDebug + case "warn", "warning": + level = LevelWarn + case "error": + level = LevelError + case "fatal": + level = LevelFatal + default: + level = LevelInfo + } + + l.SetLevel(level) +} + +// Output records the log with special callstack depth and log level. +func (l *Logger) Output(callDepth int, level Level, msg string) { + if l.level > level { + return + } + + buf := l.popBuf() + + if l.flag&Ltime > 0 { + now := time.Now().Format(timeFormat) + buf = append(buf, '[') + buf = append(buf, now...) + buf = append(buf, "] "...) + } + + if l.flag&Llevel > 0 { + buf = append(buf, '[') + buf = append(buf, level.String()...) + buf = append(buf, "] "...) + } + + if l.flag&Lfile > 0 { + _, file, line, ok := runtime.Caller(callDepth) + if !ok { + file = "???" + line = 0 + } else { + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + file = file[i+1:] + break + } + } + } + + buf = append(buf, file...) + buf = append(buf, ':') + + buf = strconv.AppendInt(buf, int64(line), 10) + buf = append(buf, ' ') + } + + buf = append(buf, msg...) + if len(msg) == 0 || msg[len(msg)-1] != '\n' { + buf = append(buf, '\n') + } + l.msg <- buf +} + +// Fatal records the log with fatal level and exits +func (l *Logger) Fatal(args ...interface{}) { + l.Output(2, LevelFatal, fmt.Sprint(args...)) + os.Exit(1) +} + +// Fatalf records the log with fatal level and exits +func (l *Logger) Fatalf(format string, args ...interface{}) { + l.Output(2, LevelFatal, fmt.Sprintf(format, args...)) + os.Exit(1) +} + +// Fatalln records the log with fatal level and exits +func (l *Logger) Fatalln(args ...interface{}) { + l.Output(2, LevelFatal, fmt.Sprintln(args...)) + os.Exit(1) +} + +// Panic records the log with fatal level and panics +func (l *Logger) Panic(args ...interface{}) { + msg := fmt.Sprint(args...) + l.Output(2, LevelError, msg) + panic(msg) +} + +// Panicf records the log with fatal level and panics +func (l *Logger) Panicf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + l.Output(2, LevelError, msg) + panic(msg) +} + +// Panicln records the log with fatal level and panics +func (l *Logger) Panicln(args ...interface{}) { + msg := fmt.Sprintln(args...) + l.Output(2, LevelError, msg) + panic(msg) +} + +// Print records the log with trace level +func (l *Logger) Print(args ...interface{}) { + l.Output(2, LevelTrace, fmt.Sprint(args...)) +} + +// Printf records the log with trace level +func (l *Logger) Printf(format string, args ...interface{}) { + l.Output(2, LevelTrace, fmt.Sprintf(format, args...)) +} + +// Println records the log with trace level +func (l *Logger) Println(args ...interface{}) { + l.Output(2, LevelTrace, fmt.Sprintln(args...)) +} + +// Debug records the log with debug level +func (l *Logger) Debug(args ...interface{}) { + l.Output(2, LevelDebug, fmt.Sprint(args...)) +} + +// Debugf records the log with debug level +func (l *Logger) Debugf(format string, args ...interface{}) { + l.Output(2, LevelDebug, fmt.Sprintf(format, args...)) +} + +// Debugln records the log with debug level +func (l *Logger) Debugln(args ...interface{}) { + l.Output(2, LevelDebug, fmt.Sprintln(args...)) +} + +// Error records the log with error level +func (l *Logger) Error(args ...interface{}) { + l.Output(2, LevelError, fmt.Sprint(args...)) +} + +// Errorf records the log with error level +func (l *Logger) Errorf(format string, args ...interface{}) { + l.Output(2, LevelError, fmt.Sprintf(format, args...)) +} + +// Errorln records the log with error level +func (l *Logger) Errorln(args ...interface{}) { + l.Output(2, LevelError, fmt.Sprintln(args...)) +} + +// Info records the log with info level +func (l *Logger) Info(args ...interface{}) { + l.Output(2, LevelInfo, fmt.Sprint(args...)) +} + +// Infof records the log with info level +func (l *Logger) Infof(format string, args ...interface{}) { + l.Output(2, LevelInfo, fmt.Sprintf(format, args...)) +} + +// Infoln records the log with info level +func (l *Logger) Infoln(args ...interface{}) { + l.Output(2, LevelInfo, fmt.Sprintln(args...)) +} + +// Warn records the log with warn level +func (l *Logger) Warn(args ...interface{}) { + l.Output(2, LevelWarn, fmt.Sprint(args...)) +} + +// Warnf records the log with warn level +func (l *Logger) Warnf(format string, args ...interface{}) { + l.Output(2, LevelWarn, fmt.Sprintf(format, args...)) +} + +// Warnln records the log with warn level +func (l *Logger) Warnln(args ...interface{}) { + l.Output(2, LevelWarn, fmt.Sprintln(args...)) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/loggers/loggers.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/loggers/loggers.go new file mode 100644 index 0000000..2723b24 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go-log/loggers/loggers.go @@ -0,0 +1,68 @@ +// MIT License + +// Copyright (c) 2017 Birkir A. Barkarson + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package loggers + +// Standard is the interface used by Go's standard library's log package. +type Standard interface { + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) + Fatalln(args ...interface{}) + + Panic(args ...interface{}) + Panicf(format string, args ...interface{}) + Panicln(args ...interface{}) + + Print(args ...interface{}) + Printf(format string, args ...interface{}) + Println(args ...interface{}) +} + +// Advanced is an interface with commonly used log level methods. +type Advanced interface { + Standard + + Debug(args ...interface{}) + Debugf(format string, args ...interface{}) + Debugln(args ...interface{}) + + Error(args ...interface{}) + Errorf(format string, args ...interface{}) + Errorln(args ...interface{}) + + Info(args ...interface{}) + Infof(format string, args ...interface{}) + Infoln(args ...interface{}) + + Warn(args ...interface{}) + Warnf(format string, args ...interface{}) + Warnln(args ...interface{}) +} + +// Contextual is an interface that allows context addition to a log statement before +// calling the final print (message/level) method. +type Contextual interface { + Advanced + + WithField(key string, value interface{}) Advanced + WithFields(fields ...interface{}) Advanced +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/LICENSE new file mode 100644 index 0000000..80511a0 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 siddontang + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/bson/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/bson/LICENSE new file mode 100644 index 0000000..8903260 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/bson/LICENSE @@ -0,0 +1,25 @@ +BSON library for Go + +Copyright (c) 2010-2012 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/filelock/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/filelock/LICENSE new file mode 100644 index 0000000..fec05ce --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/filelock/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The LevelDB-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/hack/hack.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/hack/hack.go new file mode 100644 index 0000000..74ee83c --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/hack/hack.go @@ -0,0 +1,27 @@ +package hack + +import ( + "reflect" + "unsafe" +) + +// no copy to change slice to string +// use your own risk +func String(b []byte) (s string) { + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pstring.Data = pbytes.Data + pstring.Len = pbytes.Len + return +} + +// no copy to change string to slice +// use your own risk +func Slice(s string) (b []byte) { + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pbytes.Data = pstring.Data + pbytes.Len = pstring.Len + pbytes.Cap = pstring.Len + return +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/snappy/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/snappy/LICENSE new file mode 100644 index 0000000..6050c10 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/snappy/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/atomic.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/atomic.go new file mode 100644 index 0000000..382fc20 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/atomic.go @@ -0,0 +1,146 @@ +// Copyright 2013, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync2 + +import ( + "sync" + "sync/atomic" + "time" +) + +type AtomicInt32 int32 + +func (i *AtomicInt32) Add(n int32) int32 { + return atomic.AddInt32((*int32)(i), n) +} + +func (i *AtomicInt32) Set(n int32) { + atomic.StoreInt32((*int32)(i), n) +} + +func (i *AtomicInt32) Get() int32 { + return atomic.LoadInt32((*int32)(i)) +} + +func (i *AtomicInt32) CompareAndSwap(oldval, newval int32) (swapped bool) { + return atomic.CompareAndSwapInt32((*int32)(i), oldval, newval) +} + +type AtomicUint32 uint32 + +func (i *AtomicUint32) Add(n uint32) uint32 { + return atomic.AddUint32((*uint32)(i), n) +} + +func (i *AtomicUint32) Set(n uint32) { + atomic.StoreUint32((*uint32)(i), n) +} + +func (i *AtomicUint32) Get() uint32 { + return atomic.LoadUint32((*uint32)(i)) +} + +func (i *AtomicUint32) CompareAndSwap(oldval, newval uint32) (swapped bool) { + return atomic.CompareAndSwapUint32((*uint32)(i), oldval, newval) +} + +type AtomicInt64 int64 + +func (i *AtomicInt64) Add(n int64) int64 { + return atomic.AddInt64((*int64)(i), n) +} + +func (i *AtomicInt64) Set(n int64) { + atomic.StoreInt64((*int64)(i), n) +} + +func (i *AtomicInt64) Get() int64 { + return atomic.LoadInt64((*int64)(i)) +} + +func (i *AtomicInt64) CompareAndSwap(oldval, newval int64) (swapped bool) { + return atomic.CompareAndSwapInt64((*int64)(i), oldval, newval) +} + +type AtomicUint64 uint64 + +func (i *AtomicUint64) Add(n uint64) uint64 { + return atomic.AddUint64((*uint64)(i), n) +} + +func (i *AtomicUint64) Set(n uint64) { + atomic.StoreUint64((*uint64)(i), n) +} + +func (i *AtomicUint64) Get() uint64 { + return atomic.LoadUint64((*uint64)(i)) +} + +func (i *AtomicUint64) CompareAndSwap(oldval, newval uint64) (swapped bool) { + return atomic.CompareAndSwapUint64((*uint64)(i), oldval, newval) +} + +type AtomicDuration int64 + +func (d *AtomicDuration) Add(duration time.Duration) time.Duration { + return time.Duration(atomic.AddInt64((*int64)(d), int64(duration))) +} + +func (d *AtomicDuration) Set(duration time.Duration) { + atomic.StoreInt64((*int64)(d), int64(duration)) +} + +func (d *AtomicDuration) Get() time.Duration { + return time.Duration(atomic.LoadInt64((*int64)(d))) +} + +func (d *AtomicDuration) CompareAndSwap(oldval, newval time.Duration) (swapped bool) { + return atomic.CompareAndSwapInt64((*int64)(d), int64(oldval), int64(newval)) +} + +// AtomicString gives you atomic-style APIs for string, but +// it's only a convenience wrapper that uses a mutex. So, it's +// not as efficient as the rest of the atomic types. +type AtomicString struct { + mu sync.Mutex + str string +} + +func (s *AtomicString) Set(str string) { + s.mu.Lock() + s.str = str + s.mu.Unlock() +} + +func (s *AtomicString) Get() string { + s.mu.Lock() + str := s.str + s.mu.Unlock() + return str +} + +func (s *AtomicString) CompareAndSwap(oldval, newval string) (swapped bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.str == oldval { + s.str = newval + return true + } + return false +} + +type AtomicBool int32 + +func (b *AtomicBool) Set(v bool) { + if v { + atomic.StoreInt32((*int32)(b), 1) + } else { + atomic.StoreInt32((*int32)(b), 0) + } +} + +func (b *AtomicBool) Get() bool { + return atomic.LoadInt32((*int32)(b)) == 1 +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/semaphore.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/semaphore.go new file mode 100644 index 0000000..d310da7 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/siddontang/go/sync2/semaphore.go @@ -0,0 +1,65 @@ +package sync2 + +import ( + "sync" + "sync/atomic" + "time" +) + +func NewSemaphore(initialCount int) *Semaphore { + res := &Semaphore{ + counter: int64(initialCount), + } + res.cond.L = &res.lock + return res +} + +type Semaphore struct { + lock sync.Mutex + cond sync.Cond + counter int64 +} + +func (s *Semaphore) Release() { + s.lock.Lock() + s.counter += 1 + if s.counter >= 0 { + s.cond.Signal() + } + s.lock.Unlock() +} + +func (s *Semaphore) Acquire() { + s.lock.Lock() + for s.counter < 1 { + s.cond.Wait() + } + s.counter -= 1 + s.lock.Unlock() +} + +func (s *Semaphore) AcquireTimeout(timeout time.Duration) bool { + done := make(chan bool, 1) + // Gate used to communicate between the threads and decide what the result + // is. If the main thread decides, we have timed out, otherwise we succeed. + decided := new(int32) + go func() { + s.Acquire() + if atomic.SwapInt32(decided, 1) == 0 { + done <- true + } else { + // If we already decided the result, and this thread did not win + s.Release() + } + }() + select { + case <-done: + return true + case <-time.NewTimer(timeout).C: + if atomic.SwapInt32(decided, 1) == 1 { + // The other thread already decided the result + return true + } + return false + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/LICENSE b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql.go b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql.go new file mode 100644 index 0000000..7b27e6b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql.go @@ -0,0 +1,62 @@ +// Copyright 2013 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +/* +Package cloudsql exposes access to Google Cloud SQL databases. + +This package does not work in App Engine "flexible environment". + +This package is intended for MySQL drivers to make App Engine-specific +connections. Applications should use this package through database/sql: +Select a pure Go MySQL driver that supports this package, and use sql.Open +with protocol "cloudsql" and an address of the Cloud SQL instance. + +A Go MySQL driver that has been tested to work well with Cloud SQL +is the go-sql-driver: + import "database/sql" + import _ "github.com/go-sql-driver/mysql" + + db, err := sql.Open("mysql", "user@cloudsql(project-id:instance-name)/dbname") + + +Another driver that works well with Cloud SQL is the mymysql driver: + import "database/sql" + import _ "github.com/ziutek/mymysql/godrv" + + db, err := sql.Open("mymysql", "cloudsql:instance-name*dbname/user/password") + + +Using either of these drivers, you can perform a standard SQL query. +This example assumes there is a table named 'users' with +columns 'first_name' and 'last_name': + + rows, err := db.Query("SELECT first_name, last_name FROM users") + if err != nil { + log.Errorf(ctx, "db.Query: %v", err) + } + defer rows.Close() + + for rows.Next() { + var firstName string + var lastName string + if err := rows.Scan(&firstName, &lastName); err != nil { + log.Errorf(ctx, "rows.Scan: %v", err) + continue + } + log.Infof(ctx, "First: %v - Last: %v", firstName, lastName) + } + if err := rows.Err(); err != nil { + log.Errorf(ctx, "Row error: %v", err) + } +*/ +package cloudsql + +import ( + "net" +) + +// Dial connects to the named Cloud SQL instance. +func Dial(instance string) (net.Conn, error) { + return connect(instance) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_classic.go b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_classic.go new file mode 100644 index 0000000..af62dba --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_classic.go @@ -0,0 +1,17 @@ +// Copyright 2013 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// +build appengine + +package cloudsql + +import ( + "net" + + "appengine/cloudsql" +) + +func connect(instance string) (net.Conn, error) { + return cloudsql.Dial(instance) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_vm.go b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_vm.go new file mode 100644 index 0000000..90fa7b3 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/google.golang.org/appengine/cloudsql/cloudsql_vm.go @@ -0,0 +1,16 @@ +// Copyright 2013 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// +build !appengine + +package cloudsql + +import ( + "errors" + "net" +) + +func connect(instance string) (net.Conn, error) { + return nil, errors.New(`cloudsql: not supported in App Engine "flexible environment"`) +} From f25d0b1a3c16254568e8f19888e33777ca93f36d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 1 Jan 2019 11:02:27 +0200 Subject: [PATCH 084/176] applying datetime patch --- .../siddontang/go-mysql/replication/row_event.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index 7cdd941..ddfab86 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -438,8 +438,8 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ case MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) - if i64 == 0 { - v = formatZeroTime(0, 0) + if i64 == 0 { // commented by Shlomi Noach. Yes I know about `git blame` + return "0000-00-00 00:00:00", n, nil } else { d := i64 / 1000000 t := i64 % 1000000 @@ -753,10 +753,16 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { minute := int((hms >> 6) % (1 << 6)) hour := int((hms >> 12)) + if frac != 0 { + return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second, frac), n, nil // commented by Shlomi Noach. Yes I know about `git blame` + } + return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second), n, nil // commented by Shlomi Noach. Yes I know about `git blame` + /* commented by Shlomi Noach. Yes I know about `git blame` return fracTime{ Time: time.Date(year, time.Month(month), day, hour, minute, second, int(frac*1000), time.UTC), Dec: int(dec), }, n, nil + */ } const TIMEF_OFS int64 = 0x800000000000 From 0be91ba0f2ca7aa6888afbd83d02e543ea582c1f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 1 Jan 2019 11:39:43 +0200 Subject: [PATCH 085/176] trying something... --- go/logic/applier.go | 2 -- .../siddontang/go-mysql/replication/row_event.go | 12 ++++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index ee9e066..a9faa32 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -1002,7 +1002,6 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { return err } sessionQuery := `SET - SESSION time_zone = '+00:00', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') ` if _, err := tx.Exec(sessionQuery); err != nil { @@ -1047,7 +1046,6 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) } sessionQuery := `SET - SESSION time_zone = '+00:00', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') ` if _, err := tx.Exec(sessionQuery); err != nil { diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index ddfab86..6265318 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -434,7 +434,9 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ } case MYSQL_TYPE_TIMESTAMP2: v, n, err = decodeTimestamp2(data, meta, e.timestampStringLocation) + fmt.Printf("============= v0: %+v\n", v) v = e.parseFracTime(v) + fmt.Printf("============= v1: %+v\n", v) case MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) @@ -753,16 +755,14 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { minute := int((hms >> 6) % (1 << 6)) hour := int((hms >> 12)) - if frac != 0 { - return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second, frac), n, nil // commented by Shlomi Noach. Yes I know about `git blame` - } - return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second), n, nil // commented by Shlomi Noach. Yes I know about `git blame` - /* commented by Shlomi Noach. Yes I know about `git blame` + // if frac != 0 { + // return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second, frac), n, nil // commented by Shlomi Noach. Yes I know about `git blame` + // } + // return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second), n, nil // commented by Shlomi Noach. Yes I know about `git blame` return fracTime{ Time: time.Date(year, time.Month(month), day, hour, minute, second, int(frac*1000), time.UTC), Dec: int(dec), }, n, nil - */ } const TIMEF_OFS int64 = 0x800000000000 From 7c17cee0104ac0fdb903abfc028b5000d774157f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 1 Jan 2019 11:47:41 +0200 Subject: [PATCH 086/176] reverting some changes --- go/logic/applier.go | 2 ++ .../go-mysql/replication/row_event.go | 27 ++++++++----------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index a9faa32..ee9e066 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -1002,6 +1002,7 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { return err } sessionQuery := `SET + SESSION time_zone = '+00:00', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') ` if _, err := tx.Exec(sessionQuery); err != nil { @@ -1046,6 +1047,7 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) } sessionQuery := `SET + SESSION time_zone = '+00:00', sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') ` if _, err := tx.Exec(sessionQuery); err != nil { diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index 6265318..c51ac71 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -434,9 +434,7 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ } case MYSQL_TYPE_TIMESTAMP2: v, n, err = decodeTimestamp2(data, meta, e.timestampStringLocation) - fmt.Printf("============= v0: %+v\n", v) - v = e.parseFracTime(v) - fmt.Printf("============= v1: %+v\n", v) + //v = e.parseFracTime(v) case MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) @@ -705,11 +703,8 @@ func decodeTimestamp2(data []byte, dec uint16, timestampStringLocation *time.Loc return formatZeroTime(int(usec), int(dec)), n, nil } - return fracTime{ - Time: time.Unix(sec, usec*1000), - Dec: int(dec), - timestampStringLocation: timestampStringLocation, - }, n, nil + t := time.Unix(sec, usec*1000) + return t, n, nil } const DATETIMEF_INT_OFS int64 = 0x8000000000 @@ -755,14 +750,14 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { minute := int((hms >> 6) % (1 << 6)) hour := int((hms >> 12)) - // if frac != 0 { - // return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second, frac), n, nil // commented by Shlomi Noach. Yes I know about `git blame` - // } - // return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second), n, nil // commented by Shlomi Noach. Yes I know about `git blame` - return fracTime{ - Time: time.Date(year, time.Month(month), day, hour, minute, second, int(frac*1000), time.UTC), - Dec: int(dec), - }, n, nil + if frac != 0 { + return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second, frac), n, nil // commented by Shlomi Noach. Yes I know about `git blame` + } + return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second), n, nil // commented by Shlomi Noach. Yes I know about `git blame` + // return fracTime{ + // Time: time.Date(year, time.Month(month), day, hour, minute, second, int(frac*1000), time.UTC), + // Dec: int(dec), + // }, n, nil } const TIMEF_OFS int64 = 0x800000000000 From 7e0bdbe72a7f81d7a5546bc74279ade9f79baec5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 1 Jan 2019 12:28:34 +0200 Subject: [PATCH 087/176] old timestamp, old datetime --- .../go-mysql/replication/row_event.go | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index c51ac71..9172f6e 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -423,40 +423,27 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16) (v interface{ case MYSQL_TYPE_TIMESTAMP: n = 4 t := binary.LittleEndian.Uint32(data) - if t == 0 { - v = formatZeroTime(0, 0) - } else { - v = e.parseFracTime(fracTime{ - Time: time.Unix(int64(t), 0), - Dec: 0, - timestampStringLocation: e.timestampStringLocation, - }) - } + v = time.Unix(int64(t), 0) case MYSQL_TYPE_TIMESTAMP2: v, n, err = decodeTimestamp2(data, meta, e.timestampStringLocation) //v = e.parseFracTime(v) case MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) + if i64 == 0 { // commented by Shlomi Noach. Yes I know about `git blame` return "0000-00-00 00:00:00", n, nil - } else { - d := i64 / 1000000 - t := i64 % 1000000 - v = e.parseFracTime(fracTime{ - Time: time.Date( - int(d/10000), - time.Month((d%10000)/100), - int(d%100), - int(t/10000), - int((t%10000)/100), - int(t%100), - 0, - time.UTC, - ), - Dec: 0, - }) } + d := i64 / 1000000 + t := i64 % 1000000 + v = time.Date(int(d/10000), + time.Month((d%10000)/100), + int(d%100), + int(t/10000), + int((t%10000)/100), + int(t%100), + 0, + time.UTC).Format(TimeFormat) case MYSQL_TYPE_DATETIME2: v, n, err = decodeDatetime2(data, meta) v = e.parseFracTime(v) From dd41d9a4b1b4b0511152aea81920c0eead8873a3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 3 Jan 2019 11:18:07 +0200 Subject: [PATCH 088/176] Testing nullable int --- localtests/bigint-change-nullable/create.sql | 21 ++++++++++++++++++++ localtests/bigint-change-nullable/extra_args | 1 + 2 files changed, 22 insertions(+) create mode 100644 localtests/bigint-change-nullable/create.sql create mode 100644 localtests/bigint-change-nullable/extra_args diff --git a/localtests/bigint-change-nullable/create.sql b/localtests/bigint-change-nullable/create.sql new file mode 100644 index 0000000..f4f0548 --- /dev/null +++ b/localtests/bigint-change-nullable/create.sql @@ -0,0 +1,21 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint auto_increment, + val bigint not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 18446744073709551615); + insert into gh_ost_test values (null, 18446744073709551614); + insert into gh_ost_test values (null, 18446744073709551613); +end ;; diff --git a/localtests/bigint-change-nullable/extra_args b/localtests/bigint-change-nullable/extra_args new file mode 100644 index 0000000..784d522 --- /dev/null +++ b/localtests/bigint-change-nullable/extra_args @@ -0,0 +1 @@ +--alter="change val val bigint" From 5f587c3262259c4d6ca5a17e955fc96f8f03e0d5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 14 Jan 2019 13:27:44 +0200 Subject: [PATCH 089/176] Adding --force-named-panic option --- go/base/context.go | 1 + go/cmd/gh-ost/main.go | 1 + go/logic/server.go | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/go/base/context.go b/go/base/context.go index 0043d07..318fdf9 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -123,6 +123,7 @@ type MigrationContext struct { CutOverExponentialBackoff bool ExponentialBackoffMaxInterval int64 ForceNamedCutOverCommand bool + ForceNamedPanicCommand bool PanicFlagFile string HooksPath string HooksHintMessage string diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 3024702..a745c21 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -81,6 +81,7 @@ func main() { flag.BoolVar(&migrationContext.TimestampOldTable, "timestamp-old-table", false, "Use a timestamp in old table name. This makes old table names unique and non conflicting cross migrations") cutOver := flag.String("cut-over", "atomic", "choose cut-over type (default|atomic, two-step)") flag.BoolVar(&migrationContext.ForceNamedCutOverCommand, "force-named-cut-over", false, "When true, the 'unpostpone|cut-over' interactive command must name the migrated table") + flag.BoolVar(&migrationContext.ForceNamedPanicCommand, "force-named-panic", false, "When true, the 'panic' interactive command must name the migrated table") flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running") flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") diff --git a/go/logic/server.go b/go/logic/server.go index 919f178..831fdfa 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -322,6 +322,15 @@ help # This message } case "panic": { + if arg == "" && this.migrationContext.ForceNamedPanicCommand { + err := fmt.Errorf("User commanded 'panic' without specifying table name, but --force-named-panic is set") + return NoPrintStatusRule, err + } + if arg != "" && arg != this.migrationContext.OriginalTableName { + // User explicitly provided table name. This is a courtesy protection mechanism + err := fmt.Errorf("User commanded 'panic' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) + return NoPrintStatusRule, err + } err := fmt.Errorf("User commanded 'panic'. I will now panic, without cleanup. PANIC!") this.migrationContext.PanicAbort <- err return NoPrintStatusRule, err From 6c5805d844935776a0ebdb4260b91f598f878c81 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 14 Jan 2019 13:32:43 +0200 Subject: [PATCH 090/176] documenting new flag --- doc/command-line-flags.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 804ba58..3d96ce4 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -111,6 +111,14 @@ While the ongoing estimated number of rows is still heuristic, it's almost exact Without this parameter, migration is a _noop_: testing table creation and validity of migration, but not touching data. +### force-named-cut-over + +If given, a `cut-over` command must name the migrated table, or else ignored. + +### force-named-panic + +If given, a `panic` command must name the migrated table, or else ignored. + ### force-table-names Table name prefix to be used on the temporary tables. From 5eec74132750f4119b1b761e7607aeee83928a67 Mon Sep 17 00:00:00 2001 From: Will Currie Date: Wed, 16 Jan 2019 06:28:05 +1100 Subject: [PATCH 091/176] Add Why Is "Connect to Replica" mode preferred? --- doc/questions.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/questions.md b/doc/questions.md index be6eab0..7585bc7 100644 --- a/doc/questions.md +++ b/doc/questions.md @@ -28,3 +28,9 @@ It is therefore unlikely that `gh-ost` will support this behavior. Yes. TL;DR if running all on same replica/master, make sure to provide `--replica-server-id`. [Read more](cheatsheet.md#concurrent-migrations) # Why + +### Why Is the "Connect to Replica" mode preferred? + +To avoid placing extra load on the master. `gh-ost` connects as a replication client. Each additional replica adds some load to the master. + +To monitor replication lag from a replica. This makes the replication lag throttle, `--max-lag-millis`, more representative of the lag experienced by other replicas following the master (perhaps N levels deep in a tree of replicas). From afa108f8fa49ec66a3b873c094f1bd09e4aaa169 Mon Sep 17 00:00:00 2001 From: twotwotwo Date: Tue, 22 Jan 2019 12:26:55 -0800 Subject: [PATCH 092/176] inspect: remove redundant join conditions The TABLE_SCHEMA and TABLE_NAME are already guaranteed to have the same value in COLUMNS and UNIQUES because of the WHEREs in each query. Dropping it from the ON clause makes it complete much faster. On (at least) MySQL 5.6 installs with thousands of tables, this query completes in a few seconds without the additional join conditions, but takes more than a minute with it. --- go/logic/inspect.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 5f875a0..5643c58 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -622,8 +622,6 @@ func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](* GROUP BY TABLE_SCHEMA, TABLE_NAME, INDEX_NAME ) AS UNIQUES ON ( - COLUMNS.TABLE_SCHEMA = UNIQUES.TABLE_SCHEMA AND - COLUMNS.TABLE_NAME = UNIQUES.TABLE_NAME AND COLUMNS.COLUMN_NAME = UNIQUES.FIRST_COLUMN_NAME ) WHERE From 3161cd5823ccb3e213f8783e972c082f72ad6571 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 24 Jan 2019 13:06:20 +0200 Subject: [PATCH 093/176] added: --hooks-hint-owner, --hooks-hint-token --- doc/hooks.md | 2 ++ go/base/context.go | 2 ++ go/cmd/gh-ost/main.go | 2 ++ go/logic/hooks.go | 2 ++ 4 files changed, 8 insertions(+) diff --git a/doc/hooks.md b/doc/hooks.md index 1b1958f..93f500f 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -69,6 +69,8 @@ The following variables are available on all hooks: - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` - `GH_OST_HOOKS_HINT` - copy of `--hooks-hint` value +- `GH_OST_HOOKS_HINT_OWNER` - copy of `--hooks-hint-owner` value +- `GH_OST_HOOKS_HINT_TOKEN` - copy of `--hooks-hint-token` value - `GH_OST_DRY_RUN` - whether or not the `gh-ost` run is a dry run The following variable are available on particular hooks: diff --git a/go/base/context.go b/go/base/context.go index 0043d07..22a1fd0 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -126,6 +126,8 @@ type MigrationContext struct { PanicFlagFile string HooksPath string HooksHintMessage string + HooksHintOwner string + HooksHintToken string DropServeSocket bool ServeSocketFile string diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 3024702..9a4407b 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -109,6 +109,8 @@ func main() { flag.StringVar(&migrationContext.HooksPath, "hooks-path", "", "directory where hook files are found (default: empty, ie. hooks disabled). Hook files found on this path, and conforming to hook naming conventions will be executed") flag.StringVar(&migrationContext.HooksHintMessage, "hooks-hint", "", "arbitrary message to be injected to hooks via GH_OST_HOOKS_HINT, for your convenience") + flag.StringVar(&migrationContext.HooksHintOwner, "hooks-hint-owner", "", "arbitrary name of owner to be injected to hooks via GH_OST_HOOKS_HINT_OWNER, for your convenience") + flag.StringVar(&migrationContext.HooksHintToken, "hooks-hint-token", "", "arbitrary token to be injected to hooks via GH_OST_HOOKS_HINT_TOKEN, for your convenience") flag.UintVar(&migrationContext.ReplicaServerId, "replica-server-id", 99999, "server id used by gh-ost process. Default: 99999") diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 6515d7f..be11130 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -64,6 +64,8 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) + env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) + env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) env = append(env, fmt.Sprintf("GH_OST_DRY_RUN=%t", this.migrationContext.Noop)) for _, variable := range extraVariables { From 23617f287fbf0903e8979ec4ba5086b032e0bc23 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 10:03:48 -0600 Subject: [PATCH 094/176] Add initial support for ssl encryption connections to database servers. - Adding a command line option for users to enforce tls/ssl connections for the applier, inspector, and binlog reader. - The user can optionally request server certificate verification through a command line option to specify the ca cert via a file path. - Fixes an existing bug appending the timeout option to the singleton applier connection. --- go/base/context.go | 9 +++++++ go/binlog/gomysql_reader.go | 1 + go/cmd/gh-ost/main.go | 8 ++++++ go/logic/applier.go | 2 +- go/mysql/connection.go | 51 +++++++++++++++++++++++++++++++++---- go/mysql/connection_test.go | 19 +++++++++++++- 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 0043d07..856a29c 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -99,6 +99,8 @@ type MigrationContext struct { ConfigFile string CliUser string CliPassword string + UseTLS bool + TlsCACertificate string CliMasterUser string CliMasterPassword string @@ -695,6 +697,13 @@ func (this *MigrationContext) ApplyCredentials() { } } +func (this *MigrationContext) SetupTLS() error { + if this.UseTLS { + return this.InspectorConnectionConfig.UseTLS(this.TlsCACertificate) + } + return nil +} + // ReadConfigFile attempts to read the config file, if it exists func (this *MigrationContext) ReadConfigFile() error { this.configMutex.Lock() diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 794d545..41d1ead 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -46,6 +46,7 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go Port: uint16(binlogReader.connectionConfig.Key.Port), User: binlogReader.connectionConfig.User, Password: binlogReader.connectionConfig.Password, + TLSConfig: binlogReader.connectionConfig.TLSConfig(), UseDecimal: true, } binlogReader.binlogSyncer = replication.NewBinlogSyncer(binlogSyncerConfig) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 3024702..8bd8c78 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -14,6 +14,7 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" + _ "github.com/go-sql-driver/mysql" "github.com/outbrain/golib/log" "golang.org/x/crypto/ssh/terminal" @@ -54,6 +55,9 @@ func main() { flag.StringVar(&migrationContext.ConfigFile, "conf", "", "Config file") askPass := flag.Bool("ask-pass", false, "prompt for MySQL password") + flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL") + flag.StringVar(&migrationContext.TlsCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") + flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)") @@ -194,6 +198,9 @@ func main() { if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" { log.Fatalf("--master-password requires --assume-master-host") } + if migrationContext.TlsCACertificate != "" && !migrationContext.UseTLS { + log.Fatalf("--ssl-ca requires --ssl") + } if *replicationLagQuery != "" { log.Warningf("--replication-lag-query is deprecated") } @@ -238,6 +245,7 @@ func main() { migrationContext.SetThrottleHTTP(*throttleHTTP) migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() + migrationContext.SetupTLS() if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { log.Errore(err) } diff --git a/go/logic/applier.go b/go/logic/applier.go index 140b1a4..b3de0e1 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -73,7 +73,7 @@ func (this *Applier) InitDBConnections() (err error) { if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, applierUri); err != nil { return err } - singletonApplierUri := fmt.Sprintf("%s?timeout=0", applierUri) + singletonApplierUri := fmt.Sprintf("%s&timeout=0", applierUri) if this.singletonDB, _, err = mysql.GetDB(this.migrationContext.Uuid, singletonApplierUri); err != nil { return err } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index c9c75f2..bfb445a 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -6,8 +6,14 @@ package mysql import ( + "crypto/tls" + "crypto/x509" + "errors" "fmt" + "io/ioutil" "net" + + "github.com/go-sql-driver/mysql" ) // ConnectionConfig is the minimal configuration required to connect to a MySQL server @@ -16,6 +22,7 @@ type ConnectionConfig struct { User string Password string ImpliedKey *InstanceKey + tlsConfig *tls.Config } func NewConnectionConfig() *ConnectionConfig { @@ -29,9 +36,10 @@ func NewConnectionConfig() *ConnectionConfig { // DuplicateCredentials creates a new connection config with given key and with same credentials as this config func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig { config := &ConnectionConfig{ - Key: key, - User: this.User, - Password: this.Password, + Key: key, + User: this.User, + Password: this.Password, + tlsConfig: this.tlsConfig, } config.ImpliedKey = &config.Key return config @@ -42,13 +50,42 @@ func (this *ConnectionConfig) Duplicate() *ConnectionConfig { } func (this *ConnectionConfig) String() string { - return fmt.Sprintf("%s, user=%s", this.Key.DisplayString(), this.User) + return fmt.Sprintf("%s, user=%s, usingTLS=%t", this.Key.DisplayString(), this.User, this.tlsConfig != nil) } func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } +func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { + skipVerify := caCertificatePath == "" + var rootCertPool *x509.CertPool + if !skipVerify { + rootCertPool = x509.NewCertPool() + pem, err := ioutil.ReadFile(caCertificatePath) + if err != nil { + return err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return errors.New("could not add ca certificate to cert pool") + } + } + + this.tlsConfig = &tls.Config{ + RootCAs: rootCertPool, + InsecureSkipVerify: skipVerify, + } + + if err := mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig); err != nil { + return err + } + return nil +} + +func (this *ConnectionConfig) TLSConfig() *tls.Config { + return this.tlsConfig +} + func (this *ConnectionConfig) GetDBUri(databaseName string) string { hostname := this.Key.Hostname var ip = net.ParseIP(hostname) @@ -57,5 +94,9 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { hostname = fmt.Sprintf("[%s]", hostname) } interpolateParams := true - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams) + tlsOption := "false" + if this.tlsConfig != nil { + tlsOption = this.Key.StringCode() + } + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 19badf3..3a6cf29 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -6,6 +6,7 @@ package mysql import ( + "crypto/tls" "testing" "github.com/outbrain/golib/log" @@ -31,6 +32,10 @@ func TestDuplicateCredentials(t *testing.T) { c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" + c.tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + ServerName: "feathers", + } dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310}) test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost") @@ -39,6 +44,7 @@ func TestDuplicateCredentials(t *testing.T) { test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3310) test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") + test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig) } func TestDuplicate(t *testing.T) { @@ -63,5 +69,16 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") +} + +func TestGetDBUriWithTLSSetup(t *testing.T) { + c := NewConnectionConfig() + c.Key = InstanceKey{Hostname: "myhost", Port: 3306} + c.User = "gromit" + c.Password = "penguin" + c.tlsConfig = &tls.Config{} + + uri := c.GetDBUri("test") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=myhost:3306") } From 4f9367e6909dc93d824cda9f536921e449cf689e Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 16:59:42 -0600 Subject: [PATCH 095/176] Fix casing for initialism --- go/base/context.go | 4 ++-- go/cmd/gh-ost/main.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 856a29c..90581e2 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -100,7 +100,7 @@ type MigrationContext struct { CliUser string CliPassword string UseTLS bool - TlsCACertificate string + TLSCACertificate string CliMasterUser string CliMasterPassword string @@ -699,7 +699,7 @@ func (this *MigrationContext) ApplyCredentials() { func (this *MigrationContext) SetupTLS() error { if this.UseTLS { - return this.InspectorConnectionConfig.UseTLS(this.TlsCACertificate) + return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate) } return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 8bd8c78..db044f3 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -56,7 +56,7 @@ func main() { askPass := flag.Bool("ask-pass", false, "prompt for MySQL password") flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL") - flag.StringVar(&migrationContext.TlsCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") + flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") @@ -198,7 +198,7 @@ func main() { if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" { log.Fatalf("--master-password requires --assume-master-host") } - if migrationContext.TlsCACertificate != "" && !migrationContext.UseTLS { + if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } if *replicationLagQuery != "" { From dc599bb0368f0ccd15a605348a6615cfe8f03faf Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 17:20:11 -0600 Subject: [PATCH 096/176] Remove unnecessary branching --- go/mysql/connection.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/mysql/connection.go b/go/mysql/connection.go index bfb445a..7fa8268 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -76,10 +76,7 @@ func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { InsecureSkipVerify: skipVerify, } - if err := mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig); err != nil { - return err - } - return nil + return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) } func (this *ConnectionConfig) TLSConfig() *tls.Config { From c440112d440112e819d17f8d6997f3b41882edac Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Thu, 31 Jan 2019 17:23:19 -0600 Subject: [PATCH 097/176] Explain default setting for TLS param in DSN --- go/mysql/connection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 7fa8268..742ca7e 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -91,6 +91,8 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { hostname = fmt.Sprintf("[%s]", hostname) } interpolateParams := true + // go-mysql-driver defaults to false if tls param is not provided; explicitly setting here to + // simplify construction of the DSN below. tlsOption := "false" if this.tlsConfig != nil { tlsOption = this.Key.StringCode() From f67ec15f2b972fd6b6b5b1892fe9838a3f0c9e34 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Fri, 1 Feb 2019 13:16:54 -0600 Subject: [PATCH 098/176] Handle returned error --- go/cmd/gh-ost/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index db044f3..9d7d60f 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -245,7 +245,9 @@ func main() { migrationContext.SetThrottleHTTP(*throttleHTTP) migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() - migrationContext.SetupTLS() + if err := migrationContext.SetupTLS(); err != nil { + log.Fatale(err) + } if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { log.Errore(err) } From 5319157789e317d296e298c0eb86240d3ef3cd56 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Fri, 1 Feb 2019 13:20:17 -0600 Subject: [PATCH 099/176] Expand usage statement to indicate setting applies to multiple hosts --- go/cmd/gh-ost/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 9d7d60f..d8add4f 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -55,8 +55,8 @@ func main() { flag.StringVar(&migrationContext.ConfigFile, "conf", "", "Config file") askPass := flag.Bool("ask-pass", false, "prompt for MySQL password") - flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL") - flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections. Requires --ssl") + flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") + flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") From 79df0d1c5de9eec208da73f2657b6ff666af6239 Mon Sep 17 00:00:00 2001 From: Matt Belisle Date: Mon, 4 Feb 2019 14:46:08 -0600 Subject: [PATCH 100/176] Adding --ssl-insecure flag --- go/base/context.go | 19 ++++++++++--------- go/cmd/gh-ost/main.go | 4 ++++ go/mysql/connection.go | 28 ++++++++++++++++++---------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 90581e2..9f9de9b 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -94,15 +94,16 @@ type MigrationContext struct { AliyunRDS bool GoogleCloudPlatform bool - config ContextConfig - configMutex *sync.Mutex - ConfigFile string - CliUser string - CliPassword string - UseTLS bool - TLSCACertificate string - CliMasterUser string - CliMasterPassword string + config ContextConfig + configMutex *sync.Mutex + ConfigFile string + CliUser string + CliPassword string + UseTLS bool + TLSInsecureSkipVerify bool + TLSCACertificate string + CliMasterUser string + CliMasterPassword string HeartbeatIntervalMilliseconds int64 defaultNumRetries int64 diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index d8add4f..96bd2dc 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -57,6 +57,7 @@ func main() { flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") + flag.StringVar(&migrationContext.TLSInsecureSkipVerify, "ssl-insecure", false, "Do not verify that the TLS connection is secure. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") @@ -201,6 +202,9 @@ func main() { if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } + if migrationContext.TLSInsecureSkipVerify && !migrationContext.UseTLS { + log.Fatalf("--ssl-insecure requires --ssl") + } if *replicationLagQuery != "" { log.Warningf("--replication-lag-query is deprecated") } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 742ca7e..f0f73ce 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -58,22 +58,30 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { } func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { - skipVerify := caCertificatePath == "" var rootCertPool *x509.CertPool - if !skipVerify { - rootCertPool = x509.NewCertPool() - pem, err := ioutil.ReadFile(caCertificatePath) - if err != nil { - return err - } - if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { - return errors.New("could not add ca certificate to cert pool") + var err error + + if !this.TLSInsecureSkipVerify { + if caCertificatePath == "" { + rootCertPool, err = x509.SystemCertPool() + if err != nil { + return err + } + } else { + rootCertPool = x509.NewCertPool() + pem, err := ioutil.ReadFile(caCertificatePath) + if err != nil { + return err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return errors.New("could not add ca certificate to cert pool") + } } } this.tlsConfig = &tls.Config{ RootCAs: rootCertPool, - InsecureSkipVerify: skipVerify, + InsecureSkipVerify: this.TLSInsecureSkipVerify, } return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) From 5b0dfb009c5e0dfb815d125da2faec1f132a4c75 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Mon, 4 Feb 2019 16:21:25 -0600 Subject: [PATCH 101/176] Wireup allowing insecure ssl --- go/base/context.go | 22 +++++++++++----------- go/cmd/gh-ost/main.go | 6 +++--- go/mysql/connection.go | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 9f9de9b..e9991d8 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -94,16 +94,16 @@ type MigrationContext struct { AliyunRDS bool GoogleCloudPlatform bool - config ContextConfig - configMutex *sync.Mutex - ConfigFile string - CliUser string - CliPassword string - UseTLS bool - TLSInsecureSkipVerify bool - TLSCACertificate string - CliMasterUser string - CliMasterPassword string + config ContextConfig + configMutex *sync.Mutex + ConfigFile string + CliUser string + CliPassword string + UseTLS bool + TLSAllowInsecure bool + TLSCACertificate string + CliMasterUser string + CliMasterPassword string HeartbeatIntervalMilliseconds int64 defaultNumRetries int64 @@ -700,7 +700,7 @@ func (this *MigrationContext) ApplyCredentials() { func (this *MigrationContext) SetupTLS() error { if this.UseTLS { - return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate) + return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate, this.TLSAllowInsecure) } return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 96bd2dc..b02d6b7 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -57,7 +57,7 @@ func main() { flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") - flag.StringVar(&migrationContext.TLSInsecureSkipVerify, "ssl-insecure", false, "Do not verify that the TLS connection is secure. Requires --ssl") + flag.BoolVar(&migrationContext.TLSAllowInsecure, "ssl-allow-insecure", false, "Skips verification of MySQL hosts' certificate chain and host name. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") @@ -202,8 +202,8 @@ func main() { if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } - if migrationContext.TLSInsecureSkipVerify && !migrationContext.UseTLS { - log.Fatalf("--ssl-insecure requires --ssl") + if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS { + log.Fatalf("--ssl-allow-insecure requires --ssl") } if *replicationLagQuery != "" { log.Warningf("--replication-lag-query is deprecated") diff --git a/go/mysql/connection.go b/go/mysql/connection.go index f0f73ce..d6c7215 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -57,11 +57,11 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { +func (this *ConnectionConfig) UseTLS(caCertificatePath string, allowInsecure bool) error { var rootCertPool *x509.CertPool var err error - if !this.TLSInsecureSkipVerify { + if !allowInsecure { if caCertificatePath == "" { rootCertPool, err = x509.SystemCertPool() if err != nil { @@ -81,7 +81,7 @@ func (this *ConnectionConfig) UseTLS(caCertificatePath string) error { this.tlsConfig = &tls.Config{ RootCAs: rootCertPool, - InsecureSkipVerify: this.TLSInsecureSkipVerify, + InsecureSkipVerify: allowInsecure, } return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) From 1543098891296e496e16f63b671d6d0ea80342e8 Mon Sep 17 00:00:00 2001 From: Brandon Bodnar <33429657+brandonbodnar-wk@users.noreply.github.com> Date: Tue, 5 Feb 2019 08:02:47 -0600 Subject: [PATCH 102/176] Document ssl-related command line flags. --- doc/command-line-flags.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 804ba58..c20a5a8 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -177,6 +177,18 @@ By default `gh-ost` verifies no foreign keys exist on the migrated table. On ser See [`approve-renamed-columns`](#approve-renamed-columns) +### ssl + +By default `gh-ost` does not use ssl/tls connections to the database servers when performing migrations. This flag instructs `gh-ost` to use encrypted connections. If enabled, `gh-ost` will use the system's ca certificate pool for server certificate verification. If a different certificate is needed for server verification, see `--ssl-ca`. If you wish to skip server verification, but still use encrypted connections, use with `--ssl-allow-insecure`. + +### ssl-allow-insecure + +Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but without verifying the validity of the certificate provided by the server during the connection. Requires `--ssl`. + +### ssl-ca + +`--ssl-ca=/path/to/ca-cert.pem`: ca certificate file (in PEM format) to use for server certificate verification. If specified, the default system ca cert pool will not be used for verification, only the ca cert provided here. Requires `--ssl`. + ### test-on-replica Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) From b82fc452212e6757b1ab97fb1762d142424c1fd4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 12 Feb 2019 08:01:47 +0200 Subject: [PATCH 103/176] Adding maintainer email in DEB package --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 264620b..df9ac7d 100755 --- a/build.sh +++ b/build.sh @@ -42,8 +42,8 @@ function build { builddir=$(setuptree) cp $buildpath/$target $builddir/gh-ost/usr/bin cd $buildpath - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . fi } From a8fae9818daa42681c7806f843067e2d5d360273 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 14 Feb 2019 12:07:25 +0200 Subject: [PATCH 104/176] v1.0.48 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index bde91a2..56d0dad 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.47 +1.0.48 From 7d5749b84a30b1e41ec99bbb55795db1f0352425 Mon Sep 17 00:00:00 2001 From: babinomec Date: Thu, 14 Feb 2019 15:58:49 +0100 Subject: [PATCH 105/176] add --skip-strict-mode option --- doc/command-line-flags.md | 4 ++++ go/base/context.go | 1 + go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 24 ++++++++++++------------ 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index c20a5a8..02e7c64 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -173,6 +173,10 @@ See also: [`concurrent-migrations`](cheatsheet.md#concurrent-migrations) on the By default `gh-ost` verifies no foreign keys exist on the migrated table. On servers with large number of tables this check can take a long time. If you're absolutely certain no foreign keys exist (table does not reference other table nor is referenced by other tables) and wish to save the check time, provide with `--skip-foreign-key-checks`. +### skip-strict-mode + +By default `gh-ost` enforces STRICT_ALL_TABLES sql_mode as a safety measure. In some cases this changes the behaviour of other modes (namely ERROR_FOR_DIVISION_BY_ZERO, NO_ZERO_DATE, and NO_ZERO_IN_DATE) which may lead to errors during migration. Use `--skip-strict-mode` to explicitly tell `gh-ost` not to enforce this. **Danger** This may have some unexpected disastrous side effects. + ### skip-renamed-columns See [`approve-renamed-columns`](#approve-renamed-columns) diff --git a/go/base/context.go b/go/base/context.go index b2a6c15..09bc9c0 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -86,6 +86,7 @@ type MigrationContext struct { SwitchToRowBinlogFormat bool AssumeRBR bool SkipForeignKeyChecks bool + SkipStrictMode bool NullableUniqueKeyAllowed bool ApproveRenamedColumns bool SkipRenamedColumns bool diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a467f07..55e65d8 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -72,6 +72,7 @@ func main() { flag.BoolVar(&migrationContext.IsTungsten, "tungsten", false, "explicitly let gh-ost know that you are running on a tungsten-replication based topology (you are likely to also provide --assume-master-host)") flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys") flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that") + flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode") flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") diff --git a/go/logic/applier.go b/go/logic/applier.go index b3de0e1..1544359 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -482,10 +482,10 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected return nil, err } defer tx.Rollback() - sessionQuery := fmt.Sprintf(`SET - SESSION time_zone = '%s', - sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') - `, this.migrationContext.ApplierTimeZone) + sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone) + if !this.migrationContext.SkipStrictMode { + sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + } if _, err := tx.Exec(sessionQuery); err != nil { return nil, err } @@ -1005,10 +1005,10 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { tx.Rollback() return err } - sessionQuery := `SET - SESSION time_zone = '+00:00', - sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') - ` + sessionQuery := fmt.Sprintf("SET SESSION time_zone = '+00:00'") + if !this.migrationContext.SkipStrictMode { + sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + } if _, err := tx.Exec(sessionQuery); err != nil { return rollback(err) } @@ -1050,10 +1050,10 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) return err } - sessionQuery := `SET - SESSION time_zone = '+00:00', - sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') - ` + sessionQuery := "SET SESSION time_zone = '+00:00'" + if !this.migrationContext.SkipStrictMode { + sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + } if _, err := tx.Exec(sessionQuery); err != nil { return rollback(err) } From b4da7cfbfb0a08a94e13df82403789713da4ed71 Mon Sep 17 00:00:00 2001 From: Rafe Kettler Date: Fri, 22 Feb 2019 10:33:19 -0800 Subject: [PATCH 106/176] Improve SSL support Related issue: https://github.com/github/gh-ost/issues/521 - Add --ssl-cert and --ssl-key options to specify SSL public/private key files - Allow combining --ssl-allow-insecure with other --ssl* flags. `mysql.RegisterTLSConfig` allows combining the corresponding parameters in the `tls.Config` it receives, so gh-ost should allow this. I found being able to pass --ssl-allow-insecure along with --ssl-ca, --ssl-cert, and --ssl-key useful in testing. - Use the same TLS config everywhere. Since the CLI only supports a single set of --ssl* configuration parameters, this should be fine -- `mysql.RegisterTLSConfig` documentation indicates the TLS config given will not be modified, so it can safely be used in many goroutines provided we also do not modify it. The previous implementation did not work when the TLS config was duplicated, which happens when gh-ost walks up the replication chain trying to find the master. This is because, when the config is duplicated, we must call `RegisterTLSConfig` again with the new config. This config is exactly the same, so it's easiest to side-step the issue by registering the TLS config once and using it everywhere. --- doc/command-line-flags.md | 8 +++++++ go/base/context.go | 4 +++- go/cmd/gh-ost/main.go | 8 +++++++ go/mysql/connection.go | 47 +++++++++++++++++++++++-------------- go/mysql/connection_test.go | 2 +- 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index c20a5a8..2d9bfd1 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -189,6 +189,14 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but `--ssl-ca=/path/to/ca-cert.pem`: ca certificate file (in PEM format) to use for server certificate verification. If specified, the default system ca cert pool will not be used for verification, only the ca cert provided here. Requires `--ssl`. +### ssl-cert + +`--ssl-cert=/path/to/ssl-cert.crt`: SSL public key certificate file (in PEM format). + +### ssl-key + +`--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format). + ### test-on-replica Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) diff --git a/go/base/context.go b/go/base/context.go index b2a6c15..8a8f721 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -102,6 +102,8 @@ type MigrationContext struct { UseTLS bool TLSAllowInsecure bool TLSCACertificate string + TLSCertificate string + TLSKey string CliMasterUser string CliMasterPassword string @@ -702,7 +704,7 @@ func (this *MigrationContext) ApplyCredentials() { func (this *MigrationContext) SetupTLS() error { if this.UseTLS { - return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate, this.TLSAllowInsecure) + return this.InspectorConnectionConfig.UseTLS(this.TLSCACertificate, this.TLSCertificate, this.TLSKey, this.TLSAllowInsecure) } return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a467f07..5c30ffd 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -57,6 +57,8 @@ func main() { flag.BoolVar(&migrationContext.UseTLS, "ssl", false, "Enable SSL encrypted connections to MySQL hosts") flag.StringVar(&migrationContext.TLSCACertificate, "ssl-ca", "", "CA certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") + flag.StringVar(&migrationContext.TLSCertificate, "ssl-cert", "", "Certificate in PEM format for TLS connections to MySQL hosts. Requires --ssl") + flag.StringVar(&migrationContext.TLSKey, "ssl-key", "", "Key in PEM format for TLS connections to MySQL hosts. Requires --ssl") flag.BoolVar(&migrationContext.TLSAllowInsecure, "ssl-allow-insecure", false, "Skips verification of MySQL hosts' certificate chain and host name. Requires --ssl") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") @@ -204,6 +206,12 @@ func main() { if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { log.Fatalf("--ssl-ca requires --ssl") } + if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS { + log.Fatalf("--ssl-cert requires --ssl") + } + if migrationContext.TLSKey != "" && !migrationContext.UseTLS { + log.Fatalf("--ssl-key requires --ssl") + } if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS { log.Fatalf("--ssl-allow-insecure requires --ssl") } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index d6c7215..654998c 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -16,6 +16,10 @@ import ( "github.com/go-sql-driver/mysql" ) +const ( + TLS_CONFIG_KEY = "ghost" +) + // ConnectionConfig is the minimal configuration required to connect to a MySQL server type ConnectionConfig struct { Key InstanceKey @@ -57,34 +61,41 @@ func (this *ConnectionConfig) Equals(other *ConnectionConfig) bool { return this.Key.Equals(&other.Key) || this.ImpliedKey.Equals(other.ImpliedKey) } -func (this *ConnectionConfig) UseTLS(caCertificatePath string, allowInsecure bool) error { +func (this *ConnectionConfig) UseTLS(caCertificatePath, clientCertificate, clientKey string, allowInsecure bool) error { var rootCertPool *x509.CertPool + var certs []tls.Certificate var err error - if !allowInsecure { - if caCertificatePath == "" { - rootCertPool, err = x509.SystemCertPool() - if err != nil { - return err - } - } else { - rootCertPool = x509.NewCertPool() - pem, err := ioutil.ReadFile(caCertificatePath) - if err != nil { - return err - } - if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { - return errors.New("could not add ca certificate to cert pool") - } + if caCertificatePath == "" { + rootCertPool, err = x509.SystemCertPool() + if err != nil { + return err } + } else { + rootCertPool = x509.NewCertPool() + pem, err := ioutil.ReadFile(caCertificatePath) + if err != nil { + return err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return errors.New("could not add ca certificate to cert pool") + } + } + if clientCertificate != "" || clientKey != "" { + cert, err := tls.LoadX509KeyPair(clientCertificate, clientKey) + if err != nil { + return err + } + certs = []tls.Certificate{cert} } this.tlsConfig = &tls.Config{ + Certificates: certs, RootCAs: rootCertPool, InsecureSkipVerify: allowInsecure, } - return mysql.RegisterTLSConfig(this.Key.StringCode(), this.tlsConfig) + return mysql.RegisterTLSConfig(TLS_CONFIG_KEY, this.tlsConfig) } func (this *ConnectionConfig) TLSConfig() *tls.Config { @@ -103,7 +114,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { // simplify construction of the DSN below. tlsOption := "false" if this.tlsConfig != nil { - tlsOption = this.Key.StringCode() + tlsOption = TLS_CONFIG_KEY } return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 3a6cf29..6ce85a7 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=myhost:3306") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") } From 7ead4c4a56dda163fbf50fce6bb22ac6d96b690f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 25 Feb 2019 14:02:57 +0200 Subject: [PATCH 107/176] named throttle, no-throttle --- go/logic/server.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/logic/server.go b/go/logic/server.go index 831fdfa..774c4ab 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -292,12 +292,22 @@ help # This message } case "throttle", "pause", "suspend": { + if arg != "" && arg != this.migrationContext.OriginalTableName { + // User explicitly provided table name. This is a courtesy protection mechanism + err := fmt.Errorf("User commanded 'throttle' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) + return NoPrintStatusRule, err + } atomic.StoreInt64(&this.migrationContext.ThrottleCommandedByUser, 1) fmt.Fprintf(writer, throttleHint) return ForcePrintStatusAndHintRule, nil } case "no-throttle", "unthrottle", "resume", "continue": { + if arg != "" && arg != this.migrationContext.OriginalTableName { + // User explicitly provided table name. This is a courtesy protection mechanism + err := fmt.Errorf("User commanded 'no-throttle' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) + return NoPrintStatusRule, err + } atomic.StoreInt64(&this.migrationContext.ThrottleCommandedByUser, 0) return ForcePrintStatusAndHintRule, nil } From 4d7370529b833de4452d034bd284465526d99b9d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 24 Mar 2019 11:32:37 +0200 Subject: [PATCH 108/176] Always use NO_AUTO_VALUE_ON_ZERO --- localtests/autoinc-zero-value/create.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 localtests/autoinc-zero-value/create.sql diff --git a/localtests/autoinc-zero-value/create.sql b/localtests/autoinc-zero-value/create.sql new file mode 100644 index 0000000..ba08bd4 --- /dev/null +++ b/localtests/autoinc-zero-value/create.sql @@ -0,0 +1,9 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + primary key(id) +) auto_increment=1; + +set session sql_mode='NO_AUTO_VALUE_ON_ZERO'; +insert into gh_ost_test values (0, 23); From f0ed7aeb0b09ba9e11ac887802d246214210a4ff Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 24 Mar 2019 11:32:42 +0200 Subject: [PATCH 109/176] Always use NO_AUTO_VALUE_ON_ZERO --- go/logic/applier.go | 68 ++++++--------------------------------------- 1 file changed, 9 insertions(+), 59 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 1544359..5fb795b 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -483,9 +483,12 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } defer tx.Rollback() sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone) + sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO` if !this.migrationContext.SkipStrictMode { - sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum) } + sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum) + if _, err := tx.Exec(sessionQuery); err != nil { return nil, err } @@ -977,63 +980,6 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (result return append(results, newDmlBuildResultError(fmt.Errorf("Unknown dml event type: %+v", dmlEvent.DML))) } -// ApplyDMLEventQuery writes an entry to the ghost table, in response to an intercepted -// original-table binlog event -func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { - for _, buildResult := range this.buildDMLEventQuery(dmlEvent) { - if buildResult.err != nil { - return buildResult.err - } - // TODO The below is in preparation for transactional writes on the ghost tables. - // Such writes would be, for example: - // - prepended with sql_mode setup - // - prepended with time zone setup - // - prepended with SET SQL_LOG_BIN=0 - // - prepended with SET FK_CHECKS=0 - // etc. - // - // a known problem: https://github.com/golang/go/issues/9373 -- bitint unsigned values, not supported in database/sql - // is solved by silently converting unsigned bigints to string values. - // - - err := func() error { - tx, err := this.db.Begin() - if err != nil { - return err - } - rollback := func(err error) error { - tx.Rollback() - return err - } - sessionQuery := fmt.Sprintf("SET SESSION time_zone = '+00:00'") - if !this.migrationContext.SkipStrictMode { - sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" - } - if _, err := tx.Exec(sessionQuery); err != nil { - return rollback(err) - } - if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil { - return rollback(err) - } - if err := tx.Commit(); err != nil { - return err - } - return nil - }() - - if err != nil { - err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), buildResult.query, buildResult.args) - return log.Errore(err) - } - // no error - atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, 1) - if this.migrationContext.CountTableRows { - atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, buildResult.rowsDelta) - } - } - return nil -} - // ApplyDMLEventQueries applies multiple DML queries onto the _ghost_ table func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) error { @@ -1051,9 +997,13 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) } sessionQuery := "SET SESSION time_zone = '+00:00'" + + sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO` if !this.migrationContext.SkipStrictMode { - sessionQuery += ", sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')" + sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum) } + sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum) + if _, err := tx.Exec(sessionQuery); err != nil { return rollback(err) } From 773cddc88acaa47b52f99157a3faf746b3708f78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 29 Mar 2019 05:05:40 -0700 Subject: [PATCH 110/176] Clarify it's foreign key /constraints/ that don't work --- doc/requirements-and-limitations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 628451d..8376e18 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -22,7 +22,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th ### Limitations -- Foreign keys not supported. They may be supported in the future, to some extent. +- Foreign key constraints are not supported. They may be supported in the future, to some extent. - Triggers are not supported. They may be supported in the future. From 256025853c3cb386453ed223f865c9affc249b93 Mon Sep 17 00:00:00 2001 From: Jon Chambers Date: Fri, 12 Apr 2019 12:07:00 -0400 Subject: [PATCH 111/176] Fix a trivial it's/its typo. --- doc/requirements-and-limitations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 8376e18..096e2c9 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -38,7 +38,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - It is not allowed to migrate a table where another table exists with same name and different upper/lower case. - For example, you may not migrate `MyTable` if another table called `MYtable` exists in the same schema. -- Amazon RDS works, but has it's own [limitations](rds.md). +- Amazon RDS works, but has its own [limitations](rds.md). - Google Cloud SQL works, `--gcp` flag required. - Aliyun RDS works, `--aliyun-rds` flag required. From b8cdd5d133b97ac308dc33323b797a6d57bdc785 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 16 May 2019 13:29:40 +0300 Subject: [PATCH 112/176] Update go-sql-driver/mysql 2019-05-16 --- .../mysql/.github/ISSUE_TEMPLATE.md | 21 + .../mysql/.github/PULL_REQUEST_TEMPLATE.md | 9 + .../github.com/go-sql-driver/mysql/.gitignore | 1 + .../go-sql-driver/mysql/.travis.yml | 126 +- .../go-sql-driver/mysql/.travis/docker.cnf | 5 + .../go-sql-driver/mysql/.travis/gofmt.sh | 7 + .../go-sql-driver/mysql/.travis/wait_mysql.sh | 8 + vendor/github.com/go-sql-driver/mysql/AUTHORS | 57 +- .../go-sql-driver/mysql/CHANGELOG.md | 89 +- .../go-sql-driver/mysql/CONTRIBUTING.md | 17 - .../github.com/go-sql-driver/mysql/README.md | 197 +- .../go-sql-driver/mysql/appengine.go | 10 +- vendor/github.com/go-sql-driver/mysql/auth.go | 422 +++ .../go-sql-driver/mysql/auth_test.go | 1330 ++++++++ .../go-sql-driver/mysql/benchmark_test.go | 373 ++ .../github.com/go-sql-driver/mysql/buffer.go | 122 +- .../go-sql-driver/mysql/collations.go | 379 ++- .../go-sql-driver/mysql/conncheck.go | 53 + .../go-sql-driver/mysql/conncheck_dummy.go | 17 + .../go-sql-driver/mysql/conncheck_test.go | 38 + .../go-sql-driver/mysql/connection.go | 414 ++- .../go-sql-driver/mysql/connection_test.go | 175 + .../go-sql-driver/mysql/connector.go | 143 + .../github.com/go-sql-driver/mysql/const.go | 26 +- .../github.com/go-sql-driver/mysql/driver.go | 136 +- .../go-sql-driver/mysql/driver_go110.go | 37 + .../go-sql-driver/mysql/driver_go110_test.go | 137 + .../go-sql-driver/mysql/driver_test.go | 2996 +++++++++++++++++ vendor/github.com/go-sql-driver/mysql/dsn.go | 636 ++++ .../go-sql-driver/mysql/dsn_test.go | 415 +++ .../github.com/go-sql-driver/mysql/errors.go | 104 +- .../go-sql-driver/mysql/errors_test.go | 42 + .../github.com/go-sql-driver/mysql/fields.go | 194 ++ .../github.com/go-sql-driver/mysql/infile.go | 26 +- .../github.com/go-sql-driver/mysql/packets.go | 614 ++-- .../go-sql-driver/mysql/packets_test.go | 336 ++ vendor/github.com/go-sql-driver/mysql/rows.go | 191 +- .../go-sql-driver/mysql/statement.go | 136 +- .../go-sql-driver/mysql/statement_test.go | 126 + .../go-sql-driver/mysql/transaction.go | 4 +- .../github.com/go-sql-driver/mysql/utils.go | 730 ++-- .../go-sql-driver/mysql/utils_test.go | 334 ++ .../mysql/.github/ISSUE_TEMPLATE.md | 21 + .../mysql/.github/PULL_REQUEST_TEMPLATE.md | 9 + .../github.com/go-sql-driver/mysql/.gitignore | 9 + .../go-sql-driver/mysql/.travis.yml | 128 + .../go-sql-driver/mysql/.travis/docker.cnf | 5 + .../go-sql-driver/mysql/.travis/gofmt.sh | 7 + .../go-sql-driver/mysql/.travis/wait_mysql.sh | 8 + .../github.com/go-sql-driver/mysql/AUTHORS | 11 + .../go-sql-driver/mysql/CHANGELOG.md | 167 + .../go-sql-driver/mysql/CONTRIBUTING.md | 23 + .../github.com/go-sql-driver/mysql/README.md | 495 +++ .../go-sql-driver/mysql/appengine.go | 8 +- .../github.com/go-sql-driver/mysql/auth.go | 44 +- .../go-sql-driver/mysql/auth_test.go | 1330 ++++++++ .../go-sql-driver/mysql/benchmark_test.go | 373 ++ .../github.com/go-sql-driver/mysql/buffer.go | 97 +- .../go-sql-driver/mysql/collations.go | 376 ++- .../go-sql-driver/mysql/conncheck.go | 53 + .../go-sql-driver/mysql/conncheck_dummy.go | 17 + .../go-sql-driver/mysql/conncheck_test.go | 38 + .../go-sql-driver/mysql/connection.go | 45 +- .../go-sql-driver/mysql/connection_test.go | 175 + .../go-sql-driver/mysql/connector.go | 143 + .../github.com/go-sql-driver/mysql/driver.go | 136 +- .../go-sql-driver/mysql/driver_go110.go | 37 + .../go-sql-driver/mysql/driver_go110_test.go | 137 + .../go-sql-driver/mysql/driver_test.go | 2996 +++++++++++++++++ .../github.com/go-sql-driver/mysql/dsn.go | 73 +- .../go-sql-driver/mysql/dsn_test.go | 415 +++ .../go-sql-driver/mysql/errors_test.go | 42 + .../github.com/go-sql-driver/mysql/packets.go | 132 +- .../go-sql-driver/mysql/packets_test.go | 336 ++ .../github.com/go-sql-driver/mysql/rows.go | 7 + .../go-sql-driver/mysql/statement.go | 11 +- .../go-sql-driver/mysql/statement_test.go | 126 + .../github.com/go-sql-driver/mysql/utils.go | 6 +- .../go-sql-driver/mysql/utils_test.go | 334 ++ 79 files changed, 17791 insertions(+), 1812 deletions(-) create mode 100644 vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md create mode 100644 vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf create mode 100755 vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh create mode 100755 vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh create mode 100644 vendor/github.com/go-sql-driver/mysql/auth.go create mode 100644 vendor/github.com/go-sql-driver/mysql/auth_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/benchmark_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/conncheck.go create mode 100644 vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go create mode 100644 vendor/github.com/go-sql-driver/mysql/conncheck_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/connection_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/connector.go create mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110.go create mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/driver_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/dsn.go create mode 100644 vendor/github.com/go-sql-driver/mysql/dsn_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/errors_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/fields.go create mode 100644 vendor/github.com/go-sql-driver/mysql/packets_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/statement_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/utils_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf create mode 100755 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh create mode 100755 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go create mode 100644 vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go diff --git a/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md b/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..d9771f1 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,21 @@ +### Issue description +Tell us what should happen and what happens instead + +### Example code +```go +If possible, please enter some example code here to reproduce the issue. +``` + +### Error log +``` +If you have an error log, please paste it here. +``` + +### Configuration +*Driver version (or git SHA):* + +*Go version:* run `go version` in your console + +*Server version:* E.g. MySQL 5.6, MariaDB 10.0.20 + +*Server OS:* E.g. Debian 8.1 (Jessie), Windows 10 diff --git a/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md b/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6f5c7eb --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +### Description +Please explain the changes you made here. + +### Checklist +- [ ] Code compiles correctly +- [ ] Created tests which fail without the change (if possible) +- [ ] All tests passing +- [ ] Extended the README / documentation, if necessary +- [ ] Added myself / the copyright holder to the AUTHORS file diff --git a/vendor/github.com/go-sql-driver/mysql/.gitignore b/vendor/github.com/go-sql-driver/mysql/.gitignore index ba8e0cb..2de28da 100644 --- a/vendor/github.com/go-sql-driver/mysql/.gitignore +++ b/vendor/github.com/go-sql-driver/mysql/.gitignore @@ -6,3 +6,4 @@ Icon? ehthumbs.db Thumbs.db +.idea diff --git a/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/go-sql-driver/mysql/.travis.yml index 2f4e3c2..eae311b 100644 --- a/vendor/github.com/go-sql-driver/mysql/.travis.yml +++ b/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -1,10 +1,128 @@ sudo: false language: go go: - - 1.2 - - 1.3 - - 1.4 - - tip + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - master + +before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB" | sudo tee -a /etc/mysql/my.cnf + - sudo service mysql restart + - .travis/wait_mysql.sh - mysql -e 'create database gotest;' + +matrix: + include: + - env: DB=MYSQL8 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:8.0 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:8.0 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MYSQL57 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:5.7 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:5.7 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA55 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:5.5 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:5.5 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA10_1 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:10.1 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:10.1 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - os: osx + osx_image: xcode10.1 + addons: + homebrew: + packages: + - mysql + go: 1.12.x + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + - mysql.server start + - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + - mysql -uroot -e 'create database gotest;' + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3306 + - export MYSQL_TEST_CONCURRENT=1 + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - go vet ./... + - .travis/gofmt.sh +after_script: + - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf b/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf new file mode 100644 index 0000000..e57754e --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf @@ -0,0 +1,5 @@ +[client] +user = gotest +password = secret +host = 127.0.0.1 +port = 3307 diff --git a/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh b/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh new file mode 100755 index 0000000..9bf0d16 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ev + +# Only check for go1.10+ since the gofmt style changed +if [[ $(go version) =~ go1\.([0-9]+) ]] && ((${BASH_REMATCH[1]} >= 10)); then + test -z "$(gofmt -d -s . | tee /dev/stderr)" +fi diff --git a/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh b/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh new file mode 100755 index 0000000..e87993e --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh @@ -0,0 +1,8 @@ +#!/bin/sh +while : +do + if mysql -e 'select version()' 2>&1 | grep 'version()\|ERROR 2059 (HY000):'; then + break + fi + sleep 3 +done diff --git a/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/go-sql-driver/mysql/AUTHORS index 6dd0167..bfe74c4 100644 --- a/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -12,35 +12,90 @@ # Individual Persons Aaron Hopkins +Achille Roussel +Alexey Palazhchenko +Andrew Reid Arne Hormann +Asta Xie +Bulat Gaifullin Carlos Nieto Chris Moos +Craig Wilson +Daniel Montoya +Daniel Nichter +Daniël van Eeden +Dave Protasowski DisposaBoy +Egor Smolyakov +Erwan Martin +Evan Shaw Frederick Mayle Gustavo Kristic +Hajime Nakagami Hanno Braun Henri Yandell Hirotaka Yamamoto +Huyiguang +ICHINOSE Shogo +Ilia Cimpoes INADA Naoki +Jacek Szwec James Harr +Jeff Hodges +Jeffrey Charles +Jerome Meyer Jian Zhen Joshua Prunier +Julien Lefevre Julien Schmidt +Justin Li +Justin Nuß Kamil Dziedzic +Kevin Malachowski +Kieron Woodhouse +Lennart Rudolph Leonardo YongUk Kim +Linh Tran Tuan +Lion Yang +Luca Looz Lucas Liu Luke Scott +Maciej Zimnoch Michael Woolnough Nicola Peduzzi +Olivier Mengué +oscarzhao +Paul Bonser +Peter Schultz +Rebecca Chin +Reed Allman +Richard Wilkes +Robert Russell Runrioter Wung +Shuode Li +Simon J Mudd Soroush Pour Stan Putrya +Stanley Gunawan +Steven Hartland +Thomas Wodarek +Tim Ruffles +Tom Jenkinson +Xiangyu Hu Xiaobing Jiang Xiuming Chen -Julien Lefevre +Zhenye Xie # Organizations Barracuda Networks, Inc. +Counting Ltd. +Facebook Inc. +GitHub Inc. Google Inc. +InfoSum Ltd. +Keybase Inc. +Multiplay Ltd. +Percona LLC +Pivotal Inc. Stripe Inc. diff --git a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md index 161ad0f..2d87d74 100644 --- a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md +++ b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -1,21 +1,96 @@ -## HEAD +## Version 1.4 (2018-06-03) + +Changes: + + - Documentation fixes (#530, #535, #567) + - Refactoring (#575, #579, #580, #581, #603, #615, #704) + - Cache column names (#444) + - Sort the DSN parameters in DSNs generated from a config (#637) + - Allow native password authentication by default (#644) + - Use the default port if it is missing in the DSN (#668) + - Removed the `strict` mode (#676) + - Do not query `max_allowed_packet` by default (#680) + - Dropped support Go 1.6 and lower (#696) + - Updated `ConvertValue()` to match the database/sql/driver implementation (#760) + - Document the usage of `0000-00-00T00:00:00` as the time.Time zero value (#783) + - Improved the compatibility of the authentication system (#807) + +New Features: + + - Multi-Results support (#537) + - `rejectReadOnly` DSN option (#604) + - `context.Context` support (#608, #612, #627, #761) + - Transaction isolation level support (#619, #744) + - Read-Only transactions support (#618, #634) + - `NewConfig` function which initializes a config with default values (#679) + - Implemented the `ColumnType` interfaces (#667, #724) + - Support for custom string types in `ConvertValue` (#623) + - Implemented `NamedValueChecker`, improving support for uint64 with high bit set (#690, #709, #710) + - `caching_sha2_password` authentication plugin support (#794, #800, #801, #802) + - Implemented `driver.SessionResetter` (#779) + - `sha256_password` authentication plugin support (#808) + +Bugfixes: + + - Use the DSN hostname as TLS default ServerName if `tls=true` (#564, #718) + - Fixed LOAD LOCAL DATA INFILE for empty files (#590) + - Removed columns definition cache since it sometimes cached invalid data (#592) + - Don't mutate registered TLS configs (#600) + - Make RegisterTLSConfig concurrency-safe (#613) + - Handle missing auth data in the handshake packet correctly (#646) + - Do not retry queries when data was written to avoid data corruption (#302, #736) + - Cache the connection pointer for error handling before invalidating it (#678) + - Fixed imports for appengine/cloudsql (#700) + - Fix sending STMT_LONG_DATA for 0 byte data (#734) + - Set correct capacity for []bytes read from length-encoded strings (#766) + - Make RegisterDial concurrency-safe (#773) + + +## Version 1.3 (2016-12-01) Changes: - Go 1.1 is no longer supported - - Use decimals field from MySQL to format time types (#249) + - Use decimals fields in MySQL to format time types (#249) - Buffer optimizations (#269) - TLS ServerName defaults to the host (#283) + - Refactoring (#400, #410, #437) + - Adjusted documentation for second generation CloudSQL (#485) + - Documented DSN system var quoting rules (#502) + - Made statement.Close() calls idempotent to avoid errors in Go 1.6+ (#512) + +New Features: + + - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) + - Support for returning table alias on Columns() (#289, #359, #382) + - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490) + - Support for uint64 parameters with high bit set (#332, #345) + - Cleartext authentication plugin support (#327) + - Exported ParseDSN function and the Config struct (#403, #419, #429) + - Read / Write timeouts (#401) + - Support for JSON field type (#414) + - Support for multi-statements and multi-results (#411, #431) + - DSN parameter to set the driver-side max_allowed_packet value manually (#489) + - Native password authentication plugin support (#494, #524) Bugfixes: - - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) - Fixed handling of queries without columns and rows (#255) - Fixed a panic when SetKeepAlive() failed (#298) - -New Features: - - Support for returning table alias on Columns() (#289) - - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318) + - Handle ERR packets while reading rows (#321) + - Fixed reading NULL length-encoded integers in MySQL 5.6+ (#349) + - Fixed absolute paths support in LOAD LOCAL DATA INFILE (#356) + - Actually zero out bytes in handshake response (#378) + - Fixed race condition in registering LOAD DATA INFILE handler (#383) + - Fixed tests with MySQL 5.7.9+ (#380) + - QueryUnescape TLS config names (#397) + - Fixed "broken pipe" error by writing to closed socket (#390) + - Fixed LOAD LOCAL DATA INFILE buffering (#424) + - Fixed parsing of floats into float64 when placeholders are used (#434) + - Fixed DSN tests with Go 1.7+ (#459) + - Handle ERR packets while waiting for EOF (#473) + - Invalidate connection on error while discarding additional results (#513) + - Allow terminating packets of length 0 (#516) ## Version 1.2 (2014-06-03) diff --git a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md index f87c198..8fe16bc 100644 --- a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md +++ b/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md @@ -4,28 +4,11 @@ Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed). -Please provide the following minimum information: -* Your Go-MySQL-Driver version (or git SHA) -* Your Go version (run `go version` in your console) -* A detailed issue description -* Error Log if present -* If possible, a short example - - ## Contributing Code By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file. Don't forget to add yourself to the AUTHORS file. -### Pull Requests Checklist - -Please check the following points before submitting your pull request: -- [x] Code compiles correctly -- [x] Created tests, if possible -- [x] All tests pass -- [x] Extended the README / documentation, if necessary -- [x] Added yourself to the AUTHORS file - ### Code Review Everyone is invited to review and comment on pull requests. diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md index 706b7ef..c6adf1d 100644 --- a/vendor/github.com/go-sql-driver/mysql/README.md +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -1,13 +1,9 @@ # Go-MySQL-Driver -A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) package +A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package ![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin") -**Latest stable Release:** [Version 1.2 (June 03, 2014)](https://github.com/go-sql-driver/mysql/releases) - -[![Build Status](https://travis-ci.org/go-sql-driver/mysql.png?branch=master)](https://travis-ci.org/go-sql-driver/mysql) - --------------------------------------- * [Features](#features) * [Requirements](#requirements) @@ -19,6 +15,9 @@ A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) packa * [Address](#address) * [Parameters](#parameters) * [Examples](#examples) + * [Connection pool and timeouts](#connection-pool-and-timeouts) + * [context.Context Support](#contextcontext-support) + * [ColumnType Support](#columntype-support) * [LOAD DATA LOCAL INFILE support](#load-data-local-infile-support) * [time.Time support](#timetime-support) * [Unicode support](#unicode-support) @@ -30,31 +29,31 @@ A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) packa ## Features * Lightweight and [fast](https://github.com/go-sql-driver/sql-benchmark "golang MySQL-Driver performance") * Native Go implementation. No C-bindings, just pure Go - * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](http://godoc.org/github.com/go-sql-driver/mysql#DialFunc) + * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](https://godoc.org/github.com/go-sql-driver/mysql#DialFunc) * Automatic handling of broken connections * Automatic Connection Pooling *(by database/sql package)* * Supports queries larger than 16MB - * Full [`sql.RawBytes`](http://golang.org/pkg/database/sql/#RawBytes) support. + * Full [`sql.RawBytes`](https://golang.org/pkg/database/sql/#RawBytes) support. * Intelligent `LONG DATA` handling in prepared statements * Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support * Optional `time.Time` parsing * Optional placeholder interpolation ## Requirements - * Go 1.2 or higher + * Go 1.9 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- ## Installation -Simple install the package to your [$GOPATH](http://code.google.com/p/go-wiki/wiki/GOPATH "GOPATH") with the [go tool](http://golang.org/cmd/go/ "go command") from shell: +Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell: ```bash -$ go get github.com/go-sql-driver/mysql +$ go get -u github.com/go-sql-driver/mysql ``` -Make sure [Git is installed](http://git-scm.com/downloads) on your machine and in your system's `PATH`. +Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`. ## Usage -_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](http://golang.org/pkg/database/sql) API then. +_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](https://golang.org/pkg/database/sql/) API then. Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: ```go @@ -93,17 +92,20 @@ This has the same effect as an empty DSN string: ``` +Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. + #### Password Passwords can consist of any character. Escaping is **not** necessary. #### Protocol -See [net.Dial](http://golang.org/pkg/net/#Dial) for more information which networks are available. +See [net.Dial](https://golang.org/pkg/net/#Dial) for more information which networks are available. In general you should use an Unix domain socket if available and TCP otherwise for best performance. #### Address -For TCP and UDP networks, addresses have the form `host:port`. +For TCP and UDP networks, addresses have the form `host[:port]`. +If `port` is omitted, the default port will be used. If `host` is a literal IPv6 address, it must be enclosed in square brackets. -The functions [net.JoinHostPort](http://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](http://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. +The functions [net.JoinHostPort](https://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](https://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. For Unix domain sockets the address is the absolute path to the MySQL-Server-socket, e.g. `/var/run/mysqld/mysqld.sock` or `/tmp/mysql.sock`. @@ -133,6 +135,15 @@ Default: false `allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. +##### `allowNativePasswords` + +``` +Type: bool +Valid Values: true, false +Default: true +``` +`allowNativePasswords=false` disallows the usage of MySQL native password method. + ##### `allowOldPasswords` ``` @@ -160,13 +171,18 @@ Unless you need the fallback behavior, please use `collation` instead. ``` Type: string Valid Values: -Default: utf8_general_ci +Default: utf8mb4_general_ci ``` Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. A list of valid charsets for a server is retrievable with `SHOW COLLATION`. +The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You should use an older collation (e.g. `utf8_general_ci`) for older MySQL. + +Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). + + ##### `clientFoundRows` ``` @@ -213,12 +229,31 @@ Valid Values: Default: UTC ``` -Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](http://golang.org/pkg/time/#LoadLocation) for details. +Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](https://golang.org/pkg/time/#LoadLocation) for details. Note that this sets the location for time.Time values but does not change MySQL's [time_zone setting](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html). For that see the [time_zone system variable](#system-variables), which can also be set as a DSN parameter. -Please keep in mind, that param values must be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. +Please keep in mind, that param values must be [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. +##### `maxAllowedPacket` +``` +Type: decimal number +Default: 4194304 +``` + +Max packet size allowed in bytes. The default value is 4 MiB and should be adjusted to match the server settings. `maxAllowedPacket=0` can be used to automatically fetch the `max_allowed_packet` variable from server *on every connection*. + +##### `multiStatements` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. + +When `multiStatements` is used, `?` parameters must only be used in the first statement. ##### `parseTime` @@ -229,9 +264,19 @@ Default: false ``` `parseTime=true` changes the output type of `DATE` and `DATETIME` values to `time.Time` instead of `[]byte` / `string` +The date or datetime like `0000-00-00 00:00:00` is converted into zero value of `time.Time`. -##### `strict` +##### `readTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O read timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + +##### `rejectReadOnly` ``` Type: bool @@ -239,41 +284,89 @@ Valid Values: true, false Default: false ``` -`strict=true` enables the strict mode in which MySQL warnings are treated as errors. -By default MySQL also treats notes as warnings. Use [`sql_notes=false`](http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sql_notes) to ignore notes. See the [examples](#examples) for an DSN example. +`rejectReadOnly=true` causes the driver to reject read-only connections. This +is for a possible race condition during an automatic failover, where the mysql +client gets connected to a read-only replica after the failover. + +Note that this should be a fairly rare case, as an automatic failover normally +happens when the primary is down, and the race condition shouldn't happen +unless it comes back up online as soon as the failover is kicked off. On the +other hand, when this happens, a MySQL application can get stuck on a +read-only connection until restarted. It is however fairly easy to reproduce, +for example, using a manual failover on AWS Aurora's MySQL-compatible cluster. + +If you are not relying on read-only transactions to reject writes that aren't +supposed to happen, setting this on some MySQL providers (such as AWS Aurora) +is safer for failovers. + +Note that ERROR 1290 can be returned for a `read-only` server and this option will +cause a retry for that error. However the same error number is used for some +other cases. You should ensure your application will never cause an ERROR 1290 +except for `read-only` mode when enabling this option. + + +##### `serverPubKey` + +``` +Type: string +Valid Values: +Default: none +``` + +Server public keys can be registered with [`mysql.RegisterServerPubKey`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterServerPubKey), which can then be used by the assigned name in the DSN. +Public keys are used to transmit encrypted data, e.g. for authentication. +If the server's public key is known, it should be set manually to avoid expensive and potentially insecure transmissions of the public key from the server to the client each time it is required. ##### `timeout` ``` -Type: decimal number +Type: duration Default: OS default ``` -*Driver* side connection timeout. The value must be a string of decimal numbers, each with optional fraction and a unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. To set a server side timeout, use the parameter [`wait_timeout`](http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_wait_timeout). +Timeout for establishing connections, aka dial timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. ##### `tls` ``` Type: bool / string -Valid Values: true, false, skip-verify, +Valid Values: true, false, skip-verify, preferred, Default: false ``` -`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use a custom value registered with [`mysql.RegisterTLSConfig`](http://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side) or use `preferred` to use TLS only when advertised by the server. This is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Neither `skip-verify` nor `preferred` add any reliable security. You can use a custom TLS config after registering it with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). + + +##### `writeTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O write timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. ##### System Variables -All other parameters are interpreted as system variables: - * `autocommit`: `"SET autocommit="` - * [`time_zone`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `"SET time_zone="` - * [`tx_isolation`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `"SET tx_isolation="` - * `param`: `"SET ="` +Any other parameters are interpreted as system variables: + * `=`: `SET =` + * `=`: `SET =` + * `=%27%27`: `SET =''` + +Rules: +* The values for string variables must be quoted with `'`. +* The values must also be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed! + (which implies values of string variables must be wrapped with `%27`). + +Examples: + * `autocommit=1`: `SET autocommit=1` + * [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'` + * [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'` -*The values must be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed!* #### Examples ``` @@ -288,9 +381,9 @@ root:pw@unix(/tmp/mysql.sock)/myDatabase?loc=Local user:password@tcp(localhost:5555)/dbname?tls=skip-verify&autocommit=true ``` -Use the [strict mode](#strict) but ignore notes: +Treat warnings as errors by setting the system variable [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html): ``` -user:password@/dbname?strict=true&sql_notes=false +user:password@/dbname?sql_mode=TRADITIONAL ``` TCP via IPv6: @@ -303,11 +396,16 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine: +Google Cloud SQL on App Engine (First Generation MySQL Server): ``` user@cloudsql(project-id:instance-name)/dbname ``` +Google Cloud SQL on App Engine (Second Generation MySQL Server): +``` +user@cloudsql(project-id:regionname:instance-name)/dbname +``` + TCP using default port (3306) on localhost: ``` user:password@tcp/dbname?charset=utf8mb4,utf8&sys_var=esc%40ped @@ -323,6 +421,18 @@ No Database preselected: user:password@/ ``` + +### Connection pool and timeouts +The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively. + +## `ColumnType` Support +This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. + +## `context.Context` Support +Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts. +See [context support in the database/sql package](https://golang.org/doc/go1.8#database_sql) for more details. + + ### `LOAD DATA LOCAL INFILE` support For this feature you need direct access to the package. Therefore you must change the import path (no `_`): ```go @@ -333,17 +443,17 @@ Files must be whitelisted by registering them with `mysql.RegisterLocalFile(file To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. -See the [godoc of Go-MySQL-Driver](http://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. +See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. ### `time.Time` support -The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your programm. +The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program. -However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](http://golang.org/pkg/time/#Location) with the `loc` DSN parameter. +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical equivalent in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. **Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). -Alternatively you can use the [`NullTime`](http://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. +Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. ### Unicode support @@ -355,7 +465,6 @@ Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAM See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. - ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. @@ -374,13 +483,13 @@ Mozilla summarizes the license scope as follows: That means: - * You can **use** the **unchanged** source code both in private and commercially - * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0) - * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged** + * You can **use** the **unchanged** source code both in private and commercially. + * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0). + * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged**. -Please read the [MPL 2.0 FAQ](http://www.mozilla.org/MPL/2.0/FAQ.html) if you have further questions regarding the license. +Please read the [MPL 2.0 FAQ](https://www.mozilla.org/en-US/MPL/2.0/FAQ/) if you have further questions regarding the license. -You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) +You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE). ![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow") diff --git a/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/go-sql-driver/mysql/appengine.go index 565614e..914e662 100644 --- a/vendor/github.com/go-sql-driver/mysql/appengine.go +++ b/vendor/github.com/go-sql-driver/mysql/appengine.go @@ -11,9 +11,15 @@ package mysql import ( - "appengine/cloudsql" + "context" + "net" + + "google.golang.org/appengine/cloudsql" ) func init() { - RegisterDial("cloudsql", cloudsql.Dial) + RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { + // XXX: the cloudsql driver still does not export a Context-aware dialer. + return cloudsql.Dial(instance) + }) } diff --git a/vendor/github.com/go-sql-driver/mysql/auth.go b/vendor/github.com/go-sql-driver/mysql/auth.go new file mode 100644 index 0000000..fec7040 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/auth.go @@ -0,0 +1,422 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/x509" + "encoding/pem" + "sync" +) + +// server pub keys registry +var ( + serverPubKeyLock sync.RWMutex + serverPubKeyRegistry map[string]*rsa.PublicKey +) + +// RegisterServerPubKey registers a server RSA public key which can be used to +// send data in a secure manner to the server without receiving the public key +// in a potentially insecure way from the server first. +// Registered keys can afterwards be used adding serverPubKey= to the DSN. +// +// Note: The provided rsa.PublicKey instance is exclusively owned by the driver +// after registering it and may not be modified. +// +// data, err := ioutil.ReadFile("mykey.pem") +// if err != nil { +// log.Fatal(err) +// } +// +// block, _ := pem.Decode(data) +// if block == nil || block.Type != "PUBLIC KEY" { +// log.Fatal("failed to decode PEM block containing public key") +// } +// +// pub, err := x509.ParsePKIXPublicKey(block.Bytes) +// if err != nil { +// log.Fatal(err) +// } +// +// if rsaPubKey, ok := pub.(*rsa.PublicKey); ok { +// mysql.RegisterServerPubKey("mykey", rsaPubKey) +// } else { +// log.Fatal("not a RSA public key") +// } +// +func RegisterServerPubKey(name string, pubKey *rsa.PublicKey) { + serverPubKeyLock.Lock() + if serverPubKeyRegistry == nil { + serverPubKeyRegistry = make(map[string]*rsa.PublicKey) + } + + serverPubKeyRegistry[name] = pubKey + serverPubKeyLock.Unlock() +} + +// DeregisterServerPubKey removes the public key registered with the given name. +func DeregisterServerPubKey(name string) { + serverPubKeyLock.Lock() + if serverPubKeyRegistry != nil { + delete(serverPubKeyRegistry, name) + } + serverPubKeyLock.Unlock() +} + +func getServerPubKey(name string) (pubKey *rsa.PublicKey) { + serverPubKeyLock.RLock() + if v, ok := serverPubKeyRegistry[name]; ok { + pubKey = v + } + serverPubKeyLock.RUnlock() + return +} + +// Hash password using pre 4.1 (old password) method +// https://github.com/atcurtis/mariadb/blob/master/mysys/my_rnd.c +type myRnd struct { + seed1, seed2 uint32 +} + +const myRndMaxVal = 0x3FFFFFFF + +// Pseudo random number generator +func newMyRnd(seed1, seed2 uint32) *myRnd { + return &myRnd{ + seed1: seed1 % myRndMaxVal, + seed2: seed2 % myRndMaxVal, + } +} + +// Tested to be equivalent to MariaDB's floating point variant +// http://play.golang.org/p/QHvhd4qved +// http://play.golang.org/p/RG0q4ElWDx +func (r *myRnd) NextByte() byte { + r.seed1 = (r.seed1*3 + r.seed2) % myRndMaxVal + r.seed2 = (r.seed1 + r.seed2 + 33) % myRndMaxVal + + return byte(uint64(r.seed1) * 31 / myRndMaxVal) +} + +// Generate binary hash from byte string using insecure pre 4.1 method +func pwHash(password []byte) (result [2]uint32) { + var add uint32 = 7 + var tmp uint32 + + result[0] = 1345345333 + result[1] = 0x12345671 + + for _, c := range password { + // skip spaces and tabs in password + if c == ' ' || c == '\t' { + continue + } + + tmp = uint32(c) + result[0] ^= (((result[0] & 63) + add) * tmp) + (result[0] << 8) + result[1] += (result[1] << 8) ^ result[0] + add += tmp + } + + // Remove sign bit (1<<31)-1) + result[0] &= 0x7FFFFFFF + result[1] &= 0x7FFFFFFF + + return +} + +// Hash password using insecure pre 4.1 method +func scrambleOldPassword(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + scramble = scramble[:8] + + hashPw := pwHash([]byte(password)) + hashSc := pwHash(scramble) + + r := newMyRnd(hashPw[0]^hashSc[0], hashPw[1]^hashSc[1]) + + var out [8]byte + for i := range out { + out[i] = r.NextByte() + 64 + } + + mask := r.NextByte() + for i := range out { + out[i] ^= mask + } + + return out[:] +} + +// Hash password using 4.1+ method (SHA1) +func scramblePassword(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + // stage1Hash = SHA1(password) + crypt := sha1.New() + crypt.Write([]byte(password)) + stage1 := crypt.Sum(nil) + + // scrambleHash = SHA1(scramble + SHA1(stage1Hash)) + // inner Hash + crypt.Reset() + crypt.Write(stage1) + hash := crypt.Sum(nil) + + // outer Hash + crypt.Reset() + crypt.Write(scramble) + crypt.Write(hash) + scramble = crypt.Sum(nil) + + // token = scrambleHash XOR stage1Hash + for i := range scramble { + scramble[i] ^= stage1[i] + } + return scramble +} + +// Hash password using MySQL 8+ method (SHA256) +func scrambleSHA256Password(scramble []byte, password string) []byte { + if len(password) == 0 { + return nil + } + + // XOR(SHA256(password), SHA256(SHA256(SHA256(password)), scramble)) + + crypt := sha256.New() + crypt.Write([]byte(password)) + message1 := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1) + message1Hash := crypt.Sum(nil) + + crypt.Reset() + crypt.Write(message1Hash) + crypt.Write(scramble) + message2 := crypt.Sum(nil) + + for i := range message1 { + message1[i] ^= message2[i] + } + + return message1 +} + +func encryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, error) { + plain := make([]byte, len(password)+1) + copy(plain, password) + for i := range plain { + j := i % len(seed) + plain[i] ^= seed[j] + } + sha1 := sha1.New() + return rsa.EncryptOAEP(sha1, rand.Reader, pub, plain, nil) +} + +func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) error { + enc, err := encryptPassword(mc.cfg.Passwd, seed, pub) + if err != nil { + return err + } + return mc.writeAuthSwitchPacket(enc) +} + +func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { + switch plugin { + case "caching_sha2_password": + authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) + return authResp, nil + + case "mysql_old_password": + if !mc.cfg.AllowOldPasswords { + return nil, ErrOldPassword + } + // Note: there are edge cases where this should work but doesn't; + // this is currently "wontfix": + // https://github.com/go-sql-driver/mysql/issues/184 + authResp := append(scrambleOldPassword(authData[:8], mc.cfg.Passwd), 0) + return authResp, nil + + case "mysql_clear_password": + if !mc.cfg.AllowCleartextPasswords { + return nil, ErrCleartextPassword + } + // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html + // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html + return append([]byte(mc.cfg.Passwd), 0), nil + + case "mysql_native_password": + if !mc.cfg.AllowNativePasswords { + return nil, ErrNativePassword + } + // https://dev.mysql.com/doc/internals/en/secure-password-authentication.html + // Native password authentication only need and will need 20-byte challenge. + authResp := scramblePassword(authData[:20], mc.cfg.Passwd) + return authResp, nil + + case "sha256_password": + if len(mc.cfg.Passwd) == 0 { + return []byte{0}, nil + } + if mc.cfg.tls != nil || mc.cfg.Net == "unix" { + // write cleartext auth packet + return append([]byte(mc.cfg.Passwd), 0), nil + } + + pubKey := mc.cfg.pubKey + if pubKey == nil { + // request public key from server + return []byte{1}, nil + } + + // encrypted password + enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) + return enc, err + + default: + errLog.Print("unknown auth plugin:", plugin) + return nil, ErrUnknownPlugin + } +} + +func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { + // Read Result Packet + authData, newPlugin, err := mc.readAuthResult() + if err != nil { + return err + } + + // handle auth plugin switch, if requested + if newPlugin != "" { + // If CLIENT_PLUGIN_AUTH capability is not supported, no new cipher is + // sent and we have to keep using the cipher sent in the init packet. + if authData == nil { + authData = oldAuthData + } else { + // copy data from read buffer to owned slice + copy(oldAuthData, authData) + } + + plugin = newPlugin + + authResp, err := mc.auth(authData, plugin) + if err != nil { + return err + } + if err = mc.writeAuthSwitchPacket(authResp); err != nil { + return err + } + + // Read Result Packet + authData, newPlugin, err = mc.readAuthResult() + if err != nil { + return err + } + + // Do not allow to change the auth plugin more than once + if newPlugin != "" { + return ErrMalformPkt + } + } + + switch plugin { + + // https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/ + case "caching_sha2_password": + switch len(authData) { + case 0: + return nil // auth successful + case 1: + switch authData[0] { + case cachingSha2PasswordFastAuthSuccess: + if err = mc.readResultOK(); err == nil { + return nil // auth successful + } + + case cachingSha2PasswordPerformFullAuthentication: + if mc.cfg.tls != nil || mc.cfg.Net == "unix" { + // write cleartext auth packet + err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0)) + if err != nil { + return err + } + } else { + pubKey := mc.cfg.pubKey + if pubKey == nil { + // request public key from server + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + return err + } + data[4] = cachingSha2PasswordRequestPublicKey + mc.writePacket(data) + + // parse public key + if data, err = mc.readPacket(); err != nil { + return err + } + + block, _ := pem.Decode(data[1:]) + pkix, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + pubKey = pkix.(*rsa.PublicKey) + } + + // send encrypted password + err = mc.sendEncryptedPassword(oldAuthData, pubKey) + if err != nil { + return err + } + } + return mc.readResultOK() + + default: + return ErrMalformPkt + } + default: + return ErrMalformPkt + } + + case "sha256_password": + switch len(authData) { + case 0: + return nil // auth successful + default: + block, _ := pem.Decode(authData) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + + // send encrypted password + err = mc.sendEncryptedPassword(oldAuthData, pub.(*rsa.PublicKey)) + if err != nil { + return err + } + return mc.readResultOK() + } + + default: + return nil // auth successful + } + + return err +} diff --git a/vendor/github.com/go-sql-driver/mysql/auth_test.go b/vendor/github.com/go-sql-driver/mysql/auth_test.go new file mode 100644 index 0000000..1920ef3 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/auth_test.go @@ -0,0 +1,1330 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "testing" +) + +var testPubKey = []byte("-----BEGIN PUBLIC KEY-----\n" + + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAol0Z8G8U+25Btxk/g/fm\n" + + "UAW/wEKjQCTjkibDE4B+qkuWeiumg6miIRhtilU6m9BFmLQSy1ltYQuu4k17A4tQ\n" + + "rIPpOQYZges/qsDFkZh3wyK5jL5WEFVdOasf6wsfszExnPmcZS4axxoYJfiuilrN\n" + + "hnwinBAqfi3S0sw5MpSI4Zl1AbOrHG4zDI62Gti2PKiMGyYDZTS9xPrBLbN95Kby\n" + + "FFclQLEzA9RJcS1nHFsWtRgHjGPhhjCQxEm9NQ1nePFhCfBfApyfH1VM2VCOQum6\n" + + "Ci9bMuHWjTjckC84mzF99kOxOWVU7mwS6gnJqBzpuz8t3zq8/iQ2y7QrmZV+jTJP\n" + + "WQIDAQAB\n" + + "-----END PUBLIC KEY-----\n") + +var testPubKeyRSA *rsa.PublicKey + +func init() { + block, _ := pem.Decode(testPubKey) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + panic(err) + } + testPubKeyRSA = pub.(*rsa.PublicKey) +} + +func TestScrambleOldPass(t *testing.T) { + scramble := []byte{9, 8, 7, 6, 5, 4, 3, 2} + vectors := []struct { + pass string + out string + }{ + {" pass", "47575c5a435b4251"}, + {"pass ", "47575c5a435b4251"}, + {"123\t456", "575c47505b5b5559"}, + {"C0mpl!ca ted#PASS123", "5d5d554849584a45"}, + } + for _, tuple := range vectors { + ours := scrambleOldPassword(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed old password %q", tuple.pass) + } + } +} + +func TestScrambleSHA256Pass(t *testing.T) { + scramble := []byte{10, 47, 74, 111, 75, 73, 34, 48, 88, 76, 114, 74, 37, 13, 3, 80, 82, 2, 23, 21} + vectors := []struct { + pass string + out string + }{ + {"secret", "f490e76f66d9d86665ce54d98c78d0acfe2fb0b08b423da807144873d30b312c"}, + {"secret2", "abc3934a012cf342e876071c8ee202de51785b430258a7a0138bc79c4d800bc6"}, + } + for _, tuple := range vectors { + ours := scrambleSHA256Password(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed SHA256 password %q", tuple.pass) + } + } +} + +func TestAuthFastCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{102, 32, 5, 35, 143, 161, 140, 241, 171, 232, 56, + 139, 43, 14, 107, 196, 249, 170, 147, 60, 220, 204, 120, 178, 214, 15, + 184, 150, 26, 61, 57, 235} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 3, // Fast Auth Success + 7, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // pub key response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{1, 0, 0, 3, 2, 0, 1, 0, 5}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCleartextPasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthFastCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowNativePasswords = false + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthFastNativePassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{53, 177, 140, 159, 251, 189, 127, 53, 109, 252, + 172, 50, 211, 192, 240, 164, 26, 48, 207, 45} + if writtenAuthRespLen != 20 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{1} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + // hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + + // unset TLS config to prevent the actual establishment of a TLS wrapper + mc.cfg.tls = nil + + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 6, 1}, testPubKey...), + + // OK + {7, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 4 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Pub Key Request + 1, 0, 0, 5, 2, + + // 3. Packet: Encrypted Password + 0, 1, 0, 7, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {2, 0, 0, 4, 1, 4}, // Perform Full Authentication + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Cleartext password + 7, 0, 0, 5, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + conn.maxReads = 1 + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthSwitchCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = false + + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + conn.maxReads = 1 + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthSwitchNativePassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{20, 0, 0, 3, 202, 41, 195, 164, 34, 226, 49, 103, + 21, 211, 167, 199, 227, 116, 8, 48, 57, 71, 149, 146} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchOldPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +// Same to TestAuthSwitchOldPasswordNotAllowed, but use OldAuthSwitch request. +func TestOldAuthSwitchNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +func TestAuthSwitchOldPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPassword, but use OldAuthSwitch request. +func TestOldAuthSwitch(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} +func TestAuthSwitchOldPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPasswordEmpty, but use OldAuthSwitch request. +func TestOldAuthSwitchPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // OldAuthSwitch request. + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Empty Password + 1, 0, 0, 3, 0, + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Pub Key Request + 1, 0, 0, 3, 1, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Encrypted Password + 0, 1, 0, 3, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Cleartext Password + 7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/benchmark_test.go b/vendor/github.com/go-sql-driver/mysql/benchmark_test.go new file mode 100644 index 0000000..3e25a3b --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/benchmark_test.go @@ -0,0 +1,373 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "fmt" + "math" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +type TB testing.B + +func (tb *TB) check(err error) { + if err != nil { + tb.Fatal(err) + } +} + +func (tb *TB) checkDB(db *sql.DB, err error) *sql.DB { + tb.check(err) + return db +} + +func (tb *TB) checkRows(rows *sql.Rows, err error) *sql.Rows { + tb.check(err) + return rows +} + +func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt { + tb.check(err) + return stmt +} + +func initDB(b *testing.B, queries ...string) *sql.DB { + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + for _, query := range queries { + if _, err := db.Exec(query); err != nil { + b.Fatalf("error on %q: %v", query, err) + } + } + return db +} + +const concurrencyLevel = 10 + +func BenchmarkQuery(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + var got string + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Errorf("query = %q; want one", got) + wg.Done() + return + } + } + }() + } +} + +func BenchmarkExec(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := tb.checkDB(sql.Open("mysql", dsn)) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("DO 1")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + if _, err := stmt.Exec(); err != nil { + b.Fatal(err.Error()) + } + } + }() + } +} + +// data, but no db writes +var roundtripSample []byte + +func initRoundtripBenchmarks() ([]byte, int, int) { + if roundtripSample == nil { + roundtripSample = []byte(strings.Repeat("0123456789abcdef", 1024*1024)) + } + return roundtripSample, 16, len(roundtripSample) +} + +func BenchmarkRoundtripTxt(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + sampleString := string(sample) + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + b.StartTimer() + var result string + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sampleString[0:length] + rows := tb.checkRows(db.Query(`SELECT "` + test + `"`)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if result != test { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkRoundtripBin(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + stmt := tb.checkStmt(db.Prepare("SELECT ?")) + defer stmt.Close() + b.StartTimer() + var result sql.RawBytes + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sample[0:length] + rows := tb.checkRows(stmt.Query(test)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if !bytes.Equal(result, test) { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkInterpolation(b *testing.B) { + mc := &mysqlConn{ + cfg: &Config{ + InterpolateParams: true, + Loc: time.UTC, + }, + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + buf: newBuffer(nil), + } + + args := []driver.Value{ + int64(42424242), + float64(math.Pi), + false, + time.Unix(1423411542, 807015000), + []byte("bytes containing special chars ' \" \a \x00"), + "string containing special chars ' \" \a \x00", + } + q := "SELECT ?, ?, ?, ?, ?, ?" + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := mc.interpolateParams(q, args) + if err != nil { + b.Fatal(err) + } + } +} + +func benchmarkQueryContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + var got string + for pb.Next() { + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Fatalf("query = %q; want one", got) + } + } + }) +} + +func BenchmarkQueryContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +func benchmarkExecContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "DO 1")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := stmt.ExecContext(ctx); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkExecContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +// BenchmarkQueryRawBytes benchmarks fetching 100 blobs using sql.RawBytes. +// "size=" means size of each blobs. +func BenchmarkQueryRawBytes(b *testing.B) { + var sizes []int = []int{100, 1000, 2000, 4000, 8000, 12000, 16000, 32000, 64000, 256000} + db := initDB(b, + "DROP TABLE IF EXISTS bench_rawbytes", + "CREATE TABLE bench_rawbytes (id INT PRIMARY KEY, val LONGBLOB)", + ) + defer db.Close() + + blob := make([]byte, sizes[len(sizes)-1]) + for i := range blob { + blob[i] = 42 + } + for i := 0; i < 100; i++ { + _, err := db.Exec("INSERT INTO bench_rawbytes VALUES (?, ?)", i, blob) + if err != nil { + b.Fatal(err) + } + } + + for _, s := range sizes { + b.Run(fmt.Sprintf("size=%v", s), func(b *testing.B) { + db.SetMaxIdleConns(0) + db.SetMaxIdleConns(1) + b.ReportAllocs() + b.ResetTimer() + + for j := 0; j < b.N; j++ { + rows, err := db.Query("SELECT LEFT(val, ?) as v FROM bench_rawbytes", s) + if err != nil { + b.Fatal(err) + } + nrows := 0 + for rows.Next() { + var buf sql.RawBytes + err := rows.Scan(&buf) + if err != nil { + b.Fatal(err) + } + if len(buf) != s { + b.Fatalf("size mismatch: expected %v, got %v", s, len(buf)) + } + nrows++ + } + rows.Close() + if nrows != 100 { + b.Fatalf("numbers of rows mismatch: expected %v, got %v", 100, nrows) + } + } + }) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/go-sql-driver/mysql/buffer.go index 509ce89..0774c5c 100644 --- a/vendor/github.com/go-sql-driver/mysql/buffer.go +++ b/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -8,53 +8,86 @@ package mysql -import "io" +import ( + "io" + "net" + "time" +) const defaultBufSize = 4096 +const maxCachedBufSize = 256 * 1024 // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. +// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { - buf []byte - rd io.Reader - idx int - length int + buf []byte // buf is a byte buffer who's length and capacity are equal. + nc net.Conn + idx int + length int + timeout time.Duration + dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer + flipcnt uint // flipccnt is the current buffer counter for double-buffering } -func newBuffer(rd io.Reader) buffer { - var b [defaultBufSize]byte +// newBuffer allocates and returns a new buffer. +func newBuffer(nc net.Conn) buffer { + fg := make([]byte, defaultBufSize) return buffer{ - buf: b[:], - rd: rd, + buf: fg, + nc: nc, + dbuf: [2][]byte{fg, nil}, } } +// flip replaces the active buffer with the background buffer +// this is a delayed flip that simply increases the buffer counter; +// the actual flip will be performed the next time we call `buffer.fill` +func (b *buffer) flip() { + b.flipcnt += 1 +} + // fill reads into the buffer until at least _need_ bytes are in it func (b *buffer) fill(need int) error { n := b.length + // fill data into its double-buffering target: if we've called + // flip on this buffer, we'll be copying to the background buffer, + // and then filling it with network data; otherwise we'll just move + // the contents of the current buffer to the front before filling it + dest := b.dbuf[b.flipcnt&1] - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) - } - - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { + // grow buffer if necessary to fit the whole packet. + if need > len(dest) { // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf + dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) + + // if the allocated buffer is not too large, move it to backing storage + // to prevent extra allocations on applications that perform large reads + if len(dest) <= maxCachedBufSize { + b.dbuf[b.flipcnt&1] = dest + } } + // if we're filling the fg buffer, move the existing data to the start of it. + // if we're filling the bg buffer, copy over the data + if n > 0 { + copy(dest[:n], b.buf[b.idx:]) + } + + b.buf = dest b.idx = 0 for { - nn, err := b.rd.Read(b.buf[n:]) + if b.timeout > 0 { + if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil { + return err + } + } + + nn, err := b.nc.Read(b.buf[n:]) n += nn switch err { @@ -94,43 +127,56 @@ func (b *buffer) readNext(need int) ([]byte, error) { return b.buf[offset:b.idx], nil } -// returns a buffer with the requested size. +// takeBuffer returns a buffer with the requested size. // If possible, a slice from the existing buffer is returned. // Otherwise a bigger buffer is made. // Only one buffer (total) can be used at a time. -func (b *buffer) takeBuffer(length int) []byte { +func (b *buffer) takeBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } // test (cheap) general case first - if length <= defaultBufSize || length <= cap(b.buf) { - return b.buf[:length] + if length <= cap(b.buf) { + return b.buf[:length], nil } if length < maxPacketSize { b.buf = make([]byte, length) - return b.buf + return b.buf, nil } - return make([]byte, length) + + // buffer is larger than we want to store. + return make([]byte, length), nil } -// shortcut which can be used if the requested buffer is guaranteed to be -// smaller than defaultBufSize +// takeSmallBuffer is shortcut which can be used if length is +// known to be smaller than defaultBufSize. // Only one buffer (total) can be used at a time. -func (b *buffer) takeSmallBuffer(length int) []byte { - if b.length == 0 { - return b.buf[:length] +func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { + if b.length > 0 { + return nil, ErrBusyBuffer } - return nil + return b.buf[:length], nil } // takeCompleteBuffer returns the complete existing buffer. // This can be used if the necessary buffer size is unknown. +// cap and len of the returned buffer will be equal. // Only one buffer (total) can be used at a time. -func (b *buffer) takeCompleteBuffer() []byte { - if b.length == 0 { - return b.buf +func (b *buffer) takeCompleteBuffer() ([]byte, error) { + if b.length > 0 { + return nil, ErrBusyBuffer + } + return b.buf, nil +} + +// store stores buf, an updated buffer, if its suitable to do so. +func (b *buffer) store(buf []byte) error { + if b.length > 0 { + return ErrBusyBuffer + } else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) { + b.buf = buf[:cap(buf)] } return nil } diff --git a/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/go-sql-driver/mysql/collations.go index 6c1d613..8d2b556 100644 --- a/vendor/github.com/go-sql-driver/mysql/collations.go +++ b/vendor/github.com/go-sql-driver/mysql/collations.go @@ -8,182 +8,190 @@ package mysql -const defaultCollation byte = 33 // utf8_general_ci +const defaultCollation = "utf8mb4_general_ci" +const binaryCollation = "binary" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. +// +// ucs2, utf16, and utf32 can't be used for connection charset. +// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset +// They are commented out to reduce this map. var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + //"ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + //"utf16_general_ci": 54, + //"utf16_bin": 55, + //"utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + //"utf32_general_ci": 60, + //"utf32_bin": 61, + //"utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "utf8_tolower_ci": 76, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + //"ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + //"utf16_unicode_ci": 101, + //"utf16_icelandic_ci": 102, + //"utf16_latvian_ci": 103, + //"utf16_romanian_ci": 104, + //"utf16_slovenian_ci": 105, + //"utf16_polish_ci": 106, + //"utf16_estonian_ci": 107, + //"utf16_spanish_ci": 108, + //"utf16_swedish_ci": 109, + //"utf16_turkish_ci": 110, + //"utf16_czech_ci": 111, + //"utf16_danish_ci": 112, + //"utf16_lithuanian_ci": 113, + //"utf16_slovak_ci": 114, + //"utf16_spanish2_ci": 115, + //"utf16_roman_ci": 116, + //"utf16_persian_ci": 117, + //"utf16_esperanto_ci": 118, + //"utf16_hungarian_ci": 119, + //"utf16_sinhala_ci": 120, + //"utf16_german2_ci": 121, + //"utf16_croatian_ci": 122, + //"utf16_unicode_520_ci": 123, + //"utf16_vietnamese_ci": 124, + //"ucs2_unicode_ci": 128, + //"ucs2_icelandic_ci": 129, + //"ucs2_latvian_ci": 130, + //"ucs2_romanian_ci": 131, + //"ucs2_slovenian_ci": 132, + //"ucs2_polish_ci": 133, + //"ucs2_estonian_ci": 134, + //"ucs2_spanish_ci": 135, + //"ucs2_swedish_ci": 136, + //"ucs2_turkish_ci": 137, + //"ucs2_czech_ci": 138, + //"ucs2_danish_ci": 139, + //"ucs2_lithuanian_ci": 140, + //"ucs2_slovak_ci": 141, + //"ucs2_spanish2_ci": 142, + //"ucs2_roman_ci": 143, + //"ucs2_persian_ci": 144, + //"ucs2_esperanto_ci": 145, + //"ucs2_hungarian_ci": 146, + //"ucs2_sinhala_ci": 147, + //"ucs2_german2_ci": 148, + //"ucs2_croatian_ci": 149, + //"ucs2_unicode_520_ci": 150, + //"ucs2_vietnamese_ci": 151, + //"ucs2_general_mysql500_ci": 159, + //"utf32_unicode_ci": 160, + //"utf32_icelandic_ci": 161, + //"utf32_latvian_ci": 162, + //"utf32_romanian_ci": 163, + //"utf32_slovenian_ci": 164, + //"utf32_polish_ci": 165, + //"utf32_estonian_ci": 166, + //"utf32_spanish_ci": 167, + //"utf32_swedish_ci": 168, + //"utf32_turkish_ci": 169, + //"utf32_czech_ci": 170, + //"utf32_danish_ci": 171, + //"utf32_lithuanian_ci": 172, + //"utf32_slovak_ci": 173, + //"utf32_spanish2_ci": 174, + //"utf32_roman_ci": 175, + //"utf32_persian_ci": 176, + //"utf32_esperanto_ci": 177, + //"utf32_hungarian_ci": 178, + //"utf32_sinhala_ci": 179, + //"utf32_german2_ci": 180, + //"utf32_croatian_ci": 181, + //"utf32_unicode_520_ci": 182, + //"utf32_vietnamese_ci": 183, "utf8_unicode_ci": 192, "utf8_icelandic_ci": 193, "utf8_latvian_ci": 194, @@ -233,18 +241,25 @@ var collations = map[string]byte{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, + "gb18030_unicode_520_ci": 250, + "utf8mb4_0900_ai_ci": 255, } // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. -var unsafeCollations = map[byte]bool{ - 1: true, // big5_chinese_ci - 13: true, // sjis_japanese_ci - 28: true, // gbk_chinese_ci - 84: true, // big5_bin - 86: true, // gb2312_bin - 87: true, // gbk_bin - 88: true, // sjis_bin - 95: true, // cp932_japanese_ci - 96: true, // cp932_bin +var unsafeCollations = map[string]bool{ + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, + "gb18030_chinese_ci": true, + "gb18030_bin": true, + "gb18030_unicode_520_ci": true, } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/go-sql-driver/mysql/conncheck.go new file mode 100644 index 0000000..cc47aa5 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -0,0 +1,53 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows,!appengine + +package mysql + +import ( + "errors" + "io" + "net" + "syscall" +) + +var errUnexpectedRead = errors.New("unexpected read from socket") + +func connCheck(c net.Conn) error { + var ( + n int + err error + buff [1]byte + ) + + sconn, ok := c.(syscall.Conn) + if !ok { + return nil + } + rc, err := sconn.SyscallConn() + if err != nil { + return err + } + rerr := rc.Read(func(fd uintptr) bool { + n, err = syscall.Read(int(fd), buff[:]) + return true + }) + switch { + case rerr != nil: + return rerr + case n == 0 && err == nil: + return io.EOF + case n > 0: + return errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + return nil + default: + return err + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go new file mode 100644 index 0000000..fd01f64 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -0,0 +1,17 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows appengine + +package mysql + +import "net" + +func connCheck(c net.Conn) error { + return nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go new file mode 100644 index 0000000..b7234b0 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go @@ -0,0 +1,38 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10,!windows + +package mysql + +import ( + "testing" + "time" +) + +func TestStaleConnectionChecks(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("SET @@SESSION.wait_timeout = 2") + + if err := dbt.db.Ping(); err != nil { + dbt.Fatal(err) + } + + // wait for MySQL to close our connection + time.Sleep(3 * time.Second) + + tx, err := dbt.db.Begin() + if err != nil { + dbt.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + dbt.Fatal(err) + } + }) +} diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index 72ed09d..565a548 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -9,9 +9,10 @@ package mysql import ( - "crypto/tls" + "context" + "database/sql" "database/sql/driver" - "errors" + "io" "net" "strconv" "strings" @@ -21,40 +22,31 @@ import ( type mysqlConn struct { buf buffer netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. affectedRows uint64 insertId uint64 - cfg *config - maxPacketAllowed int + cfg *Config + maxAllowedPacket int maxWriteSize int + writeTimeout time.Duration flags clientFlag status statusFlag sequence uint8 parseTime bool - strict bool -} + reset bool // set when the Go SQL package calls ResetSession -type config struct { - user string - passwd string - net string - addr string - dbname string - params map[string]string - loc *time.Location - tls *tls.Config - timeout time.Duration - collation uint8 - allowAllFiles bool - allowOldPasswords bool - allowCleartextPasswords bool - clientFoundRows bool - columnsWithAlias bool - interpolateParams bool + // for context support (Go 1.8+) + watching bool + watcher chan<- context.Context + closech chan struct{} + finished chan<- struct{} + canceled atomicError // set non-nil if conn is canceled + closed atomicBool // set when conn is closed, before closech is closed } // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { - for param, val := range mc.cfg.params { + for param, val := range mc.cfg.Params { switch param { // Charset case "charset": @@ -70,27 +62,6 @@ func (mc *mysqlConn) handleParams() (err error) { return } - // time.Time parsing - case "parseTime": - var isBool bool - mc.parseTime, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Strict mode - case "strict": - var isBool bool - mc.strict, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Compression - case "compress": - err = errors.New("Compression not implemented yet") - return - // System Vars default: err = mc.exec("SET " + param + "=" + val + "") @@ -103,46 +74,87 @@ func (mc *mysqlConn) handleParams() (err error) { return } +func (mc *mysqlConn) markBadConn(err error) error { + if mc == nil { + return err + } + if err != errBadConnNoWrite { + return err + } + return driver.ErrBadConn +} + func (mc *mysqlConn) Begin() (driver.Tx, error) { - if mc.netConn == nil { + return mc.begin(false) +} + +func (mc *mysqlConn) begin(readOnly bool) (driver.Tx, error) { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } - err := mc.exec("START TRANSACTION") + var q string + if readOnly { + q = "START TRANSACTION READ ONLY" + } else { + q = "START TRANSACTION" + } + err := mc.exec(q) if err == nil { return &mysqlTx{mc}, err } - - return nil, err + return nil, mc.markBadConn(err) } func (mc *mysqlConn) Close() (err error) { // Makes Close idempotent - if mc.netConn != nil { + if !mc.closed.IsSet() { err = mc.writeCommandPacket(comQuit) - if err == nil { - err = mc.netConn.Close() - } else { - mc.netConn.Close() - } - mc.netConn = nil } - mc.cfg = nil - mc.buf.rd = nil + mc.cleanup() return } -func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { +// Closes the network connection and unsets internal variables. Do not call this +// function after successfully authentication, call Close instead. This function +// is called before auth or on auth failure because MySQL will have already +// closed the network connection. +func (mc *mysqlConn) cleanup() { + if !mc.closed.TrySet(true) { + return + } + + // Makes cleanup idempotent + close(mc.closech) if mc.netConn == nil { + return + } + if err := mc.netConn.Close(); err != nil { + errLog.Print(err) + } +} + +func (mc *mysqlConn) error() error { + if mc.closed.IsSet() { + if err := mc.canceled.Value(); err != nil { + return err + } + return ErrInvalidConn + } + return nil +} + +func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, err + return nil, mc.markBadConn(err) } stmt := &mysqlStmt{ @@ -167,11 +179,16 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { } func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) { - buf := mc.buf.takeCompleteBuffer() - if buf == nil { + // Number of ? should be same to len(args) + if strings.Count(query, "?") != len(args) { + return "", driver.ErrSkip + } + + buf, err := mc.buf.takeCompleteBuffer() + if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return "", driver.ErrBadConn + errLog.Print(err) + return "", ErrInvalidConn } buf = buf[:0] argPos := 0 @@ -196,6 +213,9 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin switch v := arg.(type) { case int64: buf = strconv.AppendInt(buf, v, 10) + case uint64: + // Handle uint64 explicitly because our custom ConvertValue emits unsigned values + buf = strconv.AppendUint(buf, v, 10) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: @@ -208,7 +228,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { - v := v.In(mc.cfg.loc) + v := v.In(mc.cfg.Loc) v = v.Add(time.Nanosecond * 500) // To round under microsecond year := v.Year() year100 := year / 100 @@ -273,7 +293,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - if len(buf)+4 > mc.maxPacketAllowed { + if len(buf)+4 > mc.maxAllowedPacket { return "", driver.ErrSkip } } @@ -284,12 +304,12 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin } func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) { - if mc.netConn == nil { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement @@ -298,7 +318,6 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err return nil, err } query = prepared - args = nil } mc.affectedRows = 0 mc.insertId = 0 @@ -310,37 +329,48 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err insertId: int64(mc.insertId), }, err } - return nil, err + return nil, mc.markBadConn(err) } // Internal function to execute commands func (mc *mysqlConn) exec(query string) error { // Send command - err := mc.writeCommandPacketStr(comQuery, query) - if err != nil { - return err + if err := mc.writeCommandPacketStr(comQuery, query); err != nil { + return mc.markBadConn(err) } // Read Result resLen, err := mc.readResultSetHeaderPacket() - if err == nil && resLen > 0 { - if err = mc.readUntilEOF(); err != nil { + if err != nil { + return err + } + + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { return err } - err = mc.readUntilEOF() + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } } - return err + return mc.discardResults() } func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) { - if mc.netConn == nil { + return mc.query(query, args) +} + +func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) { + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try client-side prepare to reduce roundtrip @@ -349,7 +379,6 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro return nil, err } query = prepared - args = nil } // Send command err := mc.writeCommandPacketStr(comQuery, query) @@ -362,15 +391,22 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro rows.mc = mc if resLen == 0 { - // no columns, no more data - return emptyRows{}, nil + rows.rs.done = true + + switch err := rows.NextResultSet(); err { + case nil, io.EOF: + return rows, nil + default: + return nil, err + } } + // Columns - rows.columns, err = mc.readColumns(resLen) + rows.rs.columns, err = mc.readColumns(resLen) return rows, err } } - return nil, err + return nil, mc.markBadConn(err) } // Gets the value of the given MySQL System Variable @@ -386,6 +422,7 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { if err == nil { rows := new(textRows) rows.mc = mc + rows.rs.columns = []mysqlField{{fieldType: fieldTypeVarChar}} if resLen > 0 { // Columns @@ -401,3 +438,212 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { } return nil, err } + +// finish is called when the query has canceled. +func (mc *mysqlConn) cancel(err error) { + mc.canceled.Set(err) + mc.cleanup() +} + +// finish is called when the query has succeeded. +func (mc *mysqlConn) finish() { + if !mc.watching || mc.finished == nil { + return + } + select { + case mc.finished <- struct{}{}: + mc.watching = false + case <-mc.closech: + } +} + +// Ping implements driver.Pinger interface +func (mc *mysqlConn) Ping(ctx context.Context) (err error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return driver.ErrBadConn + } + + if err = mc.watchCancel(ctx); err != nil { + return + } + defer mc.finish() + + if err = mc.writeCommandPacket(comPing); err != nil { + return mc.markBadConn(err) + } + + return mc.readResultOK() +} + +// BeginTx implements driver.ConnBeginTx interface +func (mc *mysqlConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { + level, err := mapIsolationLevel(opts.Isolation) + if err != nil { + return nil, err + } + err = mc.exec("SET TRANSACTION ISOLATION LEVEL " + level) + if err != nil { + return nil, err + } + } + + return mc.begin(opts.ReadOnly) +} + +func (mc *mysqlConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := mc.query(query, dargs) + if err != nil { + mc.finish() + return nil, err + } + rows.finish = mc.finish + return rows, err +} + +func (mc *mysqlConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + return mc.Exec(query, dargs) +} + +func (mc *mysqlConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + stmt, err := mc.Prepare(query) + mc.finish() + if err != nil { + return nil, err + } + + select { + default: + case <-ctx.Done(): + stmt.Close() + return nil, ctx.Err() + } + return stmt, nil +} + +func (stmt *mysqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := stmt.query(dargs) + if err != nil { + stmt.mc.finish() + return nil, err + } + rows.finish = stmt.mc.finish + return rows, err +} + +func (stmt *mysqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + defer stmt.mc.finish() + + return stmt.Exec(dargs) +} + +func (mc *mysqlConn) watchCancel(ctx context.Context) error { + if mc.watching { + // Reach here if canceled, + // so the connection is already invalid + mc.cleanup() + return nil + } + // When ctx is already cancelled, don't watch it. + if err := ctx.Err(); err != nil { + return err + } + // When ctx is not cancellable, don't watch it. + if ctx.Done() == nil { + return nil + } + // When watcher is not alive, can't watch it. + if mc.watcher == nil { + return nil + } + + mc.watching = true + mc.watcher <- ctx + return nil +} + +func (mc *mysqlConn) startWatcher() { + watcher := make(chan context.Context, 1) + mc.watcher = watcher + finished := make(chan struct{}) + mc.finished = finished + go func() { + for { + var ctx context.Context + select { + case ctx = <-watcher: + case <-mc.closech: + return + } + + select { + case <-ctx.Done(): + mc.cancel(ctx.Err()) + case <-finished: + case <-mc.closech: + return + } + } + }() +} + +func (mc *mysqlConn) CheckNamedValue(nv *driver.NamedValue) (err error) { + nv.Value, err = converter{}.ConvertValue(nv.Value) + return +} + +// ResetSession implements driver.SessionResetter. +// (From Go 1.10) +func (mc *mysqlConn) ResetSession(ctx context.Context) error { + if mc.closed.IsSet() { + return driver.ErrBadConn + } + mc.reset = true + return nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/connection_test.go b/vendor/github.com/go-sql-driver/mysql/connection_test.go new file mode 100644 index 0000000..19c17ff --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connection_test.go @@ -0,0 +1,175 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "errors" + "net" + "testing" +) + +func TestInterpolateParams(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42), "gopher"}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT 42+'gopher'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + +func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42)}) + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +// We don't support placeholder in string literal for now. +// https://github.com/go-sql-driver/mysql/pull/490 +func TestInterpolateParamsPlaceholderInString(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT 'abc?xyz',?", []driver.Value{int64(42)}) + // When InterpolateParams support string literal, this should return `"SELECT 'abc?xyz', 42` + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +func TestInterpolateParamsUint64(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?", []driver.Value{uint64(42)}) + if err != nil { + t.Errorf("Expected err=nil, got err=%#v, q=%#v", err, q) + } + if q != "SELECT 42" { + t.Errorf("Expected uint64 interpolation to work, got q=%#v", q) + } +} + +func TestCheckNamedValue(t *testing.T) { + value := driver.NamedValue{Value: ^uint64(0)} + x := &mysqlConn{} + err := x.CheckNamedValue(&value) + + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if value.Value != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", value.Value, value.Value) + } +} + +// TestCleanCancel tests passed context is cancelled at start. +// No packet should be sent. Connection should keep current status. +func TestCleanCancel(t *testing.T) { + mc := &mysqlConn{ + closech: make(chan struct{}), + } + mc.startWatcher() + defer mc.cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + for i := 0; i < 3; i++ { // Repeat same behavior + err := mc.Ping(ctx) + if err != context.Canceled { + t.Errorf("expected context.Canceled, got %#v", err) + } + + if mc.closed.IsSet() { + t.Error("expected mc is not closed, closed actually") + } + + if mc.watching { + t.Error("expected watching is false, but true") + } + } +} + +func TestPingMarkBadConnection(t *testing.T) { + nc := badConnection{err: errors.New("boom")} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + } + + err := ms.Ping(context.Background()) + + if err != driver.ErrBadConn { + t.Errorf("expected driver.ErrBadConn, got %#v", err) + } +} + +func TestPingErrInvalidConn(t *testing.T) { + nc := badConnection{err: errors.New("failed to write"), n: 10} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + closech: make(chan struct{}), + } + + err := ms.Ping(context.Background()) + + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %#v", err) + } +} + +type badConnection struct { + n int + err error + net.Conn +} + +func (bc badConnection) Write(b []byte) (n int, err error) { + return bc.n, bc.err +} + +func (bc badConnection) Close() error { + return nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/go-sql-driver/mysql/connector.go new file mode 100644 index 0000000..5aaaba4 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connector.go @@ -0,0 +1,143 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "net" +) + +type connector struct { + cfg *Config // immutable private copy. +} + +// Connect implements driver.Connector interface. +// Connect returns a connection to the database. +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + cfg: c.cfg, + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + mc.netConn, err = dial(ctx, mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + } + + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Temporary() { + errLog.Print("net.Error from Dial()': ", nerr.Error()) + return nil, driver.ErrBadConn + } + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +// Driver implements driver.Connector interface. +// Driver returns &MySQLDriver{}. +func (c *connector) Driver() driver.Driver { + return &MySQLDriver{} +} diff --git a/vendor/github.com/go-sql-driver/mysql/const.go b/vendor/github.com/go-sql-driver/mysql/const.go index dddc129..b1e6b85 100644 --- a/vendor/github.com/go-sql-driver/mysql/const.go +++ b/vendor/github.com/go-sql-driver/mysql/const.go @@ -9,7 +9,9 @@ package mysql const ( - minProtocolVersion byte = 10 + defaultAuthPlugin = "mysql_native_password" + defaultMaxAllowedPacket = 4 << 20 // 4 MiB + minProtocolVersion = 10 maxPacketSize = 1<<24 - 1 timeFormat = "2006-01-02 15:04:05.999999" ) @@ -18,10 +20,11 @@ const ( // http://dev.mysql.com/doc/internals/en/client-server-protocol.html const ( - iOK byte = 0x00 - iLocalInFile byte = 0xfb - iEOF byte = 0xfe - iERR byte = 0xff + iOK byte = 0x00 + iAuthMoreData byte = 0x01 + iLocalInFile byte = 0xfb + iEOF byte = 0xfe + iERR byte = 0xff ) // https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags @@ -87,8 +90,10 @@ const ( ) // https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnType +type fieldType byte + const ( - fieldTypeDecimal byte = iota + fieldTypeDecimal fieldType = iota fieldTypeTiny fieldTypeShort fieldTypeLong @@ -107,7 +112,8 @@ const ( fieldTypeBit ) const ( - fieldTypeNewDecimal byte = iota + 0xf6 + fieldTypeJSON fieldType = iota + 0xf5 + fieldTypeNewDecimal fieldTypeEnum fieldTypeSet fieldTypeTinyBLOB @@ -160,3 +166,9 @@ const ( statusInTransReadonly statusSessionStateChanged ) + +const ( + cachingSha2PasswordRequestPublicKey = 2 + cachingSha2PasswordFastAuthSuccess = 3 + cachingSha2PasswordPerformFullAuthentication = 4 +) diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index d310624..1f9decf 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// Package mysql provides a MySQL driver for Go's database/sql package. // // The driver should be used via the database/sql package: // @@ -17,131 +17,67 @@ package mysql import ( + "context" "database/sql" "database/sql/driver" "net" + "sync" ) -// This struct is exported to make the driver directly accessible. +// MySQLDriver is exported to make the driver directly accessible. // In general the driver is used via the database/sql package. type MySQLDriver struct{} // DialFunc is a function which can be used to establish the network connection. // Custom dial functions must be registered with RegisterDial +// +// Deprecated: users should register a DialContextFunc instead type DialFunc func(addr string) (net.Conn, error) -var dials map[string]DialFunc +// DialContextFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDialContext +type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error) -// RegisterDial registers a custom dial function. It can then be used by the +var ( + dialsLock sync.RWMutex + dials map[string]DialContextFunc +) + +// RegisterDialContext registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { +// The current context for the connection and its address is passed to the dial function. +func RegisterDialContext(net string, dial DialContextFunc) { + dialsLock.Lock() + defer dialsLock.Unlock() if dials == nil { - dials = make(map[string]DialFunc) + dials = make(map[string]DialContextFunc) } dials[net] = dial } +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +// +// Deprecated: users should call RegisterDialContext instead +func RegisterDial(network string, dial DialFunc) { + RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) { + return dial(addr) + }) +} + // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how -// the DSN string is formated +// the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxPacketAllowed: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - } - mc.cfg, err = parseDSN(dsn) + cfg, err := ParseDSN(dsn) if err != nil { return nil, err } - - // Connect to Server - if dial, ok := dials[mc.cfg.net]; ok { - mc.netConn, err = dial(mc.cfg.addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.timeout} - mc.netConn, err = nd.Dial(mc.cfg.net, mc.cfg.addr) + c := &connector{ + cfg: cfg, } - if err != nil { - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - mc.buf = newBuffer(mc.netConn) - - // Reading Handshake Initialization Packet - cipher, err := mc.readInitPacket() - if err != nil { - mc.Close() - return nil, err - } - - // Send Client Authentication Packet - if err = mc.writeAuthPacket(cipher); err != nil { - mc.Close() - return nil, err - } - - // Read Result Packet - err = mc.readResultOK() - if err != nil { - // Retry with old authentication method, if allowed - if mc.cfg != nil && mc.cfg.allowOldPasswords && err == ErrOldPassword { - if err = mc.writeOldAuthPacket(cipher); err != nil { - mc.Close() - return nil, err - } - if err = mc.readResultOK(); err != nil { - mc.Close() - return nil, err - } - } else if mc.cfg != nil && mc.cfg.allowCleartextPasswords && err == ErrCleartextPassword { - if err = mc.writeClearAuthPacket(); err != nil { - mc.Close() - return nil, err - } - if err = mc.readResultOK(); err != nil { - mc.Close() - return nil, err - } - } else { - mc.Close() - return nil, err - } - - } - - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxPacketAllowed = stringToInt(maxap) - 1 - if mc.maxPacketAllowed < maxPacketSize { - mc.maxWriteSize = mc.maxPacketAllowed - } - - // Handle DSN Params - err = mc.handleParams() - if err != nil { - mc.Close() - return nil, err - } - - return mc, nil + return c.Connect(context.Background()) } func init() { diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110.go b/vendor/github.com/go-sql-driver/mysql/driver_go110.go new file mode 100644 index 0000000..eb5a8fe --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/driver_go110.go @@ -0,0 +1,37 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "database/sql/driver" +) + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go b/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go new file mode 100644 index 0000000..19a0e59 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go @@ -0,0 +1,137 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "net" + "testing" + "time" +) + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_test.go b/vendor/github.com/go-sql-driver/mysql/driver_test.go new file mode 100644 index 0000000..3dee1ba --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/driver_test.go @@ -0,0 +1,2996 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "crypto/tls" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "net" + "net/url" + "os" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Ensure that all the driver interfaces are implemented +var ( + _ driver.Rows = &binaryRows{} + _ driver.Rows = &textRows{} +) + +var ( + user string + pass string + prot string + addr string + dbname string + dsn string + netAddr string + available bool +) + +var ( + tDate = time.Date(2012, 6, 14, 0, 0, 0, 0, time.UTC) + sDate = "2012-06-14" + tDateTime = time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC) + sDateTime = "2011-11-20 21:27:37" + tDate0 = time.Time{} + sDate0 = "0000-00-00" + sDateTime0 = "0000-00-00 00:00:00" +) + +// See https://github.com/go-sql-driver/mysql/wiki/Testing +func init() { + // get environment variables + env := func(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue + } + user = env("MYSQL_TEST_USER", "root") + pass = env("MYSQL_TEST_PASS", "") + prot = env("MYSQL_TEST_PROT", "tcp") + addr = env("MYSQL_TEST_ADDR", "localhost:3306") + dbname = env("MYSQL_TEST_DBNAME", "gotest") + netAddr = fmt.Sprintf("%s(%s)", prot, addr) + dsn = fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, pass, netAddr, dbname) + c, err := net.Dial(prot, addr) + if err == nil { + available = true + c.Close() + } +} + +type DBTest struct { + *testing.T + db *sql.DB +} + +type netErrorMock struct { + temporary bool + timeout bool +} + +func (e netErrorMock) Temporary() bool { + return e.temporary +} + +func (e netErrorMock) Timeout() bool { + return e.timeout +} + +func (e netErrorMock) Error() string { + return fmt.Sprintf("mock net error. Temporary: %v, Timeout %v", e.temporary, e.timeout) +} + +func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn += "&multiStatements=true" + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + } +} + +func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + db.Exec("DROP TABLE IF EXISTS test") + + dsn2 := dsn + "&interpolateParams=true" + var db2 *sql.DB + if _, err := ParseDSN(dsn2); err != errInvalidDSNUnsafeCollation { + db2, err = sql.Open("mysql", dsn2) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db2.Close() + } + + dsn3 := dsn + "&multiStatements=true" + var db3 *sql.DB + if _, err := ParseDSN(dsn3); err != errInvalidDSNUnsafeCollation { + db3, err = sql.Open("mysql", dsn3) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db3.Close() + } + + dbt := &DBTest{t, db} + dbt2 := &DBTest{t, db2} + dbt3 := &DBTest{t, db3} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + if db2 != nil { + test(dbt2) + dbt2.db.Exec("DROP TABLE IF EXISTS test") + } + if db3 != nil { + test(dbt3) + dbt3.db.Exec("DROP TABLE IF EXISTS test") + } + } +} + +func (dbt *DBTest) fail(method, query string, err error) { + if len(query) > 300 { + query = "[query too large to print]" + } + dbt.Fatalf("error on %s %s: %s", method, query, err.Error()) +} + +func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) { + res, err := dbt.db.Exec(query, args...) + if err != nil { + dbt.fail("exec", query, err) + } + return res +} + +func (dbt *DBTest) mustQuery(query string, args ...interface{}) (rows *sql.Rows) { + rows, err := dbt.db.Query(query, args...) + if err != nil { + dbt.fail("query", query, err) + } + return rows +} + +func maybeSkip(t *testing.T, err error, skipErrno uint16) { + mySQLErr, ok := err.(*MySQLError) + if !ok { + return + } + + if mySQLErr.Number == skipErrno { + t.Skipf("skipping test for error: %v", err) + } +} + +func TestEmptyQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // just a comment, no query + rows := dbt.mustQuery("--") + defer rows.Close() + // will hang before #255 + if rows.Next() { + dbt.Errorf("next on rows must be false") + } + }) +} + +func TestCRUD(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + + // Test for unexpected data + var out bool + rows := dbt.mustQuery("SELECT * FROM test") + if rows.Next() { + dbt.Error("unexpected data in empty table") + } + rows.Close() + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + id, err := res.LastInsertId() + if err != nil { + dbt.Fatalf("res.LastInsertId() returned error: %s", err.Error()) + } + if id != 0 { + dbt.Fatalf("expected InsertId 0, got %d", id) + } + + // Read + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if true != out { + dbt.Errorf("true != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Update + res = dbt.mustExec("UPDATE test SET value = ? WHERE value = ?", false, true) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check Update + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if false != out { + dbt.Errorf("false != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Delete + res = dbt.mustExec("DELETE FROM test WHERE value = ?", false) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check for unexpected rows + res = dbt.mustExec("DELETE FROM test") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 0 { + dbt.Fatalf("expected 0 affected row, got %d", count) + } + }) +} + +func TestMultiQuery(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE `test` (`id` int(11) NOT NULL, `value` int(11) NOT NULL) ") + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1, 1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Update + res = dbt.mustExec("UPDATE test SET value = 3 WHERE id = 1; UPDATE test SET value = 4 WHERE id = 1; UPDATE test SET value = 5 WHERE id = 1;") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Read + var out int + rows := dbt.mustQuery("SELECT value FROM test WHERE id=1;") + if rows.Next() { + rows.Scan(&out) + if 5 != out { + dbt.Errorf("5 != %d", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + }) +} + +func TestInt(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [5]string{"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT"} + in := int64(42) + var out int64 + var rows *sql.Rows + + // SIGNED + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // UNSIGNED ZEROFILL + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + " ZEROFILL)") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s ZEROFILL: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s ZEROFILL: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat32(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + in := float32(42.23) + var out float32 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %g != %g", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (42.23)") + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64Placeholder(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (id int, value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (1, 42.23)") + rows = dbt.mustQuery("SELECT value FROM test WHERE id = ?", 1) + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestString(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [6]string{"CHAR(255)", "VARCHAR(255)", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"} + in := "κόσμε üöäßñóùéàâÿœ'îë Árvíztűrő いろはにほへとちりぬるを イロハニホヘト דג סקרן чащах น่าฟังเอย" + var out string + var rows *sql.Rows + + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ") CHARACTER SET utf8") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %s != %s", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // BLOB + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + + id := 2 + in = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. " + + "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." + dbt.mustExec("INSERT INTO test VALUES (?, ?)", id, in) + + err := dbt.db.QueryRow("SELECT value FROM test WHERE id = ?", id).Scan(&out) + if err != nil { + dbt.Fatalf("Error on BLOB-Query: %s", err.Error()) + } else if out != in { + dbt.Errorf("BLOB: %s != %s", in, out) + } + }) +} + +func TestRawBytes(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + v1 := []byte("aaa") + v2 := []byte("bbb") + rows := dbt.mustQuery("SELECT ?, ?", v1, v2) + defer rows.Close() + if rows.Next() { + var o1, o2 sql.RawBytes + if err := rows.Scan(&o1, &o2); err != nil { + dbt.Errorf("Got error: %v", err) + } + if !bytes.Equal(v1, o1) { + dbt.Errorf("expected %v, got %v", v1, o1) + } + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + // https://github.com/go-sql-driver/mysql/issues/765 + // Appending to RawBytes shouldn't overwrite next RawBytes. + o1 = append(o1, "xyzzy"...) + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + } else { + dbt.Errorf("no data") + } + }) +} + +type testValuer struct { + value string +} + +func (tv testValuer) Value() (driver.Value, error) { + return tv.value, nil +} + +func TestValuer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuer{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE test (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + }) +} + +type testValuerWithValidation struct { + value string +} + +func (tv testValuerWithValidation) Value() (driver.Value, error) { + if len(tv.value) == 0 { + return nil, fmt.Errorf("Invalid string valuer. Value must not be empty") + } + + return tv.value, nil +} + +func TestValuerWithValidation(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuerWithValidation{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE testValuer (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO testValuer VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM testValuer") + defer rows.Close() + + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", testValuerWithValidation{""}); err == nil { + dbt.Errorf("Failed to check valuer error") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", nil); err != nil { + dbt.Errorf("Failed to check nil") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", map[string]bool{}); err == nil { + dbt.Errorf("Failed to check not valuer") + } + + dbt.mustExec("DROP TABLE IF EXISTS testValuer") + }) +} + +type timeTests struct { + dbtype string + tlayout string + tests []timeTest +} + +type timeTest struct { + s string // leading "!": do not use t as value in queries + t time.Time +} + +type timeMode byte + +func (t timeMode) String() string { + switch t { + case binaryString: + return "binary:string" + case binaryTime: + return "binary:time.Time" + case textString: + return "text:string" + } + panic("unsupported timeMode") +} + +func (t timeMode) Binary() bool { + switch t { + case binaryString, binaryTime: + return true + } + return false +} + +const ( + binaryString timeMode = iota + binaryTime + textString +) + +func (t timeTest) genQuery(dbtype string, mode timeMode) string { + var inner string + if mode.Binary() { + inner = "?" + } else { + inner = `"%s"` + } + return `SELECT cast(` + inner + ` as ` + dbtype + `)` +} + +func (t timeTest) run(dbt *DBTest, dbtype, tlayout string, mode timeMode) { + var rows *sql.Rows + query := t.genQuery(dbtype, mode) + switch mode { + case binaryString: + rows = dbt.mustQuery(query, t.s) + case binaryTime: + rows = dbt.mustQuery(query, t.t) + case textString: + query = fmt.Sprintf(query, t.s) + rows = dbt.mustQuery(query) + default: + panic("unsupported mode") + } + defer rows.Close() + var err error + if !rows.Next() { + err = rows.Err() + if err == nil { + err = fmt.Errorf("no data") + } + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + var dst interface{} + err = rows.Scan(&dst) + if err != nil { + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + switch val := dst.(type) { + case []uint8: + str := string(val) + if str == t.s { + return + } + if mode.Binary() && dbtype == "DATETIME" && len(str) == 26 && str[:19] == t.s { + // a fix mainly for TravisCI: + // accept full microsecond resolution in result for DATETIME columns + // where the binary protocol was used + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, str, + ) + case time.Time: + if val == t.t { + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, val.Format(tlayout), + ) + default: + fmt.Printf("%#v\n", []interface{}{dbtype, tlayout, mode, t.s, t.t}) + dbt.Errorf("%s [%s]: unhandled type %T (is '%v')", + dbtype, mode, + val, val, + ) + } +} + +func TestDateTime(t *testing.T) { + afterTime := func(t time.Time, d string) time.Time { + dur, err := time.ParseDuration(d) + if err != nil { + panic(err) + } + return t.Add(dur) + } + // NOTE: MySQL rounds DATETIME(x) up - but that's not included in the tests + format := "2006-01-02 15:04:05.999999" + t0 := time.Time{} + tstr0 := "0000-00-00 00:00:00.000000" + testcases := []timeTests{ + {"DATE", format[:10], []timeTest{ + {t: time.Date(2011, 11, 20, 0, 0, 0, 0, time.UTC)}, + {t: t0, s: tstr0[:10]}, + }}, + {"DATETIME", format[:19], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(0)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(1)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 100000000, time.UTC)}, + {t: t0, s: tstr0[:21]}, + }}, + {"DATETIME(6)", format, []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 123456000, time.UTC)}, + {t: t0, s: tstr0}, + }}, + {"TIME", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(0)", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(1)", format[11:21], []timeTest{ + {t: afterTime(t0, "12345600ms")}, + {s: "!-12:34:56.7"}, + {s: "!-838:59:58.9"}, + {s: "!838:59:58.9"}, + {t: t0, s: tstr0[11:21]}, + }}, + {"TIME(6)", format[11:], []timeTest{ + {t: afterTime(t0, "1234567890123000ns")}, + {s: "!-12:34:56.789012"}, + {s: "!-838:59:58.999999"}, + {s: "!838:59:58.999999"}, + {t: t0, s: tstr0[11:]}, + }}, + } + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + microsecsSupported := false + zeroDateSupported := false + var rows *sql.Rows + var err error + rows, err = dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`) + if err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + rows, err = dbt.db.Query(`SELECT cast("0000-00-00" as DATE) = "0000-00-00"`) + if err == nil { + rows.Scan(&zeroDateSupported) + rows.Close() + } + for _, setups := range testcases { + if t := setups.dbtype; !microsecsSupported && t[len(t)-1:] == ")" { + // skip fractional second tests if unsupported by server + continue + } + for _, setup := range setups.tests { + allowBinTime := true + if setup.s == "" { + // fill time string wherever Go can reliable produce it + setup.s = setup.t.Format(setups.tlayout) + } else if setup.s[0] == '!' { + // skip tests using setup.t as source in queries + allowBinTime = false + // fix setup.s - remove the "!" + setup.s = setup.s[1:] + } + if !zeroDateSupported && setup.s == tstr0[:len(setup.s)] { + // skip disallowed 0000-00-00 date + continue + } + setup.run(dbt, setups.dbtype, setups.tlayout, textString) + setup.run(dbt, setups.dbtype, setups.tlayout, binaryString) + if allowBinTime { + setup.run(dbt, setups.dbtype, setups.tlayout, binaryTime) + } + } + } + }) + } +} + +func TestTimestampMicros(t *testing.T) { + format := "2006-01-02 15:04:05.999999" + f0 := format[:19] + f1 := format[:21] + f6 := format[:26] + runTests(t, dsn, func(dbt *DBTest) { + // check if microseconds are supported. + // Do not use timestamp(x) for that check - before 5.5.6, x would mean display width + // and not precision. + // Se last paragraph at http://dev.mysql.com/doc/refman/5.6/en/fractional-seconds.html + microsecsSupported := false + if rows, err := dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`); err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + if !microsecsSupported { + // skip test + return + } + _, err := dbt.db.Exec(` + CREATE TABLE test ( + value0 TIMESTAMP NOT NULL DEFAULT '` + f0 + `', + value1 TIMESTAMP(1) NOT NULL DEFAULT '` + f1 + `', + value6 TIMESTAMP(6) NOT NULL DEFAULT '` + f6 + `' + )`, + ) + if err != nil { + dbt.Error(err) + } + defer dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("INSERT INTO test SET value0=?, value1=?, value6=?", f0, f1, f6) + var res0, res1, res6 string + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Errorf("test contained no selectable values") + } + err = rows.Scan(&res0, &res1, &res6) + if err != nil { + dbt.Error(err) + } + if res0 != f0 { + dbt.Errorf("expected %q, got %q", f0, res0) + } + if res1 != f1 { + dbt.Errorf("expected %q, got %q", f1, res1) + } + if res6 != f6 { + dbt.Errorf("expected %q, got %q", f6, res6) + } + }) +} + +func TestNULL(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + nullStmt, err := dbt.db.Prepare("SELECT NULL") + if err != nil { + dbt.Fatal(err) + } + defer nullStmt.Close() + + nonNullStmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + defer nonNullStmt.Close() + + // NullBool + var nb sql.NullBool + // Invalid + if err = nullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if nb.Valid { + dbt.Error("valid NullBool which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if !nb.Valid { + dbt.Error("invalid NullBool which should be valid") + } else if nb.Bool != true { + dbt.Errorf("Unexpected NullBool value: %t (should be true)", nb.Bool) + } + + // NullFloat64 + var nf sql.NullFloat64 + // Invalid + if err = nullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if nf.Valid { + dbt.Error("valid NullFloat64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if !nf.Valid { + dbt.Error("invalid NullFloat64 which should be valid") + } else if nf.Float64 != float64(1) { + dbt.Errorf("unexpected NullFloat64 value: %f (should be 1.0)", nf.Float64) + } + + // NullInt64 + var ni sql.NullInt64 + // Invalid + if err = nullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if ni.Valid { + dbt.Error("valid NullInt64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if !ni.Valid { + dbt.Error("invalid NullInt64 which should be valid") + } else if ni.Int64 != int64(1) { + dbt.Errorf("unexpected NullInt64 value: %d (should be 1)", ni.Int64) + } + + // NullString + var ns sql.NullString + // Invalid + if err = nullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if ns.Valid { + dbt.Error("valid NullString which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if !ns.Valid { + dbt.Error("invalid NullString which should be valid") + } else if ns.String != `1` { + dbt.Error("unexpected NullString value:" + ns.String + " (should be `1`)") + } + + // nil-bytes + var b []byte + // Read nil + if err = nullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil []byte which should be nil") + } + // Read non-nil + if err = nonNullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil []byte which should be non-nil") + } + // Insert nil + b = nil + success := false + if err = dbt.db.QueryRow("SELECT ? IS NULL", b).Scan(&success); err != nil { + dbt.Fatal(err) + } + if !success { + dbt.Error("inserting []byte(nil) as NULL failed") + } + // Check input==output with input==nil + b = nil + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil echo from nil input") + } + // Check input==output with input!=nil + b = []byte("") + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil echo from non-nil input") + } + + // Insert NULL + dbt.mustExec("CREATE TABLE test (dummmy1 int, value int, dummy2 int)") + + dbt.mustExec("INSERT INTO test VALUES (?, ?, ?)", 1, nil, 2) + + var out interface{} + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if out != nil { + dbt.Errorf("%v != nil", out) + } + } else { + dbt.Error("no data") + } + }) +} + +func TestUint64(t *testing.T) { + const ( + u0 = uint64(0) + uall = ^u0 + uhigh = uall >> 1 + utop = ^uhigh + s0 = int64(0) + sall = ^s0 + shigh = int64(uhigh) + stop = ^shigh + ) + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare(`SELECT ?, ?, ? ,?, ?, ?, ?, ?`) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + row := stmt.QueryRow( + u0, uhigh, utop, uall, + s0, shigh, stop, sall, + ) + + var ua, ub, uc, ud uint64 + var sa, sb, sc, sd int64 + + err = row.Scan(&ua, &ub, &uc, &ud, &sa, &sb, &sc, &sd) + if err != nil { + dbt.Fatal(err) + } + switch { + case ua != u0, + ub != uhigh, + uc != utop, + ud != uall, + sa != s0, + sb != shigh, + sc != stop, + sd != sall: + dbt.Fatal("unexpected result value") + } + }) +} + +func TestLongData(t *testing.T) { + runTests(t, dsn+"&maxAllowedPacket=0", func(dbt *DBTest) { + var maxAllowedPacketSize int + err := dbt.db.QueryRow("select @@max_allowed_packet").Scan(&maxAllowedPacketSize) + if err != nil { + dbt.Fatal(err) + } + maxAllowedPacketSize-- + + // don't get too ambitious + if maxAllowedPacketSize > 1<<25 { + maxAllowedPacketSize = 1 << 25 + } + + dbt.mustExec("CREATE TABLE test (value LONGBLOB)") + + in := strings.Repeat(`a`, maxAllowedPacketSize+1) + var out string + var rows *sql.Rows + + // Long text data + const nonDataQueryLen = 28 // length query w/o value + inS := in[:maxAllowedPacketSize-nonDataQueryLen] + dbt.mustExec("INSERT INTO test VALUES('" + inS + "')") + rows = dbt.mustQuery("SELECT value FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if inS != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(inS), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + dbt.Fatalf("LONGBLOB: no data") + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Long binary data + dbt.mustExec("INSERT INTO test VALUES(?)", in) + rows = dbt.mustQuery("SELECT value FROM test WHERE 1=?", 1) + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(in), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + if err = rows.Err(); err != nil { + dbt.Fatalf("LONGBLOB: no data (err: %s)", err.Error()) + } else { + dbt.Fatal("LONGBLOB: no data (err: )") + } + } + }) +} + +func TestLoadData(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + verifyLoadDataResult := func() { + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + dbt.Fatal(err.Error()) + } + + i := 0 + values := [4]string{ + "a string", + "a string containing a \t", + "a string containing a \n", + "a string containing both \t\n", + } + + var id int + var value string + + for rows.Next() { + i++ + err = rows.Scan(&id, &value) + if err != nil { + dbt.Fatal(err.Error()) + } + if i != id { + dbt.Fatalf("%d != %d", i, id) + } + if values[i-1] != value { + dbt.Fatalf("%q != %q", values[i-1], value) + } + } + err = rows.Err() + if err != nil { + dbt.Fatal(err.Error()) + } + + if i != 4 { + dbt.Fatalf("rows count mismatch. Got %d, want 4", i) + } + } + + dbt.db.Exec("DROP TABLE IF EXISTS test") + dbt.mustExec("CREATE TABLE test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") + + // Local File + file, err := ioutil.TempFile("", "gotest") + defer os.Remove(file.Name()) + if err != nil { + dbt.Fatal(err) + } + RegisterLocalFile(file.Name()) + + // Try first with empty file + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + var count int + err = dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&count) + if err != nil { + dbt.Fatal(err.Error()) + } + if count != 0 { + dbt.Fatalf("unexpected row count: got %d, want 0", count) + } + + // Then fille File with data and try to load it + file.WriteString("1\ta string\n2\ta string containing a \\t\n3\ta string containing a \\n\n4\ta string containing both \\t\\n\n") + file.Close() + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + verifyLoadDataResult() + + // Try with non-existing file + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent file didn't fail") + } else if err.Error() != "local file 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Reader + RegisterReaderHandler("test", func() io.Reader { + file, err = os.Open(file.Name()) + if err != nil { + dbt.Fatal(err) + } + return file + }) + dbt.mustExec("LOAD DATA LOCAL INFILE 'Reader::test' INTO TABLE test") + verifyLoadDataResult() + // negative test + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'Reader::doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent Reader didn't fail") + } else if err.Error() != "Reader 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + }) +} + +func TestFoundRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + }) + runTests(t, dsn+"&clientFoundRows=true", func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 matched rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 3 { + dbt.Fatalf("Expected 3 matched rows, got %d", count) + } + }) +} + +func TestTLS(t *testing.T) { + tlsTestReq := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + if err == ErrNoTLS { + dbt.Skip("server does not support TLS") + } else { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + rows := dbt.mustQuery("SHOW STATUS LIKE 'Ssl_cipher'") + defer rows.Close() + + var variable, value *sql.RawBytes + for rows.Next() { + if err := rows.Scan(&variable, &value); err != nil { + dbt.Fatal(err.Error()) + } + + if (*value == nil) || (len(*value) == 0) { + dbt.Fatalf("no Cipher") + } else { + dbt.Logf("Cipher: %s", *value) + } + } + } + tlsTestOpt := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + runTests(t, dsn+"&tls=preferred", tlsTestOpt) + runTests(t, dsn+"&tls=skip-verify", tlsTestReq) + + // Verify that registering / using a custom cfg works + RegisterTLSConfig("custom-skip-verify", &tls.Config{ + InsecureSkipVerify: true, + }) + runTests(t, dsn+"&tls=custom-skip-verify", tlsTestReq) +} + +func TestReuseClosedConnection(t *testing.T) { + // this test does not use sql.database, it uses the driver directly + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + md := &MySQLDriver{} + conn, err := md.Open(dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + stmt, err := conn.Prepare("DO 1") + if err != nil { + t.Fatalf("error preparing statement: %s", err.Error()) + } + _, err = stmt.Exec(nil) + if err != nil { + t.Fatalf("error executing statement: %s", err.Error()) + } + err = conn.Close() + if err != nil { + t.Fatalf("error closing connection: %s", err.Error()) + } + + defer func() { + if err := recover(); err != nil { + t.Errorf("panic after reusing a closed connection: %v", err) + } + }() + _, err = stmt.Exec(nil) + if err != nil && err != driver.ErrBadConn { + t.Errorf("unexpected error '%s', expected '%s'", + err.Error(), driver.ErrBadConn.Error()) + } +} + +func TestCharset(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mustSetCharset := func(charsetParam, expected string) { + runTests(t, dsn+"&"+charsetParam, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT @@character_set_connection") + defer rows.Close() + + if !rows.Next() { + dbt.Fatalf("error getting connection charset: %s", rows.Err()) + } + + var got string + rows.Scan(&got) + + if got != expected { + dbt.Fatalf("expected connection charset %s but got %s", expected, got) + } + }) + } + + // non utf8 test + mustSetCharset("charset=ascii", "ascii") + + // when the first charset is invalid, use the second + mustSetCharset("charset=none,utf8", "utf8") + + // when the first charset is valid, use it + mustSetCharset("charset=ascii,utf8", "ascii") + mustSetCharset("charset=utf8,ascii", "utf8") +} + +func TestFailingCharset(t *testing.T) { + runTests(t, dsn+"&charset=none", func(dbt *DBTest) { + // run query to really establish connection... + _, err := dbt.db.Exec("SELECT 1") + if err == nil { + dbt.db.Close() + t.Fatalf("connection must not succeed without a valid charset") + } + }) +} + +func TestCollation(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + defaultCollation := "utf8mb4_general_ci" + testCollations := []string{ + "", // do not set + defaultCollation, // driver default + "latin1_general_ci", + "binary", + "utf8_unicode_ci", + "cp1257_bin", + } + + for _, collation := range testCollations { + var expected, tdsn string + if collation != "" { + tdsn = dsn + "&collation=" + collation + expected = collation + } else { + tdsn = dsn + expected = defaultCollation + } + + runTests(t, tdsn, func(dbt *DBTest) { + var got string + if err := dbt.db.QueryRow("SELECT @@collation_connection").Scan(&got); err != nil { + dbt.Fatal(err) + } + + if got != expected { + dbt.Fatalf("expected connection collation %s but got %s", expected, got) + } + }) + } +} + +func TestColumnsWithAlias(t *testing.T) { + runTests(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1 AS A") + defer rows.Close() + cols, _ := rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A" { + t.Fatalf("expected column name \"A\", got \"%s\"", cols[0]) + } + + rows = dbt.mustQuery("SELECT * FROM (SELECT 1 AS one) AS A") + defer rows.Close() + cols, _ = rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A.one" { + t.Fatalf("expected column name \"A.one\", got \"%s\"", cols[0]) + } + }) +} + +func TestRawBytesResultExceedsBuffer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // defaultBufSize from buffer.go + expected := strings.Repeat("abc", defaultBufSize) + + rows := dbt.mustQuery("SELECT '" + expected + "'") + defer rows.Close() + if !rows.Next() { + dbt.Error("expected result, got none") + } + var result sql.RawBytes + rows.Scan(&result) + if expected != string(result) { + dbt.Error("result did not match expected value") + } + }) +} + +func TestTimezoneConversion(t *testing.T) { + zones := []string{"UTC", "US/Central", "US/Pacific", "Local"} + + // Regression test for timezone handling + tzTest := func(dbt *DBTest) { + // Create table + dbt.mustExec("CREATE TABLE test (ts TIMESTAMP)") + + // Insert local time into database (should be converted) + usCentral, _ := time.LoadLocation("US/Central") + reftime := time.Date(2014, 05, 30, 18, 03, 17, 0, time.UTC).In(usCentral) + dbt.mustExec("INSERT INTO test VALUE (?)", reftime) + + // Retrieve time from DB + rows := dbt.mustQuery("SELECT ts FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Fatal("did not get any rows out") + } + + var dbTime time.Time + err := rows.Scan(&dbTime) + if err != nil { + dbt.Fatal("Err", err) + } + + // Check that dates match + if reftime.Unix() != dbTime.Unix() { + dbt.Errorf("times do not match.\n") + dbt.Errorf(" Now(%v)=%v\n", usCentral, reftime) + dbt.Errorf(" Now(UTC)=%v\n", dbTime) + } + } + + for _, tz := range zones { + runTests(t, dsn+"&parseTime=true&loc="+url.QueryEscape(tz), tzTest) + } +} + +// Special cases + +func TestRowsClose(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows, err := dbt.db.Query("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + err = rows.Close() + if err != nil { + dbt.Fatal(err) + } + + if rows.Next() { + dbt.Fatal("unexpected row after rows.Close()") + } + + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + }) +} + +// dangling statements +// http://code.google.com/p/go/issues/detail?id=3865 +func TestCloseStmtBeforeRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + rows, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows.Close() + + err = stmt.Close() + if err != nil { + dbt.Fatal(err) + } + + if !rows.Next() { + dbt.Fatal("getting row failed") + } else { + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + + var out bool + err = rows.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + }) +} + +// It is valid to have multiple Rows for the same Stmt +// http://code.google.com/p/go/issues/detail?id=3734 +func TestStmtMultiRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1 UNION SELECT 0") + if err != nil { + dbt.Fatal(err) + } + + rows1, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows1.Close() + + rows2, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows2.Close() + + var out bool + + // 1 + if !rows1.Next() { + dbt.Fatal("first rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + if !rows2.Next() { + dbt.Fatal("first rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + // 2 + if !rows1.Next() { + dbt.Fatal("second rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows1.Next() { + dbt.Fatal("unexpected row on rows1") + } + err = rows1.Close() + if err != nil { + dbt.Fatal(err) + } + } + + if !rows2.Next() { + dbt.Fatal("second rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows2.Next() { + dbt.Fatal("unexpected row on rows2") + } + err = rows2.Close() + if err != nil { + dbt.Fatal(err) + } + } + }) +} + +// Regression test for +// * more than 32 NULL parameters (issue 209) +// * more parameters than fit into the buffer (issue 201) +// * parameters * 64 > max_allowed_packet (issue 734) +func TestPreparedManyCols(t *testing.T) { + numParams := 65535 + runTests(t, dsn, func(dbt *DBTest) { + query := "SELECT ?" + strings.Repeat(",?", numParams-1) + stmt, err := dbt.db.Prepare(query) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + + // create more parameters than fit into the buffer + // which will take nil-values + params := make([]interface{}, numParams) + rows, err := stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + + // Create 0byte string which we can't send via STMT_LONG_DATA. + for i := 0; i < numParams; i++ { + params[i] = "" + } + rows, err = stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + }) +} + +func TestConcurrent(t *testing.T) { + if enabled, _ := readBool(os.Getenv("MYSQL_TEST_CONCURRENT")); !enabled { + t.Skip("MYSQL_TEST_CONCURRENT env var not set") + } + + runTests(t, dsn, func(dbt *DBTest) { + var max int + err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + dbt.Logf("testing up to %d concurrent connections \r\n", max) + + var remaining, succeeded int32 = int32(max), 0 + + var wg sync.WaitGroup + wg.Add(max) + + var fatalError string + var once sync.Once + fatalf := func(s string, vals ...interface{}) { + once.Do(func() { + fatalError = fmt.Sprintf(s, vals...) + }) + } + + for i := 0; i < max; i++ { + go func(id int) { + defer wg.Done() + + tx, err := dbt.db.Begin() + atomic.AddInt32(&remaining, -1) + + if err != nil { + if err.Error() != "Error 1040: Too many connections" { + fatalf("error on conn %d: %s", id, err.Error()) + } + return + } + + // keep the connection busy until all connections are open + for remaining > 0 { + if _, err = tx.Exec("DO 1"); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + } + + if err = tx.Commit(); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + + // everything went fine with this connection + atomic.AddInt32(&succeeded, 1) + }(i) + } + + // wait until all conections are open + wg.Wait() + + if fatalError != "" { + dbt.Fatal(fatalError) + } + + dbt.Logf("reached %d concurrent connections\r\n", succeeded) + }) +} + +func testDialError(t *testing.T, dialErr error, expectErr error) { + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + return nil, dialErr + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + _, err = db.Exec("DO 1") + if err != expectErr { + t.Fatalf("was expecting %s. Got: %s", dialErr, err) + } +} + +func TestDialUnknownError(t *testing.T) { + testErr := fmt.Errorf("test") + testDialError(t, testErr, testErr) +} + +func TestDialNonRetryableNetErr(t *testing.T) { + testErr := netErrorMock{} + testDialError(t, testErr, testErr) +} + +func TestDialTemporaryNetErr(t *testing.T) { + testErr := netErrorMock{temporary: true} + testDialError(t, testErr, driver.ErrBadConn) +} + +// Tests custom dial functions +func TestCustomDial(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + // our custom dial function which justs wraps net.Dial here + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + if _, err = db.Exec("DO 1"); err != nil { + t.Fatalf("connection failed: %s", err.Error()) + } +} + +func TestSQLInjection(t *testing.T) { + createTest := func(arg string) func(dbt *DBTest) { + return func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (?)", 1) + + var v int + // NULL can't be equal to anything, the idea here is to inject query so it returns row + // This test verifies that escapeQuotes and escapeBackslash are working properly + err := dbt.db.QueryRow("SELECT v FROM test WHERE NULL = ?", arg).Scan(&v) + if err == sql.ErrNoRows { + return // success, sql injection failed + } else if err == nil { + dbt.Errorf("sql injection successful with arg: %s", arg) + } else { + dbt.Errorf("error running query with arg: %s; err: %s", arg, err.Error()) + } + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, createTest("1 OR 1=1")) + runTests(t, testdsn, createTest("' OR '1'='1")) + } +} + +// Test if inserted data is correctly retrieved after being escaped +func TestInsertRetrieveEscapedData(t *testing.T) { + testData := func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v VARCHAR(255))") + + // All sequences that are escaped by escapeQuotes and escapeBackslash + v := "foo \x00\n\r\x1a\"'\\" + dbt.mustExec("INSERT INTO test VALUES (?)", v) + + var out string + err := dbt.db.QueryRow("SELECT v FROM test").Scan(&out) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + if out != v { + dbt.Errorf("%q != %q", out, v) + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, testData) + } +} + +func TestUnixSocketAuthFail(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Save the current logger so we can restore it. + oldLogger := errLog + + // Set a new logger so we can capture its output. + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + newLogger := log.New(buffer, "prefix: ", 0) + SetLogger(newLogger) + + // Restore the logger. + defer SetLogger(oldLogger) + + // Make a new DSN that uses the MySQL socket file and a bad password, which + // we can make by simply appending any character to the real password. + badPass := pass + "x" + socket := "" + if prot == "unix" { + socket = addr + } else { + // Get socket file from MySQL. + err := dbt.db.QueryRow("SELECT @@socket").Scan(&socket) + if err != nil { + t.Fatalf("error on SELECT @@socket: %s", err.Error()) + } + } + t.Logf("socket: %s", socket) + badDSN := fmt.Sprintf("%s:%s@unix(%s)/%s?timeout=30s", user, badPass, socket, dbname) + db, err := sql.Open("mysql", badDSN) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + // Connect to MySQL for real. This will cause an auth failure. + err = db.Ping() + if err == nil { + t.Error("expected Ping() to return an error") + } + + // The driver should not log anything. + if actual := buffer.String(); actual != "" { + t.Errorf("expected no output, got %q", actual) + } + }) +} + +// See Issue #422 +func TestInterruptBySignal(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + DROP PROCEDURE IF EXISTS test_signal; + CREATE PROCEDURE test_signal(ret INT) + BEGIN + SELECT ret; + SIGNAL SQLSTATE + '45001' + SET + MESSAGE_TEXT = "an error", + MYSQL_ERRNO = 45001; + END + `) + defer dbt.mustExec("DROP PROCEDURE test_signal") + + var val int + + // text protocol + rows, err := dbt.db.Query("CALL test_signal(42)") + if err != nil { + dbt.Fatalf("error on text query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + + // binary protocol + rows, err = dbt.db.Query("CALL test_signal(?)", 42) + if err != nil { + dbt.Fatalf("error on binary query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + }) +} + +func TestColumnsReusesSlice(t *testing.T) { + rows := mysqlRows{ + rs: resultSet{ + columns: []mysqlField{ + { + tableName: "test", + name: "A", + }, + { + tableName: "test", + name: "B", + }, + }, + }, + } + + allocs := testing.AllocsPerRun(1, func() { + cols := rows.Columns() + + if len(cols) != 2 { + t.Fatalf("expected 2 columns, got %d", len(cols)) + } + }) + + if allocs != 0 { + t.Fatalf("expected 0 allocations, got %d", int(allocs)) + } + + if rows.rs.columnNames == nil { + t.Fatalf("expected columnNames to be set, got nil") + } +} + +func TestRejectReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read-only. We didn't set the `rejectReadOnly` + // option, so any writes after this should fail. + _, err := dbt.db.Exec("SET SESSION TRANSACTION READ ONLY") + // Error 1193: Unknown system variable 'TRANSACTION' => skip test, + // MySQL server version is too old + maybeSkip(t, err, 1193) + if _, err := dbt.db.Exec("DROP TABLE test"); err == nil { + t.Fatalf("writing to DB in read-only session without " + + "rejectReadOnly did not error") + } + // Set the session back to read-write so runTests() can properly clean + // up the table `test`. + dbt.mustExec("SET SESSION TRANSACTION READ WRITE") + }) + + // Enable the `rejectReadOnly` option. + runTests(t, dsn+"&rejectReadOnly=true", func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read only. Any writes after this should error on + // a driver.ErrBadConn, and cause `database/sql` to initiate a new + // connection. + dbt.mustExec("SET SESSION TRANSACTION READ ONLY") + // This would error, but `database/sql` should automatically retry on a + // new connection which is not read-only, and eventually succeed. + dbt.mustExec("DROP TABLE test") + }) +} + +func TestPing(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.fail("Ping", "Ping", err) + } + }) +} + +// See Issue #799 +func TestEmptyPassword(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn := fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, "", netAddr, dbname) + db, err := sql.Open("mysql", dsn) + if err == nil { + defer db.Close() + err = db.Ping() + } + + if pass == "" { + if err != nil { + t.Fatal(err.Error()) + } + } else { + if err == nil { + t.Fatal("expected authentication error") + } + if !strings.HasPrefix(err.Error(), "Error 1045") { + t.Fatal(err.Error()) + } + } +} + +// static interface implementation checks of mysqlConn +var ( + _ driver.ConnBeginTx = &mysqlConn{} + _ driver.ConnPrepareContext = &mysqlConn{} + _ driver.ExecerContext = &mysqlConn{} + _ driver.Pinger = &mysqlConn{} + _ driver.QueryerContext = &mysqlConn{} +) + +// static interface implementation checks of mysqlStmt +var ( + _ driver.StmtExecContext = &mysqlStmt{} + _ driver.StmtQueryContext = &mysqlStmt{} +) + +// Ensure that all the driver interfaces are implemented +var ( + // _ driver.RowsColumnTypeLength = &binaryRows{} + // _ driver.RowsColumnTypeLength = &textRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &binaryRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &textRows{} + _ driver.RowsColumnTypeNullable = &binaryRows{} + _ driver.RowsColumnTypeNullable = &textRows{} + _ driver.RowsColumnTypePrecisionScale = &binaryRows{} + _ driver.RowsColumnTypePrecisionScale = &textRows{} + _ driver.RowsColumnTypeScanType = &binaryRows{} + _ driver.RowsColumnTypeScanType = &textRows{} + _ driver.RowsNextResultSet = &binaryRows{} + _ driver.RowsNextResultSet = &textRows{} +) + +func TestMultiResultSet(t *testing.T) { + type result struct { + values [][]int + columns []string + } + + // checkRows is a helper test function to validate rows containing 3 result + // sets with specific values and columns. The basic query would look like this: + // + // SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + // SELECT 0 UNION SELECT 1; + // SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + // + // to distinguish test cases the first string argument is put in front of + // every error or fatal message. + checkRows := func(desc string, rows *sql.Rows, dbt *DBTest) { + expected := []result{ + { + values: [][]int{{1, 2}, {3, 4}}, + columns: []string{"col1", "col2"}, + }, + { + values: [][]int{{1, 2, 3}, {4, 5, 6}}, + columns: []string{"col1", "col2", "col3"}, + }, + } + + var res1 result + for rows.Next() { + var res [2]int + if err := rows.Scan(&res[0], &res[1]); err != nil { + dbt.Fatal(err) + } + res1.values = append(res1.values, res[:]) + } + + cols, err := rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res1.columns = cols + + if !reflect.DeepEqual(expected[0], res1) { + dbt.Error(desc, "want =", expected[0], "got =", res1) + } + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + // ignoring one result set + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + var res2 result + cols, err = rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res2.columns = cols + + for rows.Next() { + var res [3]int + if err := rows.Scan(&res[0], &res[1], &res[2]); err != nil { + dbt.Fatal(desc, err) + } + res2.values = append(res2.values, res[:]) + } + + if !reflect.DeepEqual(expected[1], res2) { + dbt.Error(desc, "want =", expected[1], "got =", res2) + } + + if rows.NextResultSet() { + dbt.Error(desc, "unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error(desc, err) + } + } + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery(`DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6;`) + defer rows.Close() + checkRows("query: ", rows, dbt) + }) + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + queries := []string{ + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + } + + defer dbt.mustExec("DROP PROCEDURE IF EXISTS test_mrss") + + for i, query := range queries { + dbt.mustExec(query) + + stmt, err := dbt.db.Prepare("CALL test_mrss()") + if err != nil { + dbt.Fatalf("%v (i=%d)", err, i) + } + defer stmt.Close() + + for j := 0; j < 2; j++ { + rows, err := stmt.Query() + if err != nil { + dbt.Fatalf("%v (i=%d) (j=%d)", err, i, j) + } + checkRows(fmt.Sprintf("prepared stmt query (i=%d) (j=%d): ", i, j), rows, dbt) + } + } + }) +} + +func TestMultiResultSetNoSelect(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("DO 1; DO 2;") + defer rows.Close() + + if rows.Next() { + dbt.Error("unexpected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +// tests if rows are set in a proper state if some results were ignored before +// calling rows.NextResultSet. +func TestSkipResults(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1, 2") + defer rows.Close() + + if !rows.Next() { + dbt.Error("expected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +func TestPingContext(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := dbt.db.PingContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (1)"); err == nil { + dbt.Error("expected error") + } else if err.Error() != "context canceled" { + dbt.Fatalf("unexpected error: %s", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (1)"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQueryRow(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (1), (2), (3)") + ctx, cancel := context.WithCancel(context.Background()) + + rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM test") + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + // the first row will be succeed. + var v int + if !rows.Next() { + dbt.Fatalf("unexpected end") + } + if err := rows.Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + + cancel() + // make sure the driver receives the cancel request. + time.Sleep(100 * time.Millisecond) + + if rows.Next() { + dbt.Errorf("expected end, but not") + } + if err := rows.Err(); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelPrepare(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := dbt.db.PrepareContext(ctx, "SELECT 1"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelStmtExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.ExecContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelStmtQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.QueryContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query has done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelBegin(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + tx, err := dbt.db.BeginTx(ctx, nil) + if err != nil { + dbt.Fatal(err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(100*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := tx.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Transaction is canceled, so expect an error. + switch err := tx.Commit(); err { + case sql.ErrTxDone: + // because the transaction has already been rollbacked. + // the database/sql package watches ctx + // and rollbacks when ctx is canceled. + case context.Canceled: + // the database/sql package rollbacks on another goroutine, + // so the transaction may not be rollbacked depending on goroutine scheduling. + default: + dbt.Errorf("expected sql.ErrTxDone or context.Canceled, got %v", err) + } + + // Context is canceled, so cannot begin a transaction. + if _, err := dbt.db.BeginTx(ctx, nil); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextBeginIsolationLevel(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx1, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelRepeatableRead, + }) + if err != nil { + dbt.Fatal(err) + } + + tx2, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelReadCommitted, + }) + if err != nil { + dbt.Fatal(err) + } + + _, err = tx1.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if err != nil { + dbt.Fatal(err) + } + + var v int + row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Because writer transaction wasn't commited yet, it should be available + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + err = tx1.Commit() + if err != nil { + dbt.Fatal(err) + } + + row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Data written by writer transaction is already commited, it should be selectable + if v != 1 { + dbt.Errorf("expected val to be 1, got %d", v) + } + tx2.Commit() + }) +} + +func TestContextBeginReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + ReadOnly: true, + }) + if _, ok := err.(*MySQLError); ok { + dbt.Skip("It seems that your MySQL does not support READ ONLY transactions") + return + } else if err != nil { + dbt.Fatal(err) + } + + // INSERT queries fail in a READ ONLY transaction. + _, err = tx.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if _, ok := err.(*MySQLError); !ok { + dbt.Errorf("expected MySQLError, got %v", err) + } + + // SELECT queries can be executed. + var v int + row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + if err := tx.Commit(); err != nil { + dbt.Fatal(err) + } + }) +} + +func TestRowsColumnTypes(t *testing.T) { + niNULL := sql.NullInt64{Int64: 0, Valid: false} + ni0 := sql.NullInt64{Int64: 0, Valid: true} + ni1 := sql.NullInt64{Int64: 1, Valid: true} + ni42 := sql.NullInt64{Int64: 42, Valid: true} + nfNULL := sql.NullFloat64{Float64: 0.0, Valid: false} + nf0 := sql.NullFloat64{Float64: 0.0, Valid: true} + nf1337 := sql.NullFloat64{Float64: 13.37, Valid: true} + nt0 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} + nt1 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} + nt2 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} + nt6 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} + nd1 := NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} + nd2 := NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} + ndNULL := NullTime{Time: time.Time{}, Valid: false} + rbNULL := sql.RawBytes(nil) + rb0 := sql.RawBytes("0") + rb42 := sql.RawBytes("42") + rbTest := sql.RawBytes("Test") + rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 + rbx0 := sql.RawBytes("\x00") + rbx42 := sql.RawBytes("\x42") + + var columns = []struct { + name string + fieldType string // type used when creating table schema + databaseTypeName string // actual type used by MySQL + scanType reflect.Type + nullable bool + precision int64 // 0 if not ok + scale int64 + valuesIn [3]string + valuesOut [3]interface{} + }{ + {"bit8null", "BIT(8)", "BIT", scanTypeRawBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]interface{}{rbx0, rbNULL, rbx42}}, + {"boolnull", "BOOL", "TINYINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "true", "0"}, [3]interface{}{niNULL, ni1, ni0}}, + {"bool", "BOOL NOT NULL", "TINYINT", scanTypeInt8, false, 0, 0, [3]string{"1", "0", "FALSE"}, [3]interface{}{int8(1), int8(0), int8(0)}}, + {"intnull", "INTEGER", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"smallint", "SMALLINT NOT NULL", "SMALLINT", scanTypeInt16, false, 0, 0, [3]string{"0", "-32768", "32767"}, [3]interface{}{int16(0), int16(-32768), int16(32767)}}, + {"smallintnull", "SMALLINT", "SMALLINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int3null", "INT(3)", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int7", "INT(7) NOT NULL", "INT", scanTypeInt32, false, 0, 0, [3]string{"0", "-1337", "42"}, [3]interface{}{int32(0), int32(-1337), int32(42)}}, + {"mediumintnull", "MEDIUMINT", "MEDIUMINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "42", "NULL"}, [3]interface{}{ni0, ni42, niNULL}}, + {"bigint", "BIGINT NOT NULL", "BIGINT", scanTypeInt64, false, 0, 0, [3]string{"0", "65535", "-42"}, [3]interface{}{int64(0), int64(65535), int64(-42)}}, + {"bigintnull", "BIGINT", "BIGINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "1", "42"}, [3]interface{}{niNULL, ni1, ni42}}, + {"tinyuint", "TINYINT UNSIGNED NOT NULL", "TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, + {"smalluint", "SMALLINT UNSIGNED NOT NULL", "SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, + {"biguint", "BIGINT UNSIGNED NOT NULL", "BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, + {"uint13", "INT(13) UNSIGNED NOT NULL", "INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, + {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float32(0), float32(42), float32(13.37)}}, + {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"double", "DOUBLE NOT NULL", "DOUBLE", scanTypeFloat64, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float64(0), float64(42), float64(13.37)}}, + {"doublenull", "DOUBLE", "DOUBLE", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), sql.RawBytes("13.370000"), sql.RawBytes("1234.123456")}}, + {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeRawBytes, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), rbNULL, sql.RawBytes("1234.123456")}}, + {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), sql.RawBytes("13.3700"), sql.RawBytes("1234.1235")}}, + {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, + {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, + {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, + {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, + {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, + {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, + {"date", "DATE", "DATE", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02'", "NULL", "'2006-03-04'"}, [3]interface{}{nd1, ndNULL, nd2}}, + {"year", "YEAR NOT NULL", "YEAR", scanTypeUint16, false, 0, 0, [3]string{"2006", "2000", "1994"}, [3]interface{}{uint16(2006), uint16(2000), uint16(1994)}}, + } + + schema := "" + values1 := "" + values2 := "" + values3 := "" + for _, column := range columns { + schema += fmt.Sprintf("`%s` %s, ", column.name, column.fieldType) + values1 += column.valuesIn[0] + ", " + values2 += column.valuesIn[1] + ", " + values3 += column.valuesIn[2] + ", " + } + schema = schema[:len(schema)-2] + values1 = values1[:len(values1)-2] + values2 = values2[:len(values2)-2] + values3 = values3[:len(values3)-2] + + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (" + schema + ")") + dbt.mustExec("INSERT INTO test VALUES (" + values1 + "), (" + values2 + "), (" + values3 + ")") + + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + t.Fatalf("Query: %v", err) + } + + tt, err := rows.ColumnTypes() + if err != nil { + t.Fatalf("ColumnTypes: %v", err) + } + + if len(tt) != len(columns) { + t.Fatalf("unexpected number of columns: expected %d, got %d", len(columns), len(tt)) + } + + types := make([]reflect.Type, len(tt)) + for i, tp := range tt { + column := columns[i] + + // Name + name := tp.Name() + if name != column.name { + t.Errorf("column name mismatch %s != %s", name, column.name) + continue + } + + // DatabaseTypeName + databaseTypeName := tp.DatabaseTypeName() + if databaseTypeName != column.databaseTypeName { + t.Errorf("databasetypename name mismatch for column %q: %s != %s", name, databaseTypeName, column.databaseTypeName) + continue + } + + // ScanType + scanType := tp.ScanType() + if scanType != column.scanType { + if scanType == nil { + t.Errorf("scantype is null for column %q", name) + } else { + t.Errorf("scantype mismatch for column %q: %s != %s", name, scanType.Name(), column.scanType.Name()) + } + continue + } + types[i] = scanType + + // Nullable + nullable, ok := tp.Nullable() + if !ok { + t.Errorf("nullable not ok %q", name) + continue + } + if nullable != column.nullable { + t.Errorf("nullable mismatch for column %q: %t != %t", name, nullable, column.nullable) + } + + // Length + // length, ok := tp.Length() + // if length != column.length { + // if !ok { + // t.Errorf("length not ok for column %q", name) + // } else { + // t.Errorf("length mismatch for column %q: %d != %d", name, length, column.length) + // } + // continue + // } + + // Precision and Scale + precision, scale, ok := tp.DecimalSize() + if precision != column.precision { + if !ok { + t.Errorf("precision not ok for column %q", name) + } else { + t.Errorf("precision mismatch for column %q: %d != %d", name, precision, column.precision) + } + continue + } + if scale != column.scale { + if !ok { + t.Errorf("scale not ok for column %q", name) + } else { + t.Errorf("scale mismatch for column %q: %d != %d", name, scale, column.scale) + } + continue + } + } + + values := make([]interface{}, len(tt)) + for i := range values { + values[i] = reflect.New(types[i]).Interface() + } + i := 0 + for rows.Next() { + err = rows.Scan(values...) + if err != nil { + t.Fatalf("failed to scan values in %v", err) + } + for j := range values { + value := reflect.ValueOf(values[j]).Elem().Interface() + if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { + if columns[j].scanType == scanTypeRawBytes { + t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) + } else { + t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) + } + } + } + i++ + } + if i != 3 { + t.Errorf("expected 3 rows, got %d", i) + } + + if err := rows.Close(); err != nil { + t.Errorf("error closing rows: %s", err) + } + }) + } +} + +func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (value VARCHAR(255))") + dbt.db.Exec("INSERT INTO test VALUES (?)", (*testValuer)(nil)) + // This test will panic on the INSERT if ConvertValue() does not check for typed nil before calling Value() + }) +} + +// TestRawBytesAreNotModified checks for a race condition that arises when a query context +// is canceled while a user is calling rows.Scan. This is a more stringent test than the one +// proposed in https://github.com/golang/go/issues/23519. Here we're explicitly using +// `sql.RawBytes` to check the contents of our internal buffers are not modified after an implicit +// call to `Rows.Close`, so Context cancellation should **not** invalidate the backing buffers. +func TestRawBytesAreNotModified(t *testing.T) { + const blob = "abcdefghijklmnop" + const contextRaceIterations = 20 + const blobSize = defaultBufSize * 3 / 4 // Second row overwrites first row. + const insertRows = 4 + + var sqlBlobs = [2]string{ + strings.Repeat(blob, blobSize/len(blob)), + strings.Repeat(strings.ToUpper(blob), blobSize/len(blob)), + } + + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + for i := 0; i < insertRows; i++ { + dbt.mustExec("INSERT INTO test VALUES (?, ?)", i+1, sqlBlobs[i&1]) + } + + for i := 0; i < contextRaceIterations; i++ { + func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rows, err := dbt.db.QueryContext(ctx, `SELECT id, value FROM test`) + if err != nil { + t.Fatal(err) + } + + var b int + var raw sql.RawBytes + for rows.Next() { + if err := rows.Scan(&b, &raw); err != nil { + t.Fatal(err) + } + + before := string(raw) + // Ensure cancelling the query does not corrupt the contents of `raw` + cancel() + time.Sleep(time.Microsecond * 100) + after := string(raw) + + if before != after { + t.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) + } + } + rows.Close() + }() + } + }) +} diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go new file mode 100644 index 0000000..1d9b4ab --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -0,0 +1,636 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rsa" + "crypto/tls" + "errors" + "fmt" + "math/big" + "net" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +var ( + errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") + errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") + errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") + errInvalidDSNUnsafeCollation = errors.New("invalid DSN: interpolateParams can not be used with unsafe collations") +) + +// Config is a configuration parsed from a DSN string. +// If a new Config is created instead of being parsed from a DSN string, +// the NewConfig function should be used, which sets default values. +type Config struct { + User string // Username + Passwd string // Password (requires User) + Net string // Network type + Addr string // Network address (requires Net) + DBName string // Database name + Params map[string]string // Connection parameters + Collation string // Connection collation + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + ServerPubKey string // Server public key name + pubKey *rsa.PublicKey // Server public key + TLSConfig string // TLS configuration name + tls *tls.Config // TLS configuration + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + + AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE + AllowCleartextPasswords bool // Allows the cleartext client side plugin + AllowNativePasswords bool // Allows the native password authentication method + AllowOldPasswords bool // Allows the old insecure password method + ClientFoundRows bool // Return number of matching rows instead of rows changed + ColumnsWithAlias bool // Prepend table alias to column names + InterpolateParams bool // Interpolate placeholders into query string + MultiStatements bool // Allow multiple statements in one query + ParseTime bool // Parse time values to time.Time + RejectReadOnly bool // Reject read-only connections +} + +// NewConfig creates a new Config and sets default values. +func NewConfig() *Config { + return &Config{ + Collation: defaultCollation, + Loc: time.UTC, + MaxAllowedPacket: defaultMaxAllowedPacket, + AllowNativePasswords: true, + } +} + +func (cfg *Config) Clone() *Config { + cp := *cfg + if cp.tls != nil { + cp.tls = cfg.tls.Clone() + } + if len(cp.Params) > 0 { + cp.Params = make(map[string]string, len(cfg.Params)) + for k, v := range cfg.Params { + cp.Params[k] = v + } + } + if cfg.pubKey != nil { + cp.pubKey = &rsa.PublicKey{ + N: new(big.Int).Set(cfg.pubKey.N), + E: cfg.pubKey.E, + } + } + return &cp +} + +func (cfg *Config) normalize() error { + if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { + return errInvalidDSNUnsafeCollation + } + + // Set default network if empty + if cfg.Net == "" { + cfg.Net = "tcp" + } + + // Set default address if empty + if cfg.Addr == "" { + switch cfg.Net { + case "tcp": + cfg.Addr = "127.0.0.1:3306" + case "unix": + cfg.Addr = "/tmp/mysql.sock" + default: + return errors.New("default addr for network '" + cfg.Net + "' unknown") + } + } else if cfg.Net == "tcp" { + cfg.Addr = ensureHavePort(cfg.Addr) + } + + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.tls = &tls.Config{} + case "skip-verify", "preferred": + cfg.tls = &tls.Config{InsecureSkipVerify: true} + default: + cfg.tls = getTLSConfigClone(cfg.TLSConfig) + if cfg.tls == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } + } + + if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + + if cfg.ServerPubKey != "" { + cfg.pubKey = getServerPubKey(cfg.ServerPubKey) + if cfg.pubKey == nil { + return errors.New("invalid value / unknown server pub key name: " + cfg.ServerPubKey) + } + } + + return nil +} + +// FormatDSN formats the given Config into a DSN string which can be passed to +// the driver. +func (cfg *Config) FormatDSN() string { + var buf bytes.Buffer + + // [username[:password]@] + if len(cfg.User) > 0 { + buf.WriteString(cfg.User) + if len(cfg.Passwd) > 0 { + buf.WriteByte(':') + buf.WriteString(cfg.Passwd) + } + buf.WriteByte('@') + } + + // [protocol[(address)]] + if len(cfg.Net) > 0 { + buf.WriteString(cfg.Net) + if len(cfg.Addr) > 0 { + buf.WriteByte('(') + buf.WriteString(cfg.Addr) + buf.WriteByte(')') + } + } + + // /dbname + buf.WriteByte('/') + buf.WriteString(cfg.DBName) + + // [?param1=value1&...¶mN=valueN] + hasParam := false + + if cfg.AllowAllFiles { + hasParam = true + buf.WriteString("?allowAllFiles=true") + } + + if cfg.AllowCleartextPasswords { + if hasParam { + buf.WriteString("&allowCleartextPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowCleartextPasswords=true") + } + } + + if !cfg.AllowNativePasswords { + if hasParam { + buf.WriteString("&allowNativePasswords=false") + } else { + hasParam = true + buf.WriteString("?allowNativePasswords=false") + } + } + + if cfg.AllowOldPasswords { + if hasParam { + buf.WriteString("&allowOldPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowOldPasswords=true") + } + } + + if cfg.ClientFoundRows { + if hasParam { + buf.WriteString("&clientFoundRows=true") + } else { + hasParam = true + buf.WriteString("?clientFoundRows=true") + } + } + + if col := cfg.Collation; col != defaultCollation && len(col) > 0 { + if hasParam { + buf.WriteString("&collation=") + } else { + hasParam = true + buf.WriteString("?collation=") + } + buf.WriteString(col) + } + + if cfg.ColumnsWithAlias { + if hasParam { + buf.WriteString("&columnsWithAlias=true") + } else { + hasParam = true + buf.WriteString("?columnsWithAlias=true") + } + } + + if cfg.InterpolateParams { + if hasParam { + buf.WriteString("&interpolateParams=true") + } else { + hasParam = true + buf.WriteString("?interpolateParams=true") + } + } + + if cfg.Loc != time.UTC && cfg.Loc != nil { + if hasParam { + buf.WriteString("&loc=") + } else { + hasParam = true + buf.WriteString("?loc=") + } + buf.WriteString(url.QueryEscape(cfg.Loc.String())) + } + + if cfg.MultiStatements { + if hasParam { + buf.WriteString("&multiStatements=true") + } else { + hasParam = true + buf.WriteString("?multiStatements=true") + } + } + + if cfg.ParseTime { + if hasParam { + buf.WriteString("&parseTime=true") + } else { + hasParam = true + buf.WriteString("?parseTime=true") + } + } + + if cfg.ReadTimeout > 0 { + if hasParam { + buf.WriteString("&readTimeout=") + } else { + hasParam = true + buf.WriteString("?readTimeout=") + } + buf.WriteString(cfg.ReadTimeout.String()) + } + + if cfg.RejectReadOnly { + if hasParam { + buf.WriteString("&rejectReadOnly=true") + } else { + hasParam = true + buf.WriteString("?rejectReadOnly=true") + } + } + + if len(cfg.ServerPubKey) > 0 { + if hasParam { + buf.WriteString("&serverPubKey=") + } else { + hasParam = true + buf.WriteString("?serverPubKey=") + } + buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + } + + if cfg.Timeout > 0 { + if hasParam { + buf.WriteString("&timeout=") + } else { + hasParam = true + buf.WriteString("?timeout=") + } + buf.WriteString(cfg.Timeout.String()) + } + + if len(cfg.TLSConfig) > 0 { + if hasParam { + buf.WriteString("&tls=") + } else { + hasParam = true + buf.WriteString("?tls=") + } + buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + } + + if cfg.WriteTimeout > 0 { + if hasParam { + buf.WriteString("&writeTimeout=") + } else { + hasParam = true + buf.WriteString("?writeTimeout=") + } + buf.WriteString(cfg.WriteTimeout.String()) + } + + if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { + if hasParam { + buf.WriteString("&maxAllowedPacket=") + } else { + hasParam = true + buf.WriteString("?maxAllowedPacket=") + } + buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) + + } + + // other params + if cfg.Params != nil { + var params []string + for param := range cfg.Params { + params = append(params, param) + } + sort.Strings(params) + for _, param := range params { + if hasParam { + buf.WriteByte('&') + } else { + hasParam = true + buf.WriteByte('?') + } + + buf.WriteString(param) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(cfg.Params[param])) + } + } + + return buf.String() +} + +// ParseDSN parses the DSN string to a Config +func ParseDSN(dsn string) (cfg *Config, err error) { + // New config with some default values + cfg = NewConfig() + + // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] + // Find the last '/' (since the password or the net addr might contain a '/') + foundSlash := false + for i := len(dsn) - 1; i >= 0; i-- { + if dsn[i] == '/' { + foundSlash = true + var j, k int + + // left part is empty if i <= 0 + if i > 0 { + // [username[:password]@][protocol[(address)]] + // Find the last '@' in dsn[:i] + for j = i; j >= 0; j-- { + if dsn[j] == '@' { + // username[:password] + // Find the first ':' in dsn[:j] + for k = 0; k < j; k++ { + if dsn[k] == ':' { + cfg.Passwd = dsn[k+1 : j] + break + } + } + cfg.User = dsn[:k] + + break + } + } + + // [protocol[(address)]] + // Find the first '(' in dsn[j+1:i] + for k = j + 1; k < i; k++ { + if dsn[k] == '(' { + // dsn[i-1] must be == ')' if an address is specified + if dsn[i-1] != ')' { + if strings.ContainsRune(dsn[k+1:i], ')') { + return nil, errInvalidDSNUnescaped + } + return nil, errInvalidDSNAddr + } + cfg.Addr = dsn[k+1 : i-1] + break + } + } + cfg.Net = dsn[j+1 : k] + } + + // dbname[?param1=value1&...¶mN=valueN] + // Find the first '?' in dsn[i+1:] + for j = i + 1; j < len(dsn); j++ { + if dsn[j] == '?' { + if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { + return + } + break + } + } + cfg.DBName = dsn[i+1 : j] + + break + } + } + + if !foundSlash && len(dsn) > 0 { + return nil, errInvalidDSNNoSlash + } + + if err = cfg.normalize(); err != nil { + return nil, err + } + return +} + +// parseDSNParams parses the DSN "query string" +// Values must be url.QueryEscape'ed +func parseDSNParams(cfg *Config, params string) (err error) { + for _, v := range strings.Split(params, "&") { + param := strings.SplitN(v, "=", 2) + if len(param) != 2 { + continue + } + + // cfg params + switch value := param[1]; param[0] { + // Disable INFILE whitelist / enable all files + case "allowAllFiles": + var isBool bool + cfg.AllowAllFiles, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use cleartext authentication mode (MySQL 5.5.10+) + case "allowCleartextPasswords": + var isBool bool + cfg.AllowCleartextPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use native password authentication + case "allowNativePasswords": + var isBool bool + cfg.AllowNativePasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use old authentication mode (pre MySQL 4.1) + case "allowOldPasswords": + var isBool bool + cfg.AllowOldPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Switch "rowsAffected" mode + case "clientFoundRows": + var isBool bool + cfg.ClientFoundRows, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Collation + case "collation": + cfg.Collation = value + break + + case "columnsWithAlias": + var isBool bool + cfg.ColumnsWithAlias, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Compression + case "compress": + return errors.New("compression not implemented yet") + + // Enable client side placeholder substitution + case "interpolateParams": + var isBool bool + cfg.InterpolateParams, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Time Location + case "loc": + if value, err = url.QueryUnescape(value); err != nil { + return + } + cfg.Loc, err = time.LoadLocation(value) + if err != nil { + return + } + + // multiple statements in one query + case "multiStatements": + var isBool bool + cfg.MultiStatements, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // time.Time parsing + case "parseTime": + var isBool bool + cfg.ParseTime, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // I/O read Timeout + case "readTimeout": + cfg.ReadTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // Reject read-only connections + case "rejectReadOnly": + var isBool bool + cfg.RejectReadOnly, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Server public key + case "serverPubKey": + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for server pub key name: %v", err) + } + cfg.ServerPubKey = name + + // Strict mode + case "strict": + panic("strict mode has been removed. See https://github.com/go-sql-driver/mysql/wiki/strict-mode") + + // Dial Timeout + case "timeout": + cfg.Timeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // TLS-Encryption + case "tls": + boolValue, isBool := readBool(value) + if isBool { + if boolValue { + cfg.TLSConfig = "true" + } else { + cfg.TLSConfig = "false" + } + } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { + cfg.TLSConfig = vl + } else { + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for TLS config name: %v", err) + } + cfg.TLSConfig = name + } + + // I/O write Timeout + case "writeTimeout": + cfg.WriteTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + case "maxAllowedPacket": + cfg.MaxAllowedPacket, err = strconv.Atoi(value) + if err != nil { + return + } + default: + // lazy init + if cfg.Params == nil { + cfg.Params = make(map[string]string) + } + + if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { + return + } + } + } + + return +} + +func ensureHavePort(addr string) string { + if _, _, err := net.SplitHostPort(addr); err != nil { + return net.JoinHostPort(addr, "3306") + } + return addr +} diff --git a/vendor/github.com/go-sql-driver/mysql/dsn_test.go b/vendor/github.com/go-sql-driver/mysql/dsn_test.go new file mode 100644 index 0000000..50dc293 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/dsn_test.go @@ -0,0 +1,415 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/tls" + "fmt" + "net/url" + "reflect" + "testing" + "time" +) + +var testDSNs = []struct { + in string + out *Config +}{{ + "username:password@protocol(address)/dbname?param=value", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, +}, { + "user@unix(/path/to/socket)/dbname?charset=utf8", + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, +}, { + "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, +}, { + "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, +}, { + "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "@/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:p@/ssword@/", + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "unix/?arg=%2Fsome%2Fpath.ext", + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(127.0.0.1)/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(de:ad:be:ef::ca:fe)/dbname", + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, +} + +func TestDSNParser(t *testing.T) { + for i, tst := range testDSNs { + cfg, err := ParseDSN(tst.in) + if err != nil { + t.Error(err.Error()) + } + + // pointer not static + cfg.tls = nil + + if !reflect.DeepEqual(cfg, tst.out) { + t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) + } + } +} + +func TestDSNParserInvalid(t *testing.T) { + var invalidDSNs = []string{ + "@net(addr/", // no closing brace + "@tcp(/", // no closing brace + "tcp(/", // no closing brace + "(/", // no closing brace + "net(addr)//", // unescaped + "User:pass@tcp(1.2.3.4:3306)", // no trailing slash + "net()/", // unknown default addr + //"/dbname?arg=/some/unescaped/path", + } + + for i, tst := range invalidDSNs { + if _, err := ParseDSN(tst); err == nil { + t.Errorf("invalid DSN #%d. (%s) didn't error!", i, tst) + } + } +} + +func TestDSNReformat(t *testing.T) { + for i, tst := range testDSNs { + dsn1 := tst.in + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Error(err.Error()) + continue + } + cfg1.tls = nil // pointer not static + res1 := fmt.Sprintf("%+v", cfg1) + + dsn2 := cfg1.FormatDSN() + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Error(err.Error()) + continue + } + cfg2.tls = nil // pointer not static + res2 := fmt.Sprintf("%+v", cfg2) + + if res1 != res2 { + t.Errorf("%d. %q does not match %q", i, res2, res1) + } + } +} + +func TestDSNServerPubKey(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + tst := baseDSN + "testKey" + cfg, err := ParseDSN(tst) + if err != nil { + t.Error(err.Error()) + } + + if cfg.ServerPubKey != "testKey" { + t.Errorf("unexpected cfg.ServerPubKey value: %v", cfg.ServerPubKey) + } + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } + + // Key is missing + tst = baseDSN + "invalid_name" + cfg, err = ParseDSN(tst) + if err == nil { + t.Errorf("invalid name in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } +} + +func TestDSNServerPubKeyQueryEscape(t *testing.T) { + const name = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + url.QueryEscape(name) + + RegisterServerPubKey(name, testPubKeyRSA) + defer DeregisterServerPubKey(name) + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } +} + +func TestDSNWithCustomTLS(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?tls=" + tlsCfg := tls.Config{} + + RegisterTLSConfig("utils_test", &tlsCfg) + defer DeregisterTLSConfig("utils_test") + + // Custom TLS is missing + tst := baseDSN + "invalid_tls" + cfg, err := ParseDSN(tst) + if err == nil { + t.Errorf("invalid custom TLS in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } + + tst = baseDSN + "utils_test" + + // Custom TLS with a server name + name := "foohost" + tlsCfg.ServerName = name + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, tst) + } + + // Custom TLS without a server name + name = "localhost" + tlsCfg.ServerName = "" + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct ServerName (%s) parsing DSN (%s).", name, tst) + } else if tlsCfg.ServerName != "" { + t.Errorf("tlsCfg was mutated ServerName (%s) should be empty parsing DSN (%s).", name, tst) + } +} + +func TestDSNTLSConfig(t *testing.T) { + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true" + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + dsn = "tcp(example.com)/?tls=true" + cfg, err = ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host without port)", expectedServerName, cfg.tls.ServerName) + } +} + +func TestDSNWithCustomTLSQueryEscape(t *testing.T) { + const configKey = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?tls=" + url.QueryEscape(configKey) + name := "foohost" + tlsCfg := tls.Config{ServerName: name} + + RegisterTLSConfig(configKey, &tlsCfg) + defer DeregisterTLSConfig(configKey) + + cfg, err := ParseDSN(dsn) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, dsn) + } +} + +func TestDSNUnsafeCollation(t *testing.T) { + _, err := ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=true") + if err != errInvalidDSNUnsafeCollation { + t.Errorf("expected %v, got %v", errInvalidDSNUnsafeCollation, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=false") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=ascii_bin&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=latin1_german1_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8mb4_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } +} + +func TestParamsAreSorted(t *testing.T) { + expected := "/dbname?interpolateParams=true&foobar=baz&quux=loo" + cfg := NewConfig() + cfg.DBName = "dbname" + cfg.InterpolateParams = true + cfg.Params = map[string]string{ + "quux": "loo", + "foobar": "baz", + } + actual := cfg.FormatDSN() + if actual != expected { + t.Errorf("generic Config.Params were not sorted: want %#v, got %#v", expected, actual) + } +} + +func TestCloneConfig(t *testing.T) { + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true&foobar=baz&serverPubKey=testKey" + cfg, err := ParseDSN(dsn) + if err != nil { + t.Fatal(err.Error()) + } + + cfg2 := cfg.Clone() + if cfg == cfg2 { + t.Errorf("Config.Clone did not create a separate config struct") + } + + if cfg2.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + cfg2.tls.ServerName = "example2.com" + if cfg.tls.ServerName == cfg2.tls.ServerName { + t.Errorf("changed cfg.tls.Server name should not propagate to original Config") + } + + if _, ok := cfg2.Params["foobar"]; !ok { + t.Errorf("cloned Config is missing custom params") + } + + delete(cfg2.Params, "foobar") + + if _, ok := cfg.Params["foobar"]; !ok { + t.Errorf("custom params in cloned Config should not propagate to original Config") + } + + if !reflect.DeepEqual(cfg.pubKey, cfg2.pubKey) { + t.Errorf("public key in Config should be identical") + } +} + +func TestNormalizeTLSConfig(t *testing.T) { + tt := []struct { + tlsConfig string + want *tls.Config + }{ + {"", nil}, + {"false", nil}, + {"true", &tls.Config{ServerName: "myserver"}}, + {"skip-verify", &tls.Config{InsecureSkipVerify: true}}, + {"preferred", &tls.Config{InsecureSkipVerify: true}}, + {"test_tls_config", &tls.Config{ServerName: "myServerName"}}, + } + + RegisterTLSConfig("test_tls_config", &tls.Config{ServerName: "myServerName"}) + defer func() { DeregisterTLSConfig("test_tls_config") }() + + for _, tc := range tt { + t.Run(tc.tlsConfig, func(t *testing.T) { + cfg := &Config{ + Addr: "myserver:3306", + TLSConfig: tc.tlsConfig, + } + + cfg.normalize() + + if cfg.tls == nil { + if tc.want != nil { + t.Fatal("wanted a tls config but got nil instead") + } + return + } + + if cfg.tls.ServerName != tc.want.ServerName { + t.Errorf("tls.ServerName doesn't match (want: '%s', got: '%s')", + tc.want.ServerName, cfg.tls.ServerName) + } + if cfg.tls.InsecureSkipVerify != tc.want.InsecureSkipVerify { + t.Errorf("tls.InsecureSkipVerify doesn't match (want: %T, got :%T)", + tc.want.InsecureSkipVerify, cfg.tls.InsecureSkipVerify) + } + }) + } +} + +func BenchmarkParseDSN(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + for _, tst := range testDSNs { + if _, err := ParseDSN(tst.in); err != nil { + b.Error(err.Error()) + } + } + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/errors.go b/vendor/github.com/go-sql-driver/mysql/errors.go index 44cf30d..760782f 100644 --- a/vendor/github.com/go-sql-driver/mysql/errors.go +++ b/vendor/github.com/go-sql-driver/mysql/errors.go @@ -9,30 +9,35 @@ package mysql import ( - "database/sql/driver" "errors" "fmt" - "io" "log" "os" ) // Various errors the driver might return. Can change between driver versions. var ( - ErrInvalidConn = errors.New("Invalid Connection") - ErrMalformPkt = errors.New("Malformed Packet") - ErrNoTLS = errors.New("TLS encryption requested but server does not support TLS") - ErrOldPassword = errors.New("This user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") - ErrCleartextPassword = errors.New("This user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN.") - ErrUnknownPlugin = errors.New("The authentication plugin is not supported.") - ErrOldProtocol = errors.New("MySQL-Server does not support required Protocol 41+") - ErrPktSync = errors.New("Commands out of sync. You can't run this command now") - ErrPktSyncMul = errors.New("Commands out of sync. Did you run multiple statements at once?") - ErrPktTooLarge = errors.New("Packet for query is too large. You can change this value on the server by adjusting the 'max_allowed_packet' variable.") - ErrBusyBuffer = errors.New("Busy buffer") + ErrInvalidConn = errors.New("invalid connection") + ErrMalformPkt = errors.New("malformed packet") + ErrNoTLS = errors.New("TLS requested but server does not support TLS") + ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") + ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") + ErrUnknownPlugin = errors.New("this authentication plugin is not supported") + ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") + ErrPktSync = errors.New("commands out of sync. You can't run this command now") + ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") + ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") + ErrBusyBuffer = errors.New("busy buffer") + + // errBadConnNoWrite is used for connection errors where nothing was sent to the database yet. + // If this happens first in a function starting a database interaction, it should be replaced by driver.ErrBadConn + // to trigger a resend. + // See https://github.com/go-sql-driver/mysql/pull/302 + errBadConnNoWrite = errors.New("bad connection") ) -var errLog Logger = log.New(os.Stderr, "[MySQL] ", log.Ldate|log.Ltime|log.Lshortfile) +var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) // Logger is used to log critical error messages. type Logger interface { @@ -58,74 +63,3 @@ type MySQLError struct { func (me *MySQLError) Error() string { return fmt.Sprintf("Error %d: %s", me.Number, me.Message) } - -// MySQLWarnings is an error type which represents a group of one or more MySQL -// warnings -type MySQLWarnings []MySQLWarning - -func (mws MySQLWarnings) Error() string { - var msg string - for i, warning := range mws { - if i > 0 { - msg += "\r\n" - } - msg += fmt.Sprintf( - "%s %s: %s", - warning.Level, - warning.Code, - warning.Message, - ) - } - return msg -} - -// MySQLWarning is an error type which represents a single MySQL warning. -// Warnings are returned in groups only. See MySQLWarnings -type MySQLWarning struct { - Level string - Code string - Message string -} - -func (mc *mysqlConn) getWarnings() (err error) { - rows, err := mc.Query("SHOW WARNINGS", nil) - if err != nil { - return - } - - var warnings = MySQLWarnings{} - var values = make([]driver.Value, 3) - - for { - err = rows.Next(values) - switch err { - case nil: - warning := MySQLWarning{} - - if raw, ok := values[0].([]byte); ok { - warning.Level = string(raw) - } else { - warning.Level = fmt.Sprintf("%s", values[0]) - } - if raw, ok := values[1].([]byte); ok { - warning.Code = string(raw) - } else { - warning.Code = fmt.Sprintf("%s", values[1]) - } - if raw, ok := values[2].([]byte); ok { - warning.Message = string(raw) - } else { - warning.Message = fmt.Sprintf("%s", values[0]) - } - - warnings = append(warnings, warning) - - case io.EOF: - return warnings - - default: - rows.Close() - return - } - } -} diff --git a/vendor/github.com/go-sql-driver/mysql/errors_test.go b/vendor/github.com/go-sql-driver/mysql/errors_test.go new file mode 100644 index 0000000..96f9126 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/errors_test.go @@ -0,0 +1,42 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "log" + "testing" +) + +func TestErrorsSetLogger(t *testing.T) { + previous := errLog + defer func() { + errLog = previous + }() + + // set up logger + const expected = "prefix: test\n" + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + logger := log.New(buffer, "prefix: ", 0) + + // print + SetLogger(logger) + errLog.Print("test") + + // check result + if actual := buffer.String(); actual != expected { + t.Errorf("expected %q, got %q", expected, actual) + } +} + +func TestErrorsStrictIgnoreNotes(t *testing.T) { + runTests(t, dsn+"&sql_notes=false", func(dbt *DBTest) { + dbt.mustExec("DROP TABLE IF EXISTS does_not_exist") + }) +} diff --git a/vendor/github.com/go-sql-driver/mysql/fields.go b/vendor/github.com/go-sql-driver/mysql/fields.go new file mode 100644 index 0000000..e1e2ece --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/fields.go @@ -0,0 +1,194 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql" + "reflect" +) + +func (mf *mysqlField) typeDatabaseName() string { + switch mf.fieldType { + case fieldTypeBit: + return "BIT" + case fieldTypeBLOB: + if mf.charSet != collations[binaryCollation] { + return "TEXT" + } + return "BLOB" + case fieldTypeDate: + return "DATE" + case fieldTypeDateTime: + return "DATETIME" + case fieldTypeDecimal: + return "DECIMAL" + case fieldTypeDouble: + return "DOUBLE" + case fieldTypeEnum: + return "ENUM" + case fieldTypeFloat: + return "FLOAT" + case fieldTypeGeometry: + return "GEOMETRY" + case fieldTypeInt24: + return "MEDIUMINT" + case fieldTypeJSON: + return "JSON" + case fieldTypeLong: + return "INT" + case fieldTypeLongBLOB: + if mf.charSet != collations[binaryCollation] { + return "LONGTEXT" + } + return "LONGBLOB" + case fieldTypeLongLong: + return "BIGINT" + case fieldTypeMediumBLOB: + if mf.charSet != collations[binaryCollation] { + return "MEDIUMTEXT" + } + return "MEDIUMBLOB" + case fieldTypeNewDate: + return "DATE" + case fieldTypeNewDecimal: + return "DECIMAL" + case fieldTypeNULL: + return "NULL" + case fieldTypeSet: + return "SET" + case fieldTypeShort: + return "SMALLINT" + case fieldTypeString: + if mf.charSet == collations[binaryCollation] { + return "BINARY" + } + return "CHAR" + case fieldTypeTime: + return "TIME" + case fieldTypeTimestamp: + return "TIMESTAMP" + case fieldTypeTiny: + return "TINYINT" + case fieldTypeTinyBLOB: + if mf.charSet != collations[binaryCollation] { + return "TINYTEXT" + } + return "TINYBLOB" + case fieldTypeVarChar: + if mf.charSet == collations[binaryCollation] { + return "VARBINARY" + } + return "VARCHAR" + case fieldTypeVarString: + if mf.charSet == collations[binaryCollation] { + return "VARBINARY" + } + return "VARCHAR" + case fieldTypeYear: + return "YEAR" + default: + return "" + } +} + +var ( + scanTypeFloat32 = reflect.TypeOf(float32(0)) + scanTypeFloat64 = reflect.TypeOf(float64(0)) + scanTypeInt8 = reflect.TypeOf(int8(0)) + scanTypeInt16 = reflect.TypeOf(int16(0)) + scanTypeInt32 = reflect.TypeOf(int32(0)) + scanTypeInt64 = reflect.TypeOf(int64(0)) + scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) + scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) + scanTypeNullTime = reflect.TypeOf(NullTime{}) + scanTypeUint8 = reflect.TypeOf(uint8(0)) + scanTypeUint16 = reflect.TypeOf(uint16(0)) + scanTypeUint32 = reflect.TypeOf(uint32(0)) + scanTypeUint64 = reflect.TypeOf(uint64(0)) + scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) + scanTypeUnknown = reflect.TypeOf(new(interface{})) +) + +type mysqlField struct { + tableName string + name string + length uint32 + flags fieldFlag + fieldType fieldType + decimals byte + charSet uint8 +} + +func (mf *mysqlField) scanType() reflect.Type { + switch mf.fieldType { + case fieldTypeTiny: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint8 + } + return scanTypeInt8 + } + return scanTypeNullInt + + case fieldTypeShort, fieldTypeYear: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint16 + } + return scanTypeInt16 + } + return scanTypeNullInt + + case fieldTypeInt24, fieldTypeLong: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint32 + } + return scanTypeInt32 + } + return scanTypeNullInt + + case fieldTypeLongLong: + if mf.flags&flagNotNULL != 0 { + if mf.flags&flagUnsigned != 0 { + return scanTypeUint64 + } + return scanTypeInt64 + } + return scanTypeNullInt + + case fieldTypeFloat: + if mf.flags&flagNotNULL != 0 { + return scanTypeFloat32 + } + return scanTypeNullFloat + + case fieldTypeDouble: + if mf.flags&flagNotNULL != 0 { + return scanTypeFloat64 + } + return scanTypeNullFloat + + case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, + fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, + fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, + fieldTypeTime: + return scanTypeRawBytes + + case fieldTypeDate, fieldTypeNewDate, + fieldTypeTimestamp, fieldTypeDateTime: + // NullTime is always returned for more consistent behavior as it can + // handle both cases of parseTime regardless if the field is nullable. + return scanTypeNullTime + + default: + return scanTypeUnknown + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/infile.go b/vendor/github.com/go-sql-driver/mysql/infile.go index 84c53a9..273cb0b 100644 --- a/vendor/github.com/go-sql-driver/mysql/infile.go +++ b/vendor/github.com/go-sql-driver/mysql/infile.go @@ -96,6 +96,10 @@ func deferredClose(err *error, closer io.Closer) { func (mc *mysqlConn) handleInFileRequest(name string) (err error) { var rdr io.Reader var data []byte + packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP + if mc.maxWriteSize < packetSize { + packetSize = mc.maxWriteSize + } if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader // The server might return an an absolute path. See issue #355. @@ -108,8 +112,6 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { if inMap { rdr = handler() if rdr != nil { - data = make([]byte, 4+mc.maxWriteSize) - if cl, ok := rdr.(io.Closer); ok { defer deferredClose(&err, cl) } @@ -124,7 +126,7 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { fileRegisterLock.RLock() fr := fileRegister[name] fileRegisterLock.RUnlock() - if mc.cfg.allowAllFiles || fr { + if mc.cfg.AllowAllFiles || fr { var file *os.File var fi os.FileInfo @@ -134,22 +136,20 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // get file size if fi, err = file.Stat(); err == nil { rdr = file - if fileSize := int(fi.Size()); fileSize <= mc.maxWriteSize { - data = make([]byte, 4+fileSize) - } else if fileSize <= mc.maxPacketAllowed { - data = make([]byte, 4+mc.maxWriteSize) - } else { - err = fmt.Errorf("Local File '%s' too large: Size: %d, Max: %d", name, fileSize, mc.maxPacketAllowed) + if fileSize := int(fi.Size()); fileSize < packetSize { + packetSize = fileSize } } } } else { - err = fmt.Errorf("Local File '%s' is not registered. Use the DSN parameter 'allowAllFiles=true' to allow all files", name) + err = fmt.Errorf("local file '%s' is not registered", name) } } // send content packets - if err == nil { + // if packetSize == 0, the Reader contains no data + if err == nil && packetSize > 0 { + data := make([]byte, 4+packetSize) var n int for err == nil { n, err = rdr.Read(data[4:]) @@ -175,8 +175,8 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // read OK packet if err == nil { return mc.readResultOK() - } else { - mc.readPacket() } + + mc.readPacket() return err } diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index 76cb7c8..30b3352 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -13,6 +13,7 @@ import ( "crypto/tls" "database/sql/driver" "encoding/binary" + "errors" "fmt" "io" "math" @@ -24,55 +25,66 @@ import ( // Read packet to buffer 'data' func (mc *mysqlConn) readPacket() ([]byte, error) { - var payload []byte + var prevData []byte for { - // Read packet header + // read packet header data, err := mc.buf.readNext(4) if err != nil { + if cerr := mc.canceled.Value(); cerr != nil { + return nil, cerr + } errLog.Print(err) mc.Close() - return nil, driver.ErrBadConn + return nil, ErrInvalidConn } - // Packet Length [24 bit] + // packet length [24 bit] pktLen := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) - if pktLen < 1 { - errLog.Print(ErrMalformPkt) - mc.Close() - return nil, driver.ErrBadConn - } - - // Check Packet Sync [8 bit] + // check packet sync [8 bit] if data[3] != mc.sequence { if data[3] > mc.sequence { return nil, ErrPktSyncMul - } else { - return nil, ErrPktSync } + return nil, ErrPktSync } mc.sequence++ - // Read packet body [pktLen bytes] + // packets with length 0 terminate a previous packet which is a + // multiple of (2^24)-1 bytes long + if pktLen == 0 { + // there was no previous packet + if prevData == nil { + errLog.Print(ErrMalformPkt) + mc.Close() + return nil, ErrInvalidConn + } + + return prevData, nil + } + + // read packet body [pktLen bytes] data, err = mc.buf.readNext(pktLen) if err != nil { + if cerr := mc.canceled.Value(); cerr != nil { + return nil, cerr + } errLog.Print(err) mc.Close() - return nil, driver.ErrBadConn + return nil, ErrInvalidConn } - isLastPacket := (pktLen < maxPacketSize) + // return data if this was the last packet + if pktLen < maxPacketSize { + // zero allocations for non-split packets + if prevData == nil { + return data, nil + } - // Zero allocations for non-splitting packets - if isLastPacket && payload == nil { - return data, nil + return append(prevData, data...), nil } - payload = append(payload, data...) - - if isLastPacket { - return payload, nil - } + prevData = append(prevData, data...) } } @@ -80,10 +92,39 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { func (mc *mysqlConn) writePacket(data []byte) error { pktLen := len(data) - 4 - if pktLen > mc.maxPacketAllowed { + if pktLen > mc.maxAllowedPacket { return ErrPktTooLarge } + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.reset { + mc.reset = false + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + // If this connection has a ReadTimeout which we've been setting on + // reads, reset it to its default value before we attempt a non-blocking + // read, otherwise the scheduler will just time us out before we can read + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Time{}) + } + if err == nil { + err = connCheck(conn) + } + if err != nil { + errLog.Print("closing bad idle connection: ", err) + mc.Close() + return driver.ErrBadConn + } + } + for { var size int if pktLen >= maxPacketSize { @@ -100,6 +141,12 @@ func (mc *mysqlConn) writePacket(data []byte) error { data[3] = mc.sequence // Write packet + if mc.writeTimeout > 0 { + if err := mc.netConn.SetWriteDeadline(time.Now().Add(mc.writeTimeout)); err != nil { + return err + } + } + n, err := mc.netConn.Write(data[:4+size]) if err == nil && n == 4+size { mc.sequence++ @@ -113,34 +160,48 @@ func (mc *mysqlConn) writePacket(data []byte) error { // Handle error if err == nil { // n != len(data) + mc.cleanup() errLog.Print(ErrMalformPkt) } else { + if cerr := mc.canceled.Value(); cerr != nil { + return cerr + } + if n == 0 && pktLen == len(data)-4 { + // only for the first loop iteration when nothing was written yet + return errBadConnNoWrite + } + mc.cleanup() errLog.Print(err) } - return driver.ErrBadConn + return ErrInvalidConn } } /****************************************************************************** -* Initialisation Process * +* Initialization Process * ******************************************************************************/ // Handshake Initialization Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake -func (mc *mysqlConn) readInitPacket() ([]byte, error) { - data, err := mc.readPacket() +func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) { + data, err = mc.readPacket() if err != nil { - return nil, err + // for init we can rewrite this to ErrBadConn for sql.Driver to retry, since + // in connection initialization we don't risk retrying non-idempotent actions. + if err == ErrInvalidConn { + return nil, "", driver.ErrBadConn + } + return } if data[0] == iERR { - return nil, mc.handleErrorPacket(data) + return nil, "", mc.handleErrorPacket(data) } // protocol version [1 byte] if data[0] < minProtocolVersion { - return nil, fmt.Errorf( - "Unsupported MySQL Protocol Version %d. Protocol Version %d or higher is required", + return nil, "", fmt.Errorf( + "unsupported protocol version %d. Version %d or higher is required", data[0], minProtocolVersion, ) @@ -151,7 +212,7 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { pos := 1 + bytes.IndexByte(data[1:], 0x00) + 1 + 4 // first part of the password cipher [8 bytes] - cipher := data[pos : pos+8] + authData := data[pos : pos+8] // (filler) always 0x00 [1 byte] pos += 8 + 1 @@ -159,10 +220,14 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // capability flags (lower 2 bytes) [2 bytes] mc.flags = clientFlag(binary.LittleEndian.Uint16(data[pos : pos+2])) if mc.flags&clientProtocol41 == 0 { - return nil, ErrOldProtocol + return nil, "", ErrOldProtocol } if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - return nil, ErrNoTLS + if mc.cfg.TLSConfig == "preferred" { + mc.cfg.tls = nil + } else { + return nil, "", ErrNoTLS + } } pos += 2 @@ -186,32 +251,32 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // // The official Python library uses the fixed length 12 // which seems to work but technically could have a hidden bug. - cipher = append(cipher, data[pos:pos+12]...) + authData = append(authData, data[pos:pos+12]...) + pos += 13 - // TODO: Verify string termination // EOF if version (>= 5.5.7 and < 5.5.10) or (>= 5.6.0 and < 5.6.2) // \NUL otherwise - // - //if data[len(data)-1] == 0 { - // return - //} - //return ErrMalformPkt + if end := bytes.IndexByte(data[pos:], 0x00); end != -1 { + plugin = string(data[pos : pos+end]) + } else { + plugin = string(data[pos:]) + } // make a memory safe copy of the cipher slice var b [20]byte - copy(b[:], cipher) - return b[:], nil + copy(b[:], authData) + return b[:], plugin, nil } // make a memory safe copy of the cipher slice var b [8]byte - copy(b[:], cipher) - return b[:], nil + copy(b[:], authData) + return b[:], plugin, nil } // Client Authentication Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse -func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { +func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error { // Adjust client flags based on server support clientFlags := clientProtocol41 | clientSecureConn | @@ -219,9 +284,10 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientTransactions | clientLocalFiles | clientPluginAuth | + clientMultiResults | mc.flags&clientLongFlag - if mc.cfg.clientFoundRows { + if mc.cfg.ClientFoundRows { clientFlags |= clientFoundRows } @@ -230,23 +296,34 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientFlags |= clientSSL } - // User Password - scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.passwd)) + if mc.cfg.MultiStatements { + clientFlags |= clientMultiStatements + } - pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.user) + 1 + 1 + len(scrambleBuff) + 21 + 1 + // encode length of the auth plugin data + var authRespLEIBuf [9]byte + authRespLen := len(authResp) + authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen)) + if len(authRespLEI) > 1 { + // if the length can not be written in 1 byte, it must be written as a + // length encoded integer + clientFlags |= clientPluginAuthLenEncClientData + } + + pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1 // To specify a db name - if n := len(mc.cfg.dbname); n > 0 { + if n := len(mc.cfg.DBName); n > 0 { clientFlags |= clientConnectWithDB pktLen += n + 1 } // Calculate packet length and get buffer with that size - data := mc.buf.takeSmallBuffer(pktLen + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeSmallBuffer(pktLen + 4) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // ClientFlags [32 bit] @@ -262,7 +339,14 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { data[11] = 0x00 // Charset [1 byte] - data[12] = mc.cfg.collation + var found bool + data[12], found = collations[mc.cfg.Collation] + if !found { + // Note possibility for false negatives: + // could be triggered although the collation is valid if the + // collations map does not contain entries the server supports. + return errors.New("unknown collation") + } // SSL Connection Request Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest @@ -277,8 +361,9 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { if err := tlsConn.Handshake(); err != nil { return err } + mc.rawConn = mc.netConn mc.netConn = tlsConn - mc.buf.rd = tlsConn + mc.buf.nc = tlsConn } // Filler [23 bytes] (all 0x00) @@ -288,69 +373,43 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { } // User [null terminated string] - if len(mc.cfg.user) > 0 { - pos += copy(data[pos:], mc.cfg.user) + if len(mc.cfg.User) > 0 { + pos += copy(data[pos:], mc.cfg.User) } data[pos] = 0x00 pos++ - // ScrambleBuffer [length encoded integer] - data[pos] = byte(len(scrambleBuff)) - pos += 1 + copy(data[pos+1:], scrambleBuff) + // Auth Data [length encoded integer] + pos += copy(data[pos:], authRespLEI) + pos += copy(data[pos:], authResp) // Databasename [null terminated string] - if len(mc.cfg.dbname) > 0 { - pos += copy(data[pos:], mc.cfg.dbname) + if len(mc.cfg.DBName) > 0 { + pos += copy(data[pos:], mc.cfg.DBName) data[pos] = 0x00 pos++ } - // Assume native client during response - pos += copy(data[pos:], "mysql_native_password") + pos += copy(data[pos:], plugin) data[pos] = 0x00 + pos++ // Send Auth packet - return mc.writePacket(data) + return mc.writePacket(data[:pos]) } -// Client old authentication packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeOldAuthPacket(cipher []byte) error { - // User password - scrambleBuff := scrambleOldPassword(cipher, []byte(mc.cfg.passwd)) - - // Calculate the packet length and add a tailing 0 - pktLen := len(scrambleBuff) + 1 - data := mc.buf.takeSmallBuffer(4 + pktLen) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn +func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { + pktLen := 4 + len(authData) + data, err := mc.buf.takeSmallBuffer(pktLen) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } - // Add the scrambled password [null terminated string] - copy(data[4:], scrambleBuff) - data[4+pktLen-1] = 0x00 - - return mc.writePacket(data) -} - -// Client clear text authentication packet -// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeClearAuthPacket() error { - // Calculate the packet length and add a tailing 0 - pktLen := len(mc.cfg.passwd) + 1 - data := mc.buf.takeSmallBuffer(4 + pktLen) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn - } - - // Add the clear password [null terminated string] - copy(data[4:], mc.cfg.passwd) - data[4+pktLen-1] = 0x00 - + // Add the auth data [EOF] + copy(data[4:], authData) return mc.writePacket(data) } @@ -362,11 +421,11 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // Add command byte @@ -381,11 +440,11 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { mc.sequence = 0 pktLen := 1 + len(arg) - data := mc.buf.takeBuffer(pktLen + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeBuffer(pktLen + 4) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // Add command byte @@ -402,11 +461,11 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1 + 4) - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // Add command byte @@ -426,37 +485,50 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { * Result Packets * ******************************************************************************/ +func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { + data, err := mc.readPacket() + if err != nil { + return nil, "", err + } + + // packet indicator + switch data[0] { + + case iOK: + return nil, "", mc.handleOkPacket(data) + + case iAuthMoreData: + return data[1:], "", err + + case iEOF: + if len(data) == 1 { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest + return nil, "mysql_old_password", nil + } + pluginEndIndex := bytes.IndexByte(data, 0x00) + if pluginEndIndex < 0 { + return nil, "", ErrMalformPkt + } + plugin := string(data[1:pluginEndIndex]) + authData := data[pluginEndIndex+1:] + return authData, plugin, nil + + default: // Error otherwise + return nil, "", mc.handleErrorPacket(data) + } +} + // Returns error if Packet is not an 'Result OK'-Packet func (mc *mysqlConn) readResultOK() error { data, err := mc.readPacket() - if err == nil { - // packet indicator - switch data[0] { - - case iOK: - return mc.handleOkPacket(data) - - case iEOF: - if len(data) > 1 { - plugin := string(data[1:bytes.IndexByte(data, 0x00)]) - if plugin == "mysql_old_password" { - // using old_passwords - return ErrOldPassword - } else if plugin == "mysql_clear_password" { - // using clear text password - return ErrCleartextPassword - } else { - return ErrUnknownPlugin - } - } else { - return ErrOldPassword - } - - default: // Error otherwise - return mc.handleErrorPacket(data) - } + if err != nil { + return err } - return err + + if data[0] == iOK { + return mc.handleOkPacket(data) + } + return mc.handleErrorPacket(data) } // Result Set Header Packet @@ -499,6 +571,22 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { // Error Number [16 bit uint] errno := binary.LittleEndian.Uint16(data[1:3]) + // 1792: ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION + // 1290: ER_OPTION_PREVENTS_STATEMENT (returned by Aurora during failover) + if (errno == 1792 || errno == 1290) && mc.cfg.RejectReadOnly { + // Oops; we are connected to a read-only connection, and won't be able + // to issue any write statements. Since RejectReadOnly is configured, + // we throw away this connection hoping this one would have write + // permission. This is specifically for a possible race condition + // during failover (e.g. on AWS Aurora). See README.md for more. + // + // We explicitly close the connection before returning + // driver.ErrBadConn to ensure that `database/sql` purges this + // connection and initiates a new one for next statement next time. + mc.Close() + return driver.ErrBadConn + } + pos := 3 // SQL State [optional: # + 5bytes string] @@ -514,6 +602,10 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { } } +func readStatus(b []byte) statusFlag { + return statusFlag(b[0]) | statusFlag(b[1])<<8 +} + // Ok Packet // http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet func (mc *mysqlConn) handleOkPacket(data []byte) error { @@ -528,18 +620,14 @@ func (mc *mysqlConn) handleOkPacket(data []byte) error { mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) // server_status [2 bytes] - mc.status = statusFlag(data[1+n+m]) | statusFlag(data[1+n+m+1])<<8 - - // warning count [2 bytes] - if !mc.strict { - return nil - } else { - pos := 1 + n + m + 2 - if binary.LittleEndian.Uint16(data[pos:pos+2]) > 0 { - return mc.getWarnings() - } + mc.status = readStatus(data[1+n+m : 1+n+m+2]) + if mc.status&statusMoreResultsExists != 0 { return nil } + + // warning count [2 bytes] + + return nil } // Read Packets as Field Packets until EOF-Packet or an Error appears @@ -558,7 +646,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { if i == count { return columns, nil } - return nil, fmt.Errorf("ColumnsCount mismatch n:%d len:%d", count, len(columns)) + return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns)) } // Catalog @@ -575,7 +663,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { pos += n // Table [len coded string] - if mc.cfg.columnsWithAlias { + if mc.cfg.ColumnsWithAlias { tableName, _, n, err := readLengthEncodedString(data[pos:]) if err != nil { return nil, err @@ -610,14 +698,21 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { if err != nil { return nil, err } + pos += n // Filler [uint8] + pos++ + // Charset [charset, collation uint8] + columns[i].charSet = data[pos] + pos += 2 + // Length [uint32] - pos += n + 1 + 2 + 4 + columns[i].length = binary.LittleEndian.Uint32(data[pos : pos+4]) + pos += 4 // Field type [uint8] - columns[i].fieldType = data[pos] + columns[i].fieldType = fieldType(data[pos]) pos++ // Flags [uint16] @@ -640,6 +735,10 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { func (rows *textRows) readRow(dest []driver.Value) error { mc := rows.mc + if rows.rs.done { + return io.EOF + } + data, err := mc.readPacket() if err != nil { return err @@ -647,7 +746,12 @@ func (rows *textRows) readRow(dest []driver.Value) error { // EOF Packet if data[0] == iEOF && len(data) == 5 { - rows.mc = nil + // server_status [2 bytes] + rows.mc.status = readStatus(data[3:]) + rows.rs.done = true + if !rows.HasNextResultSet() { + rows.mc = nil + } return io.EOF } if data[0] == iERR { @@ -669,12 +773,12 @@ func (rows *textRows) readRow(dest []driver.Value) error { if !mc.parseTime { continue } else { - switch rows.columns[i].fieldType { + switch rows.rs.columns[i].fieldType { case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeDate, fieldTypeNewDate: dest[i], err = parseDateTime( string(dest[i].([]byte)), - mc.cfg.loc, + mc.cfg.Loc, ) if err == nil { continue @@ -699,12 +803,19 @@ func (rows *textRows) readRow(dest []driver.Value) error { func (mc *mysqlConn) readUntilEOF() error { for { data, err := mc.readPacket() - - // No Err and no EOF Packet - if err == nil && data[0] != iEOF { - continue + if err != nil { + return err + } + + switch data[0] { + case iERR: + return mc.handleErrorPacket(data) + case iEOF: + if len(data) == 5 { + mc.status = readStatus(data[3:]) + } + return nil } - return err // Err or EOF } } @@ -734,22 +845,15 @@ func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) { // Reserved [8 bit] // Warning count [16 bit uint] - if !stmt.mc.strict { - return columnCount, nil - } else { - // Check for warnings count > 0, only available in MySQL > 4.1 - if len(data) >= 12 && binary.LittleEndian.Uint16(data[10:12]) > 0 { - return columnCount, stmt.mc.getWarnings() - } - return columnCount, nil - } + + return columnCount, nil } return 0, err } // http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { - maxLen := stmt.mc.maxPacketAllowed - 1 + maxLen := stmt.mc.maxAllowedPacket - 1 pktLen := maxLen // After the header (bytes 0-3) follows before the data: @@ -758,7 +862,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { // 2 bytes paramID const dataOffset = 1 + 4 + 2 - // Can not use the write buffer since + // Cannot use the write buffer since // a) the buffer is too small // b) it is in use data := make([]byte, 4+1+4+2+len(arg)) @@ -804,7 +908,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if len(args) != stmt.paramCount { return fmt.Errorf( - "Arguments count mismatch (Got: %d Has: %d)", + "argument count mismatch (got: %d; has: %d)", len(args), stmt.paramCount, ) @@ -813,20 +917,28 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { const minPktLen = 4 + 1 + 4 + 1 + 4 mc := stmt.mc + // Determine threshold dynamically to avoid packet size shortage. + longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1) + if longDataSize < 64 { + longDataSize = 64 + } + // Reset packet-sequence mc.sequence = 0 var data []byte + var err error if len(args) == 0 { - data = mc.buf.takeBuffer(minPktLen) + data, err = mc.buf.takeBuffer(minPktLen) } else { - data = mc.buf.takeCompleteBuffer() + data, err = mc.buf.takeCompleteBuffer() + // In this case the len(data) == cap(data) which is used to optimise the flow below. } - if data == nil { - // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) - return driver.ErrBadConn + if err != nil { + // cannot take the buffer. Something must be wrong with the connection + errLog.Print(err) + return errBadConnNoWrite } // command [1 byte] @@ -851,7 +963,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { pos := minPktLen var nullMask []byte - if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { + if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= cap(data) { // buffer has to be extended but we don't know by how much so // we depend on append after all data with known sizes fit. // We stop at that because we deal with a lot of columns here @@ -860,10 +972,11 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { copy(tmp[:pos], data[:pos]) data = tmp nullMask = data[pos : pos+maskLen] + // No need to clean nullMask as make ensures that. pos += maskLen } else { nullMask = data[pos : pos+maskLen] - for i := 0; i < maskLen; i++ { + for i := range nullMask { nullMask[i] = 0 } pos += maskLen @@ -885,7 +998,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // build NULL-bitmap if arg == nil { nullMask[i/8] |= 1 << (uint(i) & 7) - paramTypes[i+i] = fieldTypeNULL + paramTypes[i+i] = byte(fieldTypeNULL) paramTypes[i+i+1] = 0x00 continue } @@ -893,7 +1006,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // cache types and values switch v := arg.(type) { case int64: - paramTypes[i+i] = fieldTypeLongLong + paramTypes[i+i] = byte(fieldTypeLongLong) paramTypes[i+i+1] = 0x00 if cap(paramValues)-len(paramValues)-8 >= 0 { @@ -908,8 +1021,24 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { ) } + case uint64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x80 // type is unsigned + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + case float64: - paramTypes[i+i] = fieldTypeDouble + paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 if cap(paramValues)-len(paramValues)-8 >= 0 { @@ -925,7 +1054,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { } case bool: - paramTypes[i+i] = fieldTypeTiny + paramTypes[i+i] = byte(fieldTypeTiny) paramTypes[i+i+1] = 0x00 if v { @@ -937,10 +1066,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { case []byte: // Common case (non-nil value) first if v != nil { - paramTypes[i+i] = fieldTypeString + paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < longDataSize { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -955,14 +1084,14 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // Handle []byte(nil) as a NULL value nullMask[i/8] |= 1 << (uint(i) & 7) - paramTypes[i+i] = fieldTypeNULL + paramTypes[i+i] = byte(fieldTypeNULL) paramTypes[i+i+1] = 0x00 case string: - paramTypes[i+i] = fieldTypeString + paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < longDataSize { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -974,23 +1103,25 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { } case time.Time: - paramTypes[i+i] = fieldTypeString + paramTypes[i+i] = byte(fieldTypeString) paramTypes[i+i+1] = 0x00 - var val []byte + var a [64]byte + var b = a[:0] + if v.IsZero() { - val = []byte("0000-00-00") + b = append(b, "0000-00-00"...) } else { - val = []byte(v.In(mc.cfg.loc).Format(timeFormat)) + b = v.In(mc.cfg.Loc).AppendFormat(b, timeFormat) } paramValues = appendLengthEncodedInteger(paramValues, - uint64(len(val)), + uint64(len(b)), ) - paramValues = append(paramValues, val...) + paramValues = append(paramValues, b...) default: - return fmt.Errorf("Can't convert type: %T", arg) + return fmt.Errorf("cannot convert type: %T", arg) } } @@ -998,7 +1129,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In that case we must build the data packet with the new values buffer if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) - mc.buf.buf = data + if err = mc.buf.store(data); err != nil { + errLog.Print(err) + return errBadConnNoWrite + } } pos += len(paramValues) @@ -1008,6 +1142,26 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { return mc.writePacket(data) } +func (mc *mysqlConn) discardResults() error { + for mc.status&statusMoreResultsExists != 0 { + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return err + } + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { + return err + } + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } + } + } + return nil +} + // http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html func (rows *binaryRows) readRow(dest []driver.Value) error { data, err := rows.mc.readPacket() @@ -1017,14 +1171,20 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // packet indicator [1 byte] if data[0] != iOK { - rows.mc = nil // EOF Packet if data[0] == iEOF && len(data) == 5 { + rows.mc.status = readStatus(data[3:]) + rows.rs.done = true + if !rows.HasNextResultSet() { + rows.mc = nil + } return io.EOF } + mc := rows.mc + rows.mc = nil // Error otherwise - return rows.mc.handleErrorPacket(data) + return mc.handleErrorPacket(data) } // NULL-bitmap, [(column-count + 7 + 2) / 8 bytes] @@ -1040,14 +1200,14 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { } // Convert to byte-coded string - switch rows.columns[i].fieldType { + switch rows.rs.columns[i].fieldType { case fieldTypeNULL: dest[i] = nil continue // Numeric Types case fieldTypeTiny: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { dest[i] = int64(data[pos]) } else { dest[i] = int64(int8(data[pos])) @@ -1056,7 +1216,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeShort, fieldTypeYear: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { dest[i] = int64(binary.LittleEndian.Uint16(data[pos : pos+2])) } else { dest[i] = int64(int16(binary.LittleEndian.Uint16(data[pos : pos+2]))) @@ -1065,7 +1225,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeInt24, fieldTypeLong: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { dest[i] = int64(binary.LittleEndian.Uint32(data[pos : pos+4])) } else { dest[i] = int64(int32(binary.LittleEndian.Uint32(data[pos : pos+4]))) @@ -1074,7 +1234,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeLongLong: - if rows.columns[i].flags&flagUnsigned != 0 { + if rows.rs.columns[i].flags&flagUnsigned != 0 { val := binary.LittleEndian.Uint64(data[pos : pos+8]) if val > math.MaxInt64 { dest[i] = uint64ToString(val) @@ -1088,7 +1248,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeFloat: - dest[i] = float64(math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4]))) + dest[i] = math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4])) pos += 4 continue @@ -1101,7 +1261,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry: + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON: var isNull bool var n int dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) @@ -1128,41 +1288,41 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { case isNull: dest[i] = nil continue - case rows.columns[i].fieldType == fieldTypeTime: + case rows.rs.columns[i].fieldType == fieldTypeTime: // database/sql does not support an equivalent to TIME, return a string var dstlen uint8 - switch decimals := rows.columns[i].decimals; decimals { + switch decimals := rows.rs.columns[i].decimals; decimals { case 0x00, 0x1f: dstlen = 8 case 1, 2, 3, 4, 5, 6: dstlen = 8 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", - rows.columns[i].decimals, + "protocol error, illegal decimals value %d", + rows.rs.columns[i].decimals, ) } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, true) + dest[i], err = formatBinaryTime(data[pos:pos+int(num)], dstlen) case rows.mc.parseTime: - dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.loc) + dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.Loc) default: var dstlen uint8 - if rows.columns[i].fieldType == fieldTypeDate { + if rows.rs.columns[i].fieldType == fieldTypeDate { dstlen = 10 } else { - switch decimals := rows.columns[i].decimals; decimals { + switch decimals := rows.rs.columns[i].decimals; decimals { case 0x00, 0x1f: dstlen = 19 case 1, 2, 3, 4, 5, 6: dstlen = 19 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", - rows.columns[i].decimals, + "protocol error, illegal decimals value %d", + rows.rs.columns[i].decimals, ) } } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, false) + dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen) } if err == nil { @@ -1174,7 +1334,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // Please report if this happens! default: - return fmt.Errorf("Unknown FieldType %d", rows.columns[i].fieldType) + return fmt.Errorf("unknown field type %d", rows.rs.columns[i].fieldType) } } diff --git a/vendor/github.com/go-sql-driver/mysql/packets_test.go b/vendor/github.com/go-sql-driver/mysql/packets_test.go new file mode 100644 index 0000000..b61e4db --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/packets_test.go @@ -0,0 +1,336 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "errors" + "net" + "testing" + "time" +) + +var ( + errConnClosed = errors.New("connection is closed") + errConnTooManyReads = errors.New("too many reads") + errConnTooManyWrites = errors.New("too many writes") +) + +// struct to mock a net.Conn for testing purposes +type mockConn struct { + laddr net.Addr + raddr net.Addr + data []byte + written []byte + queuedReplies [][]byte + closed bool + read int + reads int + writes int + maxReads int + maxWrites int +} + +func (m *mockConn) Read(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.reads++ + if m.maxReads > 0 && m.reads > m.maxReads { + return 0, errConnTooManyReads + } + + n = copy(b, m.data) + m.read += n + m.data = m.data[n:] + return +} +func (m *mockConn) Write(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.writes++ + if m.maxWrites > 0 && m.writes > m.maxWrites { + return 0, errConnTooManyWrites + } + + n = len(b) + m.written = append(m.written, b...) + + if n > 0 && len(m.queuedReplies) > 0 { + m.data = m.queuedReplies[0] + m.queuedReplies = m.queuedReplies[1:] + } + return +} +func (m *mockConn) Close() error { + m.closed = true + return nil +} +func (m *mockConn) LocalAddr() net.Addr { + return m.laddr +} +func (m *mockConn) RemoteAddr() net.Addr { + return m.raddr +} +func (m *mockConn) SetDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetReadDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetWriteDeadline(t time.Time) error { + return nil +} + +// make sure mockConn implements the net.Conn interface +var _ net.Conn = new(mockConn) + +func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: NewConfig(), + netConn: conn, + closech: make(chan struct{}), + maxAllowedPacket: defaultMaxAllowedPacket, + sequence: sequence, + } + return conn, mc +} + +func TestReadPacketSingleByte(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 1 { + t.Fatalf("unexpected packet length: expected %d, got %d", 1, len(packet)) + } + if packet[0] != 0xff { + t.Fatalf("unexpected packet content: expected %x, got %x", 0xff, packet[0]) + } +} + +func TestReadPacketWrongSequenceID(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + // too low sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + mc.sequence = 1 + _, err := mc.readPacket() + if err != ErrPktSync { + t.Errorf("expected ErrPktSync, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // too high sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x42, 0xff} + _, err = mc.readPacket() + if err != ErrPktSyncMul { + t.Errorf("expected ErrPktSyncMul, got %v", err) + } +} + +func TestReadPacketSplit(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + data := make([]byte, maxPacketSize*2+4*3) + const pkt2ofs = maxPacketSize + 4 + const pkt3ofs = 2 * (maxPacketSize + 4) + + // case 1: payload has length maxPacketSize + data = data[:pkt2ofs+4] + + // 1st packet has maxPacketSize length and sequence id 0 + // ff ff ff 00 ... + data[0] = 0xff + data[1] = 0xff + data[2] = 0xff + + // mark the payload start and end of 1st packet so that we can check if the + // content was correctly appended + data[4] = 0x11 + data[maxPacketSize+3] = 0x22 + + // 2nd packet has payload length 0 and squence id 1 + // 00 00 00 01 + data[pkt2ofs+3] = 0x01 + + conn.data = data + conn.maxReads = 3 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize-1] != 0x22 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x22, packet[maxPacketSize-1]) + } + + // case 2: payload has length which is a multiple of maxPacketSize + data = data[:cap(data)] + + // 2nd packet now has maxPacketSize length + data[pkt2ofs] = 0xff + data[pkt2ofs+1] = 0xff + data[pkt2ofs+2] = 0xff + + // mark the payload start and end of the 2nd packet + data[pkt2ofs+4] = 0x33 + data[pkt2ofs+maxPacketSize+3] = 0x44 + + // 3rd packet has payload length 0 and squence id 2 + // 00 00 00 02 + data[pkt3ofs+3] = 0x02 + + conn.data = data + conn.reads = 0 + conn.maxReads = 5 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 2*maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", 2*maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[2*maxPacketSize-1] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[2*maxPacketSize-1]) + } + + // case 3: payload has a length larger maxPacketSize, which is not an exact + // multiple of it + data = data[:pkt2ofs+4+42] + data[pkt2ofs] = 0x2a + data[pkt2ofs+1] = 0x00 + data[pkt2ofs+2] = 0x00 + data[pkt2ofs+4+41] = 0x44 + + conn.data = data + conn.reads = 0 + conn.maxReads = 4 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize+42 { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize+42, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize+41] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[maxPacketSize+41]) + } +} + +func TestReadPacketFail(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + closech: make(chan struct{}), + } + + // illegal empty (stand-alone) packet + conn.data = []byte{0x00, 0x00, 0x00, 0x00} + conn.maxReads = 1 + _, err := mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read header + conn.closed = true + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.closed = false + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read body + conn.maxReads = 1 + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } +} + +// https://github.com/go-sql-driver/mysql/pull/801 +// not-NUL terminated plugin_name in init packet +func TestRegression801(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: new(Config), + sequence: 42, + closech: make(chan struct{}), + } + + conn.data = []byte{72, 0, 0, 42, 10, 53, 46, 53, 46, 56, 0, 165, 0, 0, 0, + 60, 70, 63, 58, 68, 104, 34, 97, 0, 223, 247, 33, 2, 0, 15, 128, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 120, 114, 47, 85, 75, 109, 99, 51, 77, + 50, 64, 0, 109, 121, 115, 113, 108, 95, 110, 97, 116, 105, 118, 101, 95, + 112, 97, 115, 115, 119, 111, 114, 100} + conn.maxReads = 1 + + authData, pluginName, err := mc.readHandshakePacket() + if err != nil { + t.Fatalf("got error: %v", err) + } + + if pluginName != "mysql_native_password" { + t.Errorf("expected plugin name 'mysql_native_password', got '%s'", pluginName) + } + + expectedAuthData := []byte{60, 70, 63, 58, 68, 104, 34, 97, 98, 120, 114, + 47, 85, 75, 109, 99, 51, 77, 50, 64} + if !bytes.Equal(authData, expectedAuthData) { + t.Errorf("expected authData '%v', got '%v'", expectedAuthData, authData) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/go-sql-driver/mysql/rows.go index ba606e1..888bdb5 100644 --- a/vendor/github.com/go-sql-driver/mysql/rows.go +++ b/vendor/github.com/go-sql-driver/mysql/rows.go @@ -11,19 +11,20 @@ package mysql import ( "database/sql/driver" "io" + "math" + "reflect" ) -type mysqlField struct { - tableName string - name string - flags fieldFlag - fieldType byte - decimals byte +type resultSet struct { + columns []mysqlField + columnNames []string + done bool } type mysqlRows struct { - mc *mysqlConn - columns []mysqlField + mc *mysqlConn + rs resultSet + finish func() } type binaryRows struct { @@ -34,45 +35,163 @@ type textRows struct { mysqlRows } -type emptyRows struct{} - func (rows *mysqlRows) Columns() []string { - columns := make([]string, len(rows.columns)) - if rows.mc.cfg.columnsWithAlias { + if rows.rs.columnNames != nil { + return rows.rs.columnNames + } + + columns := make([]string, len(rows.rs.columns)) + if rows.mc != nil && rows.mc.cfg.ColumnsWithAlias { for i := range columns { - if tableName := rows.columns[i].tableName; len(tableName) > 0 { - columns[i] = tableName + "." + rows.columns[i].name + if tableName := rows.rs.columns[i].tableName; len(tableName) > 0 { + columns[i] = tableName + "." + rows.rs.columns[i].name } else { - columns[i] = rows.columns[i].name + columns[i] = rows.rs.columns[i].name } } } else { for i := range columns { - columns[i] = rows.columns[i].name + columns[i] = rows.rs.columns[i].name } } + + rows.rs.columnNames = columns return columns } -func (rows *mysqlRows) Close() error { +func (rows *mysqlRows) ColumnTypeDatabaseTypeName(i int) string { + return rows.rs.columns[i].typeDatabaseName() +} + +// func (rows *mysqlRows) ColumnTypeLength(i int) (length int64, ok bool) { +// return int64(rows.rs.columns[i].length), true +// } + +func (rows *mysqlRows) ColumnTypeNullable(i int) (nullable, ok bool) { + return rows.rs.columns[i].flags&flagNotNULL == 0, true +} + +func (rows *mysqlRows) ColumnTypePrecisionScale(i int) (int64, int64, bool) { + column := rows.rs.columns[i] + decimals := int64(column.decimals) + + switch column.fieldType { + case fieldTypeDecimal, fieldTypeNewDecimal: + if decimals > 0 { + return int64(column.length) - 2, decimals, true + } + return int64(column.length) - 1, decimals, true + case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeTime: + return decimals, decimals, true + case fieldTypeFloat, fieldTypeDouble: + if decimals == 0x1f { + return math.MaxInt64, math.MaxInt64, true + } + return math.MaxInt64, decimals, true + } + + return 0, 0, false +} + +func (rows *mysqlRows) ColumnTypeScanType(i int) reflect.Type { + return rows.rs.columns[i].scanType() +} + +func (rows *mysqlRows) Close() (err error) { + if f := rows.finish; f != nil { + f() + rows.finish = nil + } + mc := rows.mc if mc == nil { return nil } - if mc.netConn == nil { - return ErrInvalidConn + if err := mc.error(); err != nil { + return err + } + + // flip the buffer for this connection if we need to drain it. + // note that for a successful query (i.e. one where rows.next() + // has been called until it returns false), `rows.mc` will be nil + // by the time the user calls `(*Rows).Close`, so we won't reach this + // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 + mc.buf.flip() + + // Remove unread packets from stream + if !rows.rs.done { + err = mc.readUntilEOF() + } + if err == nil { + if err = mc.discardResults(); err != nil { + return err + } + } + + rows.mc = nil + return err +} + +func (rows *mysqlRows) HasNextResultSet() (b bool) { + if rows.mc == nil { + return false + } + return rows.mc.status&statusMoreResultsExists != 0 +} + +func (rows *mysqlRows) nextResultSet() (int, error) { + if rows.mc == nil { + return 0, io.EOF + } + if err := rows.mc.error(); err != nil { + return 0, err } // Remove unread packets from stream - err := mc.readUntilEOF() - rows.mc = nil + if !rows.rs.done { + if err := rows.mc.readUntilEOF(); err != nil { + return 0, err + } + rows.rs.done = true + } + + if !rows.HasNextResultSet() { + rows.mc = nil + return 0, io.EOF + } + rows.rs = resultSet{} + return rows.mc.readResultSetHeaderPacket() +} + +func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) { + for { + resLen, err := rows.nextResultSet() + if err != nil { + return 0, err + } + + if resLen > 0 { + return resLen, nil + } + + rows.rs.done = true + } +} + +func (rows *binaryRows) NextResultSet() error { + resLen, err := rows.nextNotEmptyResultSet() + if err != nil { + return err + } + + rows.rs.columns, err = rows.mc.readColumns(resLen) return err } func (rows *binaryRows) Next(dest []driver.Value) error { if mc := rows.mc; mc != nil { - if mc.netConn == nil { - return ErrInvalidConn + if err := mc.error(); err != nil { + return err } // Fetch next row from stream @@ -81,10 +200,20 @@ func (rows *binaryRows) Next(dest []driver.Value) error { return io.EOF } +func (rows *textRows) NextResultSet() (err error) { + resLen, err := rows.nextNotEmptyResultSet() + if err != nil { + return err + } + + rows.rs.columns, err = rows.mc.readColumns(resLen) + return err +} + func (rows *textRows) Next(dest []driver.Value) error { if mc := rows.mc; mc != nil { - if mc.netConn == nil { - return ErrInvalidConn + if err := mc.error(); err != nil { + return err } // Fetch next row from stream @@ -92,15 +221,3 @@ func (rows *textRows) Next(dest []driver.Value) error { } return io.EOF } - -func (rows emptyRows) Columns() []string { - return nil -} - -func (rows emptyRows) Close() error { - return nil -} - -func (rows emptyRows) Next(dest []driver.Value) error { - return io.EOF -} diff --git a/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/go-sql-driver/mysql/statement.go index 6e869b3..f7e3709 100644 --- a/vendor/github.com/go-sql-driver/mysql/statement.go +++ b/vendor/github.com/go-sql-driver/mysql/statement.go @@ -11,20 +11,22 @@ package mysql import ( "database/sql/driver" "fmt" + "io" "reflect" - "strconv" ) type mysqlStmt struct { mc *mysqlConn id uint32 paramCount int - columns []mysqlField // cached from the first query } func (stmt *mysqlStmt) Close() error { - if stmt.mc == nil || stmt.mc.netConn == nil { - errLog.Print(ErrInvalidConn) + if stmt.mc == nil || stmt.mc.closed.IsSet() { + // driver.Stmt.Close can be called more than once, thus this function + // has to be idempotent. + // See also Issue #450 and golang/go#16019. + //errLog.Print(ErrInvalidConn) return driver.ErrBadConn } @@ -42,14 +44,14 @@ func (stmt *mysqlStmt) ColumnConverter(idx int) driver.ValueConverter { } func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { - if stmt.mc.netConn == nil { + if stmt.mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command err := stmt.writeExecutePacket(args) if err != nil { - return nil, err + return nil, stmt.mc.markBadConn(err) } mc := stmt.mc @@ -59,37 +61,45 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { // Read Result resLen, err := mc.readResultSetHeaderPacket() - if err == nil { - if resLen > 0 { - // Columns - err = mc.readUntilEOF() - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } - // Rows - err = mc.readUntilEOF() + if resLen > 0 { + // Columns + if err = mc.readUntilEOF(); err != nil { + return nil, err } - if err == nil { - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, nil + + // Rows + if err := mc.readUntilEOF(); err != nil { + return nil, err } } - return nil, err + if err := mc.discardResults(); err != nil { + return nil, err + } + + return &mysqlResult{ + affectedRows: int64(mc.affectedRows), + insertId: int64(mc.insertId), + }, nil } func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { - if stmt.mc.netConn == nil { + return stmt.query(args) +} + +func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { + if stmt.mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn } // Send command err := stmt.writeExecutePacket(args) if err != nil { - return nil, err + return nil, stmt.mc.markBadConn(err) } mc := stmt.mc @@ -101,17 +111,18 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { } rows := new(binaryRows) - rows.mc = mc if resLen > 0 { - // Columns - // If not cached, read them and cache them - if stmt.columns == nil { - rows.columns, err = mc.readColumns(resLen) - stmt.columns = rows.columns - } else { - rows.columns = stmt.columns - err = mc.readUntilEOF() + rows.mc = mc + rows.rs.columns, err = mc.readColumns(resLen) + } else { + rows.rs.done = true + + switch err := rows.NextResultSet(); err { + case nil, io.EOF: + return rows, nil + default: + return nil, err } } @@ -120,31 +131,74 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { type converter struct{} +// ConvertValue mirrors the reference/default converter in database/sql/driver +// with _one_ exception. We support uint64 with their high bit and the default +// implementation does not. This function should be kept in sync with +// database/sql/driver defaultConverter.ConvertValue() except for that +// deliberate difference. func (c converter) ConvertValue(v interface{}) (driver.Value, error) { if driver.IsValue(v) { return v, nil } + if vr, ok := v.(driver.Valuer); ok { + sv, err := callValuerValue(vr) + if err != nil { + return nil, err + } + if !driver.IsValue(sv) { + return nil, fmt.Errorf("non-Value type %T returned from Value", sv) + } + return sv, nil + } + rv := reflect.ValueOf(v) switch rv.Kind() { case reflect.Ptr: // indirect pointers if rv.IsNil() { return nil, nil + } else { + return c.ConvertValue(rv.Elem().Interface()) } - return c.ConvertValue(rv.Elem().Interface()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint(), nil case reflect.Float32, reflect.Float64: return rv.Float(), nil + case reflect.Bool: + return rv.Bool(), nil + case reflect.Slice: + ek := rv.Type().Elem().Kind() + if ek == reflect.Uint8 { + return rv.Bytes(), nil + } + return nil, fmt.Errorf("unsupported type %T, a slice of %s", v, ek) + case reflect.String: + return rv.String(), nil } return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) } + +var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() + +// callValuerValue returns vr.Value(), with one exception: +// If vr.Value is an auto-generated method on a pointer type and the +// pointer is nil, it would panic at runtime in the panicwrap +// method. Treat it like nil instead. +// +// This is so people can implement driver.Value on value types and +// still use nil pointers to those types to mean nil/NULL, just like +// string/*string. +// +// This is an exact copy of the same-named unexported function from the +// database/sql package. +func callValuerValue(vr driver.Valuer) (v driver.Value, err error) { + if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr && + rv.IsNil() && + rv.Type().Elem().Implements(valuerReflectType) { + return nil, nil + } + return vr.Value() +} diff --git a/vendor/github.com/go-sql-driver/mysql/statement_test.go b/vendor/github.com/go-sql-driver/mysql/statement_test.go new file mode 100644 index 0000000..4b9914f --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/statement_test.go @@ -0,0 +1,126 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "testing" +) + +func TestConvertDerivedString(t *testing.T) { + type derived string + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Derived string type not convertible", err) + } + + if output != "value" { + t.Fatalf("Derived string type not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedByteSlice(t *testing.T) { + type derived []uint8 + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Byte slice not convertible", err) + } + + if bytes.Compare(output.([]byte), []byte("value")) != 0 { + t.Fatalf("Byte slice not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedUnsupportedSlice(t *testing.T) { + type derived []int + + _, err := converter{}.ConvertValue(derived{1}) + if err == nil || err.Error() != "unsupported type mysql.derived, a slice of int" { + t.Fatal("Unexpected error", err) + } +} + +func TestConvertDerivedBool(t *testing.T) { + type derived bool + + output, err := converter{}.ConvertValue(derived(true)) + if err != nil { + t.Fatal("Derived bool type not convertible", err) + } + + if output != true { + t.Fatalf("Derived bool type not converted, got %#v %T", output, output) + } +} + +func TestConvertPointer(t *testing.T) { + str := "value" + + output, err := converter{}.ConvertValue(&str) + if err != nil { + t.Fatal("Pointer type not convertible", err) + } + + if output != "value" { + t.Fatalf("Pointer type not converted, got %#v %T", output, output) + } +} + +func TestConvertSignedIntegers(t *testing.T) { + values := []interface{}{ + int8(-42), + int16(-42), + int32(-42), + int64(-42), + int(-42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != int64(-42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } +} + +func TestConvertUnsignedIntegers(t *testing.T) { + values := []interface{}{ + uint8(42), + uint16(42), + uint32(42), + uint64(42), + uint(42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != uint64(42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } + + output, err := converter{}.ConvertValue(^uint64(0)) + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if output != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", output, output) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/transaction.go b/vendor/github.com/go-sql-driver/mysql/transaction.go index 33c749b..417d727 100644 --- a/vendor/github.com/go-sql-driver/mysql/transaction.go +++ b/vendor/github.com/go-sql-driver/mysql/transaction.go @@ -13,7 +13,7 @@ type mysqlTx struct { } func (tx *mysqlTx) Commit() (err error) { - if tx.mc == nil || tx.mc.netConn == nil { + if tx.mc == nil || tx.mc.closed.IsSet() { return ErrInvalidConn } err = tx.mc.exec("COMMIT") @@ -22,7 +22,7 @@ func (tx *mysqlTx) Commit() (err error) { } func (tx *mysqlTx) Rollback() (err error) { - if tx.mc == nil || tx.mc.netConn == nil { + if tx.mc == nil || tx.mc.closed.IsSet() { return ErrInvalidConn } err = tx.mc.exec("ROLLBACK") diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index 6a26ad1..cfa10e9 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -9,35 +9,32 @@ package mysql import ( - "crypto/sha1" "crypto/tls" + "database/sql" "database/sql/driver" "encoding/binary" "errors" "fmt" "io" - "net" - "net/url" + "strconv" "strings" + "sync" + "sync/atomic" "time" ) +// Registry for custom tls.Configs var ( - tlsConfigRegister map[string]*tls.Config // Register for custom tls.Configs - - errInvalidDSNUnescaped = errors.New("Invalid DSN: Did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("Invalid DSN: Network Address not terminated (missing closing brace)") - errInvalidDSNNoSlash = errors.New("Invalid DSN: Missing the slash separating the database name") - errInvalidDSNUnsafeCollation = errors.New("Invalid DSN: interpolateParams can be used with ascii, latin1, utf8 and utf8mb4 charset") + tlsConfigLock sync.RWMutex + tlsConfigRegistry map[string]*tls.Config ) -func init() { - tlsConfigRegister = make(map[string]*tls.Config) -} - // RegisterTLSConfig registers a custom tls.Config to be used with sql.Open. // Use the key as a value in the DSN where tls=value. // +// Note: The provided tls.Config is exclusively owned by the driver after +// registering it. +// // rootCertPool := x509.NewCertPool() // pem, err := ioutil.ReadFile("/path/ca-cert.pem") // if err != nil { @@ -59,243 +56,35 @@ func init() { // db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") // func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { - return fmt.Errorf("Key '%s' is reserved", key) + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { + return fmt.Errorf("key '%s' is reserved", key) } - tlsConfigRegister[key] = config + tlsConfigLock.Lock() + if tlsConfigRegistry == nil { + tlsConfigRegistry = make(map[string]*tls.Config) + } + + tlsConfigRegistry[key] = config + tlsConfigLock.Unlock() return nil } // DeregisterTLSConfig removes the tls.Config associated with key. func DeregisterTLSConfig(key string) { - delete(tlsConfigRegister, key) + tlsConfigLock.Lock() + if tlsConfigRegistry != nil { + delete(tlsConfigRegistry, key) + } + tlsConfigLock.Unlock() } -// parseDSN parses the DSN string to a config -func parseDSN(dsn string) (cfg *config, err error) { - // New config with some default values - cfg = &config{ - loc: time.UTC, - collation: defaultCollation, +func getTLSConfigClone(key string) (config *tls.Config) { + tlsConfigLock.RLock() + if v, ok := tlsConfigRegistry[key]; ok { + config = v.Clone() } - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.passwd = dsn[k+1 : j] - break - } - } - cfg.user = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - cfg.addr = dsn[k+1 : i-1] - break - } - } - cfg.net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.dbname = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - if cfg.interpolateParams && unsafeCollations[cfg.collation] { - return nil, errInvalidDSNUnsafeCollation - } - - // Set default network if empty - if cfg.net == "" { - cfg.net = "tcp" - } - - // Set default address if empty - if cfg.addr == "" { - switch cfg.net { - case "tcp": - cfg.addr = "127.0.0.1:3306" - case "unix": - cfg.addr = "/tmp/mysql.sock" - default: - return nil, errors.New("Default addr for network '" + cfg.net + "' unknown") - } - - } - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.interpolateParams, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Disable INFILE whitelist / enable all files - case "allowAllFiles": - var isBool bool - cfg.allowAllFiles, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Use cleartext authentication mode (MySQL 5.5.10+) - case "allowCleartextPasswords": - var isBool bool - cfg.allowCleartextPasswords, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Use old authentication mode (pre MySQL 4.1) - case "allowOldPasswords": - var isBool bool - cfg.allowOldPasswords, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Switch "rowsAffected" mode - case "clientFoundRows": - var isBool bool - cfg.clientFoundRows, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Collation - case "collation": - collation, ok := collations[value] - if !ok { - // Note possibility for false negatives: - // could be triggered although the collation is valid if the - // collations map does not contain entries the server supports. - err = errors.New("unknown collation") - return - } - cfg.collation = collation - break - - case "columnsWithAlias": - var isBool bool - cfg.columnsWithAlias, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // Dial Timeout - case "timeout": - cfg.timeout, err = time.ParseDuration(value) - if err != nil { - return - } - - // TLS-Encryption - case "tls": - boolValue, isBool := readBool(value) - if isBool { - if boolValue { - cfg.tls = &tls.Config{} - } - } else { - if strings.ToLower(value) == "skip-verify" { - cfg.tls = &tls.Config{InsecureSkipVerify: true} - } else if tlsConfig, ok := tlsConfigRegister[value]; ok { - if len(tlsConfig.ServerName) == 0 && !tlsConfig.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.addr) - if err == nil { - tlsConfig.ServerName = host - } - } - - cfg.tls = tlsConfig - } else { - return fmt.Errorf("Invalid value / unknown config name: %s", value) - } - } - - default: - // lazy init - if cfg.params == nil { - cfg.params = make(map[string]string) - } - - if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - + tlsConfigLock.RUnlock() return } @@ -313,119 +102,6 @@ func readBool(input string) (value bool, valid bool) { return } -/****************************************************************************** -* Authentication * -******************************************************************************/ - -// Encrypt password using 4.1+ method -func scramblePassword(scramble, password []byte) []byte { - if len(password) == 0 { - return nil - } - - // stage1Hash = SHA1(password) - crypt := sha1.New() - crypt.Write(password) - stage1 := crypt.Sum(nil) - - // scrambleHash = SHA1(scramble + SHA1(stage1Hash)) - // inner Hash - crypt.Reset() - crypt.Write(stage1) - hash := crypt.Sum(nil) - - // outer Hash - crypt.Reset() - crypt.Write(scramble) - crypt.Write(hash) - scramble = crypt.Sum(nil) - - // token = scrambleHash XOR stage1Hash - for i := range scramble { - scramble[i] ^= stage1[i] - } - return scramble -} - -// Encrypt password using pre 4.1 (old password) method -// https://github.com/atcurtis/mariadb/blob/master/mysys/my_rnd.c -type myRnd struct { - seed1, seed2 uint32 -} - -const myRndMaxVal = 0x3FFFFFFF - -// Pseudo random number generator -func newMyRnd(seed1, seed2 uint32) *myRnd { - return &myRnd{ - seed1: seed1 % myRndMaxVal, - seed2: seed2 % myRndMaxVal, - } -} - -// Tested to be equivalent to MariaDB's floating point variant -// http://play.golang.org/p/QHvhd4qved -// http://play.golang.org/p/RG0q4ElWDx -func (r *myRnd) NextByte() byte { - r.seed1 = (r.seed1*3 + r.seed2) % myRndMaxVal - r.seed2 = (r.seed1 + r.seed2 + 33) % myRndMaxVal - - return byte(uint64(r.seed1) * 31 / myRndMaxVal) -} - -// Generate binary hash from byte string using insecure pre 4.1 method -func pwHash(password []byte) (result [2]uint32) { - var add uint32 = 7 - var tmp uint32 - - result[0] = 1345345333 - result[1] = 0x12345671 - - for _, c := range password { - // skip spaces and tabs in password - if c == ' ' || c == '\t' { - continue - } - - tmp = uint32(c) - result[0] ^= (((result[0] & 63) + add) * tmp) + (result[0] << 8) - result[1] += (result[1] << 8) ^ result[0] - add += tmp - } - - // Remove sign bit (1<<31)-1) - result[0] &= 0x7FFFFFFF - result[1] &= 0x7FFFFFFF - - return -} - -// Encrypt password using insecure pre 4.1 method -func scrambleOldPassword(scramble, password []byte) []byte { - if len(password) == 0 { - return nil - } - - scramble = scramble[:8] - - hashPw := pwHash(password) - hashSc := pwHash(scramble) - - r := newMyRnd(hashPw[0]^hashSc[0], hashPw[1]^hashSc[1]) - - var out [8]byte - for i := range out { - out[i] = r.NextByte() + 64 - } - - mask := r.NextByte() - for i := range out { - out[i] ^= mask - } - - return out[:] -} - /****************************************************************************** * Time related utils * ******************************************************************************/ @@ -493,7 +169,7 @@ func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { } t, err = time.Parse(timeFormat[:len(str)], str) default: - err = fmt.Errorf("Invalid Time-String: %s", str) + err = fmt.Errorf("invalid time string: %s", str) return } @@ -542,7 +218,7 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va loc, ), nil } - return nil, fmt.Errorf("Invalid DATETIME-packet length %d", num) + return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } // zeroDateTime is used in formatBinaryDateTime to avoid an allocation @@ -554,87 +230,104 @@ var zeroDateTime = []byte("0000-00-00 00:00:00.000000") const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" -func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value, error) { +func appendMicrosecs(dst, src []byte, decimals int) []byte { + if decimals <= 0 { + return dst + } + if len(src) == 0 { + return append(dst, ".000000"[:decimals+1]...) + } + + microsecs := binary.LittleEndian.Uint32(src[:4]) + p1 := byte(microsecs / 10000) + microsecs -= 10000 * uint32(p1) + p2 := byte(microsecs / 100) + microsecs -= 100 * uint32(p2) + p3 := byte(microsecs) + + switch decimals { + default: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + digits10[p3], digits01[p3], + ) + case 1: + return append(dst, '.', + digits10[p1], + ) + case 2: + return append(dst, '.', + digits10[p1], digits01[p1], + ) + case 3: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], + ) + case 4: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + ) + case 5: + return append(dst, '.', + digits10[p1], digits01[p1], + digits10[p2], digits01[p2], + digits10[p3], + ) + } +} + +func formatBinaryDateTime(src []byte, length uint8) (driver.Value, error) { // length expects the deterministic length of the zero value, // negative time and 100+ hours are automatically added if needed if len(src) == 0 { - if justTime { - return zeroDateTime[11 : 11+length], nil - } return zeroDateTime[:length], nil } - var dst []byte // return value - var pt, p1, p2, p3 byte // current digit pair - var zOffs byte // offset of value in zeroDateTime - if justTime { - switch length { - case - 8, // time (can be up to 10 when negative and 100+ hours) - 10, 11, 12, 13, 14, 15: // time with fractional seconds - default: - return nil, fmt.Errorf("illegal TIME length %d", length) + var dst []byte // return value + var p1, p2, p3 byte // current digit pair + + switch length { + case 10, 19, 21, 22, 23, 24, 25, 26: + default: + t := "DATE" + if length > 10 { + t += "TIME" } - switch len(src) { - case 8, 12: - default: - return nil, fmt.Errorf("Invalid TIME-packet length %d", len(src)) - } - // +2 to enable negative time and 100+ hours - dst = make([]byte, 0, length+2) - if src[0] == 1 { - dst = append(dst, '-') - } - if src[1] != 0 { - hour := uint16(src[1])*24 + uint16(src[5]) - pt = byte(hour / 100) - p1 = byte(hour - 100*uint16(pt)) - dst = append(dst, digits01[pt]) - } else { - p1 = src[5] - } - zOffs = 11 - src = src[6:] - } else { - switch length { - case 10, 19, 21, 22, 23, 24, 25, 26: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s length %d", t, length) - } - switch len(src) { - case 4, 7, 11: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s-packet length %d", t, len(src)) - } - dst = make([]byte, 0, length) - // start with the date - year := binary.LittleEndian.Uint16(src[:2]) - pt = byte(year / 100) - p1 = byte(year - 100*uint16(pt)) - p2, p3 = src[2], src[3] - dst = append(dst, - digits10[pt], digits01[pt], - digits10[p1], digits01[p1], '-', - digits10[p2], digits01[p2], '-', - digits10[p3], digits01[p3], - ) - if length == 10 { - return dst, nil - } - if len(src) == 4 { - return append(dst, zeroDateTime[10:length]...), nil - } - dst = append(dst, ' ') - p1 = src[4] // hour - src = src[5:] + return nil, fmt.Errorf("illegal %s length %d", t, length) } + switch len(src) { + case 4, 7, 11: + default: + t := "DATE" + if length > 10 { + t += "TIME" + } + return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) + } + dst = make([]byte, 0, length) + // start with the date + year := binary.LittleEndian.Uint16(src[:2]) + pt := year / 100 + p1 = byte(year - 100*uint16(pt)) + p2, p3 = src[2], src[3] + dst = append(dst, + digits10[pt], digits01[pt], + digits10[p1], digits01[p1], '-', + digits10[p2], digits01[p2], '-', + digits10[p3], digits01[p3], + ) + if length == 10 { + return dst, nil + } + if len(src) == 4 { + return append(dst, zeroDateTime[10:length]...), nil + } + dst = append(dst, ' ') + p1 = src[4] // hour + src = src[5:] + // p1 is 2-digit hour, src is after hour p2, p3 = src[0], src[1] dst = append(dst, @@ -642,51 +335,49 @@ func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value digits10[p2], digits01[p2], ':', digits10[p3], digits01[p3], ) - if length <= byte(len(dst)) { - return dst, nil - } - src = src[2:] + return appendMicrosecs(dst, src[2:], int(length)-20), nil +} + +func formatBinaryTime(src []byte, length uint8) (driver.Value, error) { + // length expects the deterministic length of the zero value, + // negative time and 100+ hours are automatically added if needed if len(src) == 0 { - return append(dst, zeroDateTime[19:zOffs+length]...), nil + return zeroDateTime[11 : 11+length], nil } - microsecs := binary.LittleEndian.Uint32(src[:4]) - p1 = byte(microsecs / 10000) - microsecs -= 10000 * uint32(p1) - p2 = byte(microsecs / 100) - microsecs -= 100 * uint32(p2) - p3 = byte(microsecs) - switch decimals := zOffs + length - 20; decimals { + var dst []byte // return value + + switch length { + case + 8, // time (can be up to 10 when negative and 100+ hours) + 10, 11, 12, 13, 14, 15: // time with fractional seconds default: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - digits10[p3], digits01[p3], - ), nil - case 1: - return append(dst, '.', - digits10[p1], - ), nil - case 2: - return append(dst, '.', - digits10[p1], digits01[p1], - ), nil - case 3: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], - ), nil - case 4: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - ), nil - case 5: - return append(dst, '.', - digits10[p1], digits01[p1], - digits10[p2], digits01[p2], - digits10[p3], - ), nil + return nil, fmt.Errorf("illegal TIME length %d", length) } + switch len(src) { + case 8, 12: + default: + return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) + } + // +2 to enable negative time and 100+ hours + dst = make([]byte, 0, length+2) + if src[0] == 1 { + dst = append(dst, '-') + } + days := binary.LittleEndian.Uint32(src[1:5]) + hours := int64(days)*24 + int64(src[5]) + + if hours >= 100 { + dst = strconv.AppendInt(dst, hours, 10) + } else { + dst = append(dst, digits10[hours], digits01[hours]) + } + + min, sec := src[6], src[7] + dst = append(dst, ':', + digits10[min], digits01[min], ':', + digits10[sec], digits01[sec], + ) + return appendMicrosecs(dst, src[8:], int(length)-9), nil } /****************************************************************************** @@ -752,7 +443,7 @@ func readLengthEncodedString(b []byte) ([]byte, bool, int, error) { // Check data length if len(b) >= n { - return b[n-int(num) : n], false, n, nil + return b[n-int(num) : n : n], false, n, nil } return nil, false, n, io.EOF } @@ -781,8 +472,8 @@ func readLengthEncodedInteger(b []byte) (uint64, bool, int) { if len(b) == 0 { return 0, true, 1 } - switch b[0] { + switch b[0] { // 251: NULL case 0xfb: return 0, true, 1 @@ -877,7 +568,7 @@ func escapeBytesBackslash(buf, v []byte) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } } @@ -922,7 +613,7 @@ func escapeStringBackslash(buf []byte, v string) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } } @@ -971,3 +662,94 @@ func escapeStringQuotes(buf []byte, v string) []byte { return buf[:pos] } + +/****************************************************************************** +* Sync utils * +******************************************************************************/ + +// noCopy may be embedded into structs which must not be copied +// after the first use. +// +// See https://github.com/golang/go/issues/8005#issuecomment-190753527 +// for details. +type noCopy struct{} + +// Lock is a no-op used by -copylocks checker from `go vet`. +func (*noCopy) Lock() {} + +// atomicBool is a wrapper around uint32 for usage as a boolean value with +// atomic access. +type atomicBool struct { + _noCopy noCopy + value uint32 +} + +// IsSet returns whether the current boolean value is true +func (ab *atomicBool) IsSet() bool { + return atomic.LoadUint32(&ab.value) > 0 +} + +// Set sets the value of the bool regardless of the previous value +func (ab *atomicBool) Set(value bool) { + if value { + atomic.StoreUint32(&ab.value, 1) + } else { + atomic.StoreUint32(&ab.value, 0) + } +} + +// TrySet sets the value of the bool and returns whether the value changed +func (ab *atomicBool) TrySet(value bool) bool { + if value { + return atomic.SwapUint32(&ab.value, 1) == 0 + } + return atomic.SwapUint32(&ab.value, 0) > 0 +} + +// atomicError is a wrapper for atomically accessed error values +type atomicError struct { + _noCopy noCopy + value atomic.Value +} + +// Set sets the error value regardless of the previous value. +// The value must not be nil +func (ae *atomicError) Set(value error) { + ae.value.Store(value) +} + +// Value returns the current error value +func (ae *atomicError) Value() error { + if v := ae.value.Load(); v != nil { + // this will panic if the value doesn't implement the error interface + return v.(error) + } + return nil +} + +func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { + dargs := make([]driver.Value, len(named)) + for n, param := range named { + if len(param.Name) > 0 { + // TODO: support the use of Named Parameters #561 + return nil, errors.New("mysql: driver does not support the use of Named Parameters") + } + dargs[n] = param.Value + } + return dargs, nil +} + +func mapIsolationLevel(level driver.IsolationLevel) (string, error) { + switch sql.IsolationLevel(level) { + case sql.LevelRepeatableRead: + return "REPEATABLE READ", nil + case sql.LevelReadCommitted: + return "READ COMMITTED", nil + case sql.LevelReadUncommitted: + return "READ UNCOMMITTED", nil + case sql.LevelSerializable: + return "SERIALIZABLE", nil + default: + return "", fmt.Errorf("mysql: unsupported isolation level: %v", level) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/utils_test.go b/vendor/github.com/go-sql-driver/mysql/utils_test.go new file mode 100644 index 0000000..8951a7a --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/utils_test.go @@ -0,0 +1,334 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "database/sql" + "database/sql/driver" + "encoding/binary" + "testing" + "time" +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} + +func TestLengthEncodedInteger(t *testing.T) { + var integerTests = []struct { + num uint64 + encoded []byte + }{ + {0x0000000000000000, []byte{0x00}}, + {0x0000000000000012, []byte{0x12}}, + {0x00000000000000fa, []byte{0xfa}}, + {0x0000000000000100, []byte{0xfc, 0x00, 0x01}}, + {0x0000000000001234, []byte{0xfc, 0x34, 0x12}}, + {0x000000000000ffff, []byte{0xfc, 0xff, 0xff}}, + {0x0000000000010000, []byte{0xfd, 0x00, 0x00, 0x01}}, + {0x0000000000123456, []byte{0xfd, 0x56, 0x34, 0x12}}, + {0x0000000000ffffff, []byte{0xfd, 0xff, 0xff, 0xff}}, + {0x0000000001000000, []byte{0xfe, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}}, + {0x123456789abcdef0, []byte{0xfe, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12}}, + {0xffffffffffffffff, []byte{0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + } + + for _, tst := range integerTests { + num, isNull, numLen := readLengthEncodedInteger(tst.encoded) + if isNull { + t.Errorf("%x: expected %d, got NULL", tst.encoded, tst.num) + } + if num != tst.num { + t.Errorf("%x: expected %d, got %d", tst.encoded, tst.num, num) + } + if numLen != len(tst.encoded) { + t.Errorf("%x: expected size %d, got %d", tst.encoded, len(tst.encoded), numLen) + } + encoded := appendLengthEncodedInteger(nil, num) + if !bytes.Equal(encoded, tst.encoded) { + t.Errorf("%v: expected %x, got %x", num, tst.encoded, encoded) + } + } +} + +func TestFormatBinaryDateTime(t *testing.T) { + rawDate := [11]byte{} + binary.LittleEndian.PutUint16(rawDate[:2], 1978) // years + rawDate[2] = 12 // months + rawDate[3] = 30 // days + rawDate[4] = 15 // hours + rawDate[5] = 46 // minutes + rawDate[6] = 23 // seconds + binary.LittleEndian.PutUint32(rawDate[7:], 987654) // microseconds + expect := func(expected string, inlen, outlen uint8) { + actual, _ := formatBinaryDateTime(rawDate[:inlen], outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for length in %d, out %d", + expected, actual, inlen, outlen, + ) + } + } + expect("0000-00-00", 0, 10) + expect("0000-00-00 00:00:00", 0, 19) + expect("1978-12-30", 4, 10) + expect("1978-12-30 15:46:23", 7, 19) + expect("1978-12-30 15:46:23.987654", 11, 26) +} + +func TestFormatBinaryTime(t *testing.T) { + expect := func(expected string, src []byte, outlen uint8) { + actual, _ := formatBinaryTime(src, outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for src=%q and outlen=%d", + expected, actual, src, outlen) + } + } + + // binary format: + // sign (0: positive, 1: negative), days(4), hours, minutes, seconds, micro(4) + + // Zeros + expect("00:00:00", []byte{}, 8) + expect("00:00:00.0", []byte{}, 10) + expect("00:00:00.000000", []byte{}, 15) + + // Without micro(4) + expect("12:34:56", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("-12:34:56", []byte{1, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 11) + expect("24:34:56", []byte{0, 1, 0, 0, 0, 0, 34, 56}, 8) + expect("-99:34:56", []byte{1, 4, 0, 0, 0, 3, 34, 56}, 8) + expect("103079215103:34:56", []byte{0, 255, 255, 255, 255, 23, 34, 56}, 8) + + // With micro(4) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 11) + expect("12:34:56.000099", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 15) +} + +func TestEscapeBackslash(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesBackslash([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringBackslash([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\\0bar", "foo\x00bar") + expect("foo\\nbar", "foo\nbar") + expect("foo\\rbar", "foo\rbar") + expect("foo\\Zbar", "foo\x1abar") + expect("foo\\\"bar", "foo\"bar") + expect("foo\\\\bar", "foo\\bar") + expect("foo\\'bar", "foo'bar") +} + +func TestEscapeQuotes(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesQuotes([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringQuotes([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\x00bar", "foo\x00bar") // not affected + expect("foo\nbar", "foo\nbar") // not affected + expect("foo\rbar", "foo\rbar") // not affected + expect("foo\x1abar", "foo\x1abar") // not affected + expect("foo''bar", "foo'bar") // affected + expect("foo\"bar", "foo\"bar") // not affected +} + +func TestAtomicBool(t *testing.T) { + var ab atomicBool + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(true) + if ab.value != 1 { + t.Fatal("Set(true) did not set value to 1") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(false) + if ab.value != 0 { + t.Fatal("Set(false) did not set value to 0") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(false) + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + if ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to fail") + } + if !ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to succeed") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + if ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to fail") + } + if !ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to succeed") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab._noCopy.Lock() // we've "tested" it ¯\_(ツ)_/¯ +} + +func TestAtomicError(t *testing.T) { + var ae atomicError + if ae.Value() != nil { + t.Fatal("Expected value to be nil") + } + + ae.Set(ErrMalformPkt) + if v := ae.Value(); v != ErrMalformPkt { + if v == nil { + t.Fatal("Value is still nil") + } + t.Fatal("Error did not match") + } + ae.Set(ErrPktSync) + if ae.Value() == ErrMalformPkt { + t.Fatal("Error still matches old error") + } + if v := ae.Value(); v != ErrPktSync { + t.Fatal("Error did not match") + } +} + +func TestIsolationLevelMapping(t *testing.T) { + data := []struct { + level driver.IsolationLevel + expected string + }{ + { + level: driver.IsolationLevel(sql.LevelReadCommitted), + expected: "READ COMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelRepeatableRead), + expected: "REPEATABLE READ", + }, + { + level: driver.IsolationLevel(sql.LevelReadUncommitted), + expected: "READ UNCOMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelSerializable), + expected: "SERIALIZABLE", + }, + } + + for i, td := range data { + if actual, err := mapIsolationLevel(td.level); actual != td.expected || err != nil { + t.Fatal(i, td.expected, actual, err) + } + } + + // check unsupported mapping + expectedErr := "mysql: unsupported isolation level: 7" + actual, err := mapIsolationLevel(driver.IsolationLevel(sql.LevelLinearizable)) + if actual != "" || err == nil { + t.Fatal("Expected error on unsupported isolation level") + } + if err.Error() != expectedErr { + t.Fatalf("Expected error to be %q, got %q", expectedErr, err) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..d9771f1 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,21 @@ +### Issue description +Tell us what should happen and what happens instead + +### Example code +```go +If possible, please enter some example code here to reproduce the issue. +``` + +### Error log +``` +If you have an error log, please paste it here. +``` + +### Configuration +*Driver version (or git SHA):* + +*Go version:* run `go version` in your console + +*Server version:* E.g. MySQL 5.6, MariaDB 10.0.20 + +*Server OS:* E.g. Debian 8.1 (Jessie), Windows 10 diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6f5c7eb --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +### Description +Please explain the changes you made here. + +### Checklist +- [ ] Code compiles correctly +- [ ] Created tests which fail without the change (if possible) +- [ ] All tests passing +- [ ] Extended the README / documentation, if necessary +- [ ] Added myself / the copyright holder to the AUTHORS file diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore new file mode 100644 index 0000000..2de28da --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +Icon? +ehthumbs.db +Thumbs.db +.idea diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml new file mode 100644 index 0000000..eae311b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -0,0 +1,128 @@ +sudo: false +language: go +go: + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - master + +before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + +before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB" | sudo tee -a /etc/mysql/my.cnf + - sudo service mysql restart + - .travis/wait_mysql.sh + - mysql -e 'create database gotest;' + +matrix: + include: + - env: DB=MYSQL8 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:8.0 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:8.0 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MYSQL57 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mysql:5.7 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mysql:5.7 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA55 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:5.5 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:5.5 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - env: DB=MARIA10_1 + sudo: required + dist: trusty + go: 1.10.x + services: + - docker + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - docker pull mariadb:10.1 + - docker run -d -p 127.0.0.1:3307:3306 --name mysqld -e MYSQL_DATABASE=gotest -e MYSQL_USER=gotest -e MYSQL_PASSWORD=secret -e MYSQL_ROOT_PASSWORD=verysecret + mariadb:10.1 --innodb_log_file_size=256MB --innodb_buffer_pool_size=512MB --max_allowed_packet=16MB --local-infile=1 + - cp .travis/docker.cnf ~/.my.cnf + - .travis/wait_mysql.sh + before_script: + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3307 + - export MYSQL_TEST_CONCURRENT=1 + + - os: osx + osx_image: xcode10.1 + addons: + homebrew: + packages: + - mysql + go: 1.12.x + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + - mysql.server start + - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + - mysql -uroot -e 'create database gotest;' + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3306 + - export MYSQL_TEST_CONCURRENT=1 + +script: + - go test -v -covermode=count -coverprofile=coverage.out + - go vet ./... + - .travis/gofmt.sh +after_script: + - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf new file mode 100644 index 0000000..e57754e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/docker.cnf @@ -0,0 +1,5 @@ +[client] +user = gotest +password = secret +host = 127.0.0.1 +port = 3307 diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh new file mode 100755 index 0000000..9bf0d16 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/gofmt.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ev + +# Only check for go1.10+ since the gofmt style changed +if [[ $(go version) =~ go1\.([0-9]+) ]] && ((${BASH_REMATCH[1]} >= 10)); then + test -z "$(gofmt -d -s . | tee /dev/stderr)" +fi diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh new file mode 100755 index 0000000..e87993e --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/.travis/wait_mysql.sh @@ -0,0 +1,8 @@ +#!/bin/sh +while : +do + if mysql -e 'select version()' 2>&1 | grep 'version()\|ERROR 2059 (HY000):'; then + break + fi + sleep 3 +done diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS index fbe4ec4..bfe74c4 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -27,6 +27,7 @@ Daniël van Eeden Dave Protasowski DisposaBoy Egor Smolyakov +Erwan Martin Evan Shaw Frederick Mayle Gustavo Kristic @@ -34,12 +35,15 @@ Hajime Nakagami Hanno Braun Henri Yandell Hirotaka Yamamoto +Huyiguang ICHINOSE Shogo +Ilia Cimpoes INADA Naoki Jacek Szwec James Harr Jeff Hodges Jeffrey Charles +Jerome Meyer Jian Zhen Joshua Prunier Julien Lefevre @@ -69,10 +73,14 @@ Richard Wilkes Robert Russell Runrioter Wung Shuode Li +Simon J Mudd Soroush Pour Stan Putrya Stanley Gunawan +Steven Hartland Thomas Wodarek +Tim Ruffles +Tom Jenkinson Xiangyu Hu Xiaobing Jiang Xiuming Chen @@ -82,9 +90,12 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +Facebook Inc. +GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. +Multiplay Ltd. Percona LLC Pivotal Inc. Stripe Inc. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md new file mode 100644 index 0000000..2d87d74 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -0,0 +1,167 @@ +## Version 1.4 (2018-06-03) + +Changes: + + - Documentation fixes (#530, #535, #567) + - Refactoring (#575, #579, #580, #581, #603, #615, #704) + - Cache column names (#444) + - Sort the DSN parameters in DSNs generated from a config (#637) + - Allow native password authentication by default (#644) + - Use the default port if it is missing in the DSN (#668) + - Removed the `strict` mode (#676) + - Do not query `max_allowed_packet` by default (#680) + - Dropped support Go 1.6 and lower (#696) + - Updated `ConvertValue()` to match the database/sql/driver implementation (#760) + - Document the usage of `0000-00-00T00:00:00` as the time.Time zero value (#783) + - Improved the compatibility of the authentication system (#807) + +New Features: + + - Multi-Results support (#537) + - `rejectReadOnly` DSN option (#604) + - `context.Context` support (#608, #612, #627, #761) + - Transaction isolation level support (#619, #744) + - Read-Only transactions support (#618, #634) + - `NewConfig` function which initializes a config with default values (#679) + - Implemented the `ColumnType` interfaces (#667, #724) + - Support for custom string types in `ConvertValue` (#623) + - Implemented `NamedValueChecker`, improving support for uint64 with high bit set (#690, #709, #710) + - `caching_sha2_password` authentication plugin support (#794, #800, #801, #802) + - Implemented `driver.SessionResetter` (#779) + - `sha256_password` authentication plugin support (#808) + +Bugfixes: + + - Use the DSN hostname as TLS default ServerName if `tls=true` (#564, #718) + - Fixed LOAD LOCAL DATA INFILE for empty files (#590) + - Removed columns definition cache since it sometimes cached invalid data (#592) + - Don't mutate registered TLS configs (#600) + - Make RegisterTLSConfig concurrency-safe (#613) + - Handle missing auth data in the handshake packet correctly (#646) + - Do not retry queries when data was written to avoid data corruption (#302, #736) + - Cache the connection pointer for error handling before invalidating it (#678) + - Fixed imports for appengine/cloudsql (#700) + - Fix sending STMT_LONG_DATA for 0 byte data (#734) + - Set correct capacity for []bytes read from length-encoded strings (#766) + - Make RegisterDial concurrency-safe (#773) + + +## Version 1.3 (2016-12-01) + +Changes: + + - Go 1.1 is no longer supported + - Use decimals fields in MySQL to format time types (#249) + - Buffer optimizations (#269) + - TLS ServerName defaults to the host (#283) + - Refactoring (#400, #410, #437) + - Adjusted documentation for second generation CloudSQL (#485) + - Documented DSN system var quoting rules (#502) + - Made statement.Close() calls idempotent to avoid errors in Go 1.6+ (#512) + +New Features: + + - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) + - Support for returning table alias on Columns() (#289, #359, #382) + - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490) + - Support for uint64 parameters with high bit set (#332, #345) + - Cleartext authentication plugin support (#327) + - Exported ParseDSN function and the Config struct (#403, #419, #429) + - Read / Write timeouts (#401) + - Support for JSON field type (#414) + - Support for multi-statements and multi-results (#411, #431) + - DSN parameter to set the driver-side max_allowed_packet value manually (#489) + - Native password authentication plugin support (#494, #524) + +Bugfixes: + + - Fixed handling of queries without columns and rows (#255) + - Fixed a panic when SetKeepAlive() failed (#298) + - Handle ERR packets while reading rows (#321) + - Fixed reading NULL length-encoded integers in MySQL 5.6+ (#349) + - Fixed absolute paths support in LOAD LOCAL DATA INFILE (#356) + - Actually zero out bytes in handshake response (#378) + - Fixed race condition in registering LOAD DATA INFILE handler (#383) + - Fixed tests with MySQL 5.7.9+ (#380) + - QueryUnescape TLS config names (#397) + - Fixed "broken pipe" error by writing to closed socket (#390) + - Fixed LOAD LOCAL DATA INFILE buffering (#424) + - Fixed parsing of floats into float64 when placeholders are used (#434) + - Fixed DSN tests with Go 1.7+ (#459) + - Handle ERR packets while waiting for EOF (#473) + - Invalidate connection on error while discarding additional results (#513) + - Allow terminating packets of length 0 (#516) + + +## Version 1.2 (2014-06-03) + +Changes: + + - We switched back to a "rolling release". `go get` installs the current master branch again + - Version v1 of the driver will not be maintained anymore. Go 1.0 is no longer supported by this driver + - Exported errors to allow easy checking from application code + - Enabled TCP Keepalives on TCP connections + - Optimized INFILE handling (better buffer size calculation, lazy init, ...) + - The DSN parser also checks for a missing separating slash + - Faster binary date / datetime to string formatting + - Also exported the MySQLWarning type + - mysqlConn.Close returns the first error encountered instead of ignoring all errors + - writePacket() automatically writes the packet size to the header + - readPacket() uses an iterative approach instead of the recursive approach to merge splitted packets + +New Features: + + - `RegisterDial` allows the usage of a custom dial function to establish the network connection + - Setting the connection collation is possible with the `collation` DSN parameter. This parameter should be preferred over the `charset` parameter + - Logging of critical errors is configurable with `SetLogger` + - Google CloudSQL support + +Bugfixes: + + - Allow more than 32 parameters in prepared statements + - Various old_password fixes + - Fixed TestConcurrent test to pass Go's race detection + - Fixed appendLengthEncodedInteger for large numbers + - Renamed readLengthEnodedString to readLengthEncodedString and skipLengthEnodedString to skipLengthEncodedString (fixed typo) + + +## Version 1.1 (2013-11-02) + +Changes: + + - Go-MySQL-Driver now requires Go 1.1 + - Connections now use the collation `utf8_general_ci` by default. Adding `&charset=UTF8` to the DSN should not be necessary anymore + - Made closing rows and connections error tolerant. This allows for example deferring rows.Close() without checking for errors + - `[]byte(nil)` is now treated as a NULL value. Before, it was treated like an empty string / `[]byte("")` + - DSN parameter values must now be url.QueryEscape'ed. This allows text values to contain special characters, such as '&'. + - Use the IO buffer also for writing. This results in zero allocations (by the driver) for most queries + - Optimized the buffer for reading + - stmt.Query now caches column metadata + - New Logo + - Changed the copyright header to include all contributors + - Improved the LOAD INFILE documentation + - The driver struct is now exported to make the driver directly accessible + - Refactored the driver tests + - Added more benchmarks and moved all to a separate file + - Other small refactoring + +New Features: + + - Added *old_passwords* support: Required in some cases, but must be enabled by adding `allowOldPasswords=true` to the DSN since it is insecure + - Added a `clientFoundRows` parameter: Return the number of matching rows instead of the number of rows changed on UPDATEs + - Added TLS/SSL support: Use a TLS/SSL encrypted connection to the server. Custom TLS configs can be registered and used + +Bugfixes: + + - Fixed MySQL 4.1 support: MySQL 4.1 sends packets with lengths which differ from the specification + - Convert to DB timezone when inserting `time.Time` + - Splitted packets (more than 16MB) are now merged correctly + - Fixed false positive `io.EOF` errors when the data was fully read + - Avoid panics on reuse of closed connections + - Fixed empty string producing false nil values + - Fixed sign byte for positive TIME fields + + +## Version 1.0 (2013-05-14) + +Initial Release diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md new file mode 100644 index 0000000..8fe16bc --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md @@ -0,0 +1,23 @@ +# Contributing Guidelines + +## Reporting Issues + +Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed). + +## Contributing Code + +By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file. +Don't forget to add yourself to the AUTHORS file. + +### Code Review + +Everyone is invited to review and comment on pull requests. +If it looks fine to you, comment with "LGTM" (Looks good to me). + +If changes are required, notice the reviewers with "PTAL" (Please take another look) after committing the fixes. + +Before merging the Pull Request, at least one [team member](https://github.com/go-sql-driver?tab=members) must have commented with "LGTM". + +## Development Ideas + +If you are looking for ideas for code contributions, please check our [Development Ideas](https://github.com/go-sql-driver/mysql/wiki/Development-Ideas) Wiki page. diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md new file mode 100644 index 0000000..c6adf1d --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/README.md @@ -0,0 +1,495 @@ +# Go-MySQL-Driver + +A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package + +![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin") + +--------------------------------------- + * [Features](#features) + * [Requirements](#requirements) + * [Installation](#installation) + * [Usage](#usage) + * [DSN (Data Source Name)](#dsn-data-source-name) + * [Password](#password) + * [Protocol](#protocol) + * [Address](#address) + * [Parameters](#parameters) + * [Examples](#examples) + * [Connection pool and timeouts](#connection-pool-and-timeouts) + * [context.Context Support](#contextcontext-support) + * [ColumnType Support](#columntype-support) + * [LOAD DATA LOCAL INFILE support](#load-data-local-infile-support) + * [time.Time support](#timetime-support) + * [Unicode support](#unicode-support) + * [Testing / Development](#testing--development) + * [License](#license) + +--------------------------------------- + +## Features + * Lightweight and [fast](https://github.com/go-sql-driver/sql-benchmark "golang MySQL-Driver performance") + * Native Go implementation. No C-bindings, just pure Go + * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](https://godoc.org/github.com/go-sql-driver/mysql#DialFunc) + * Automatic handling of broken connections + * Automatic Connection Pooling *(by database/sql package)* + * Supports queries larger than 16MB + * Full [`sql.RawBytes`](https://golang.org/pkg/database/sql/#RawBytes) support. + * Intelligent `LONG DATA` handling in prepared statements + * Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support + * Optional `time.Time` parsing + * Optional placeholder interpolation + +## Requirements + * Go 1.9 or higher. We aim to support the 3 latest versions of Go. + * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) + +--------------------------------------- + +## Installation +Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell: +```bash +$ go get -u github.com/go-sql-driver/mysql +``` +Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`. + +## Usage +_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](https://golang.org/pkg/database/sql/) API then. + +Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: +```go +import "database/sql" +import _ "github.com/go-sql-driver/mysql" + +db, err := sql.Open("mysql", "user:password@/dbname") +``` + +[Examples are available in our Wiki](https://github.com/go-sql-driver/mysql/wiki/Examples "Go-MySQL-Driver Examples"). + + +### DSN (Data Source Name) + +The Data Source Name has a common format, like e.g. [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php) uses it, but without type-prefix (optional parts marked by squared brackets): +``` +[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] +``` + +A DSN in its fullest form: +``` +username:password@protocol(address)/dbname?param=value +``` + +Except for the databasename, all values are optional. So the minimal DSN is: +``` +/dbname +``` + +If you do not want to preselect a database, leave `dbname` empty: +``` +/ +``` +This has the same effect as an empty DSN string: +``` + +``` + +Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. + +#### Password +Passwords can consist of any character. Escaping is **not** necessary. + +#### Protocol +See [net.Dial](https://golang.org/pkg/net/#Dial) for more information which networks are available. +In general you should use an Unix domain socket if available and TCP otherwise for best performance. + +#### Address +For TCP and UDP networks, addresses have the form `host[:port]`. +If `port` is omitted, the default port will be used. +If `host` is a literal IPv6 address, it must be enclosed in square brackets. +The functions [net.JoinHostPort](https://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](https://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. + +For Unix domain sockets the address is the absolute path to the MySQL-Server-socket, e.g. `/var/run/mysqld/mysqld.sock` or `/tmp/mysql.sock`. + +#### Parameters +*Parameters are case-sensitive!* + +Notice that any of `true`, `TRUE`, `True` or `1` is accepted to stand for a true boolean value. Not surprisingly, false can be specified as any of: `false`, `FALSE`, `False` or `0`. + +##### `allowAllFiles` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowAllFiles=true` disables the file Whitelist for `LOAD DATA LOCAL INFILE` and allows *all* files. +[*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) + +##### `allowCleartextPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. + +##### `allowNativePasswords` + +``` +Type: bool +Valid Values: true, false +Default: true +``` +`allowNativePasswords=false` disallows the usage of MySQL native password method. + +##### `allowOldPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` +`allowOldPasswords=true` allows the usage of the insecure old password method. This should be avoided, but is necessary in some cases. See also [the old_passwords wiki page](https://github.com/go-sql-driver/mysql/wiki/old_passwords). + +##### `charset` + +``` +Type: string +Valid Values: +Default: none +``` + +Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). + +Usage of the `charset` parameter is discouraged because it issues additional queries to the server. +Unless you need the fallback behavior, please use `collation` instead. + +##### `collation` + +``` +Type: string +Valid Values: +Default: utf8mb4_general_ci +``` + +Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. + +A list of valid charsets for a server is retrievable with `SHOW COLLATION`. + +The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You should use an older collation (e.g. `utf8_general_ci`) for older MySQL. + +Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). + + +##### `clientFoundRows` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`clientFoundRows=true` causes an UPDATE to return the number of matching rows instead of the number of rows changed. + +##### `columnsWithAlias` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +When `columnsWithAlias` is true, calls to `sql.Rows.Columns()` will return the table alias and the column name separated by a dot. For example: + +``` +SELECT u.id FROM users as u +``` + +will return `u.id` instead of just `id` if `columnsWithAlias=true`. + +##### `interpolateParams` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +If `interpolateParams` is true, placeholders (`?`) in calls to `db.Query()` and `db.Exec()` are interpolated into a single query string with given parameters. This reduces the number of roundtrips, since the driver has to prepare a statement, execute it with given parameters and close the statement again with `interpolateParams=false`. + +*This can not be used together with the multibyte encodings BIG5, CP932, GB2312, GBK or SJIS. These are blacklisted as they may [introduce a SQL injection vulnerability](http://stackoverflow.com/a/12118602/3430118)!* + +##### `loc` + +``` +Type: string +Valid Values: +Default: UTC +``` + +Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](https://golang.org/pkg/time/#LoadLocation) for details. + +Note that this sets the location for time.Time values but does not change MySQL's [time_zone setting](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html). For that see the [time_zone system variable](#system-variables), which can also be set as a DSN parameter. + +Please keep in mind, that param values must be [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. + +##### `maxAllowedPacket` +``` +Type: decimal number +Default: 4194304 +``` + +Max packet size allowed in bytes. The default value is 4 MiB and should be adjusted to match the server settings. `maxAllowedPacket=0` can be used to automatically fetch the `max_allowed_packet` variable from server *on every connection*. + +##### `multiStatements` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. + +When `multiStatements` is used, `?` parameters must only be used in the first statement. + +##### `parseTime` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`parseTime=true` changes the output type of `DATE` and `DATETIME` values to `time.Time` instead of `[]byte` / `string` +The date or datetime like `0000-00-00 00:00:00` is converted into zero value of `time.Time`. + + +##### `readTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O read timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + +##### `rejectReadOnly` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + + +`rejectReadOnly=true` causes the driver to reject read-only connections. This +is for a possible race condition during an automatic failover, where the mysql +client gets connected to a read-only replica after the failover. + +Note that this should be a fairly rare case, as an automatic failover normally +happens when the primary is down, and the race condition shouldn't happen +unless it comes back up online as soon as the failover is kicked off. On the +other hand, when this happens, a MySQL application can get stuck on a +read-only connection until restarted. It is however fairly easy to reproduce, +for example, using a manual failover on AWS Aurora's MySQL-compatible cluster. + +If you are not relying on read-only transactions to reject writes that aren't +supposed to happen, setting this on some MySQL providers (such as AWS Aurora) +is safer for failovers. + +Note that ERROR 1290 can be returned for a `read-only` server and this option will +cause a retry for that error. However the same error number is used for some +other cases. You should ensure your application will never cause an ERROR 1290 +except for `read-only` mode when enabling this option. + + +##### `serverPubKey` + +``` +Type: string +Valid Values: +Default: none +``` + +Server public keys can be registered with [`mysql.RegisterServerPubKey`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterServerPubKey), which can then be used by the assigned name in the DSN. +Public keys are used to transmit encrypted data, e.g. for authentication. +If the server's public key is known, it should be set manually to avoid expensive and potentially insecure transmissions of the public key from the server to the client each time it is required. + + +##### `timeout` + +``` +Type: duration +Default: OS default +``` + +Timeout for establishing connections, aka dial timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + + +##### `tls` + +``` +Type: bool / string +Valid Values: true, false, skip-verify, preferred, +Default: false +``` + +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side) or use `preferred` to use TLS only when advertised by the server. This is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Neither `skip-verify` nor `preferred` add any reliable security. You can use a custom TLS config after registering it with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). + + +##### `writeTimeout` + +``` +Type: duration +Default: 0 +``` + +I/O write timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + + +##### System Variables + +Any other parameters are interpreted as system variables: + * `=`: `SET =` + * `=`: `SET =` + * `=%27%27`: `SET =''` + +Rules: +* The values for string variables must be quoted with `'`. +* The values must also be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed! + (which implies values of string variables must be wrapped with `%27`). + +Examples: + * `autocommit=1`: `SET autocommit=1` + * [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'` + * [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'` + + +#### Examples +``` +user@unix(/path/to/socket)/dbname +``` + +``` +root:pw@unix(/tmp/mysql.sock)/myDatabase?loc=Local +``` + +``` +user:password@tcp(localhost:5555)/dbname?tls=skip-verify&autocommit=true +``` + +Treat warnings as errors by setting the system variable [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html): +``` +user:password@/dbname?sql_mode=TRADITIONAL +``` + +TCP via IPv6: +``` +user:password@tcp([de:ad:be:ef::ca:fe]:80)/dbname?timeout=90s&collation=utf8mb4_unicode_ci +``` + +TCP on a remote host, e.g. Amazon RDS: +``` +id:password@tcp(your-amazonaws-uri.com:3306)/dbname +``` + +Google Cloud SQL on App Engine (First Generation MySQL Server): +``` +user@cloudsql(project-id:instance-name)/dbname +``` + +Google Cloud SQL on App Engine (Second Generation MySQL Server): +``` +user@cloudsql(project-id:regionname:instance-name)/dbname +``` + +TCP using default port (3306) on localhost: +``` +user:password@tcp/dbname?charset=utf8mb4,utf8&sys_var=esc%40ped +``` + +Use the default protocol (tcp) and host (localhost:3306): +``` +user:password@/dbname +``` + +No Database preselected: +``` +user:password@/ +``` + + +### Connection pool and timeouts +The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively. + +## `ColumnType` Support +This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. + +## `context.Context` Support +Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts. +See [context support in the database/sql package](https://golang.org/doc/go1.8#database_sql) for more details. + + +### `LOAD DATA LOCAL INFILE` support +For this feature you need direct access to the package. Therefore you must change the import path (no `_`): +```go +import "github.com/go-sql-driver/mysql" +``` + +Files must be whitelisted by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the Whitelist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). + +To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. + +See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. + + +### `time.Time` support +The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program. + +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical equivalent in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. + +**Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). + +Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. + + +### Unicode support +Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. + +Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. + +Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. + +See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. + +## Testing / Development +To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. + +Go-MySQL-Driver is not feature-complete yet. Your help is very appreciated. +If you want to contribute, you can work on an [open issue](https://github.com/go-sql-driver/mysql/issues?state=open) or review a [pull request](https://github.com/go-sql-driver/mysql/pulls). + +See the [Contribution Guidelines](https://github.com/go-sql-driver/mysql/blob/master/CONTRIBUTING.md) for details. + +--------------------------------------- + +## License +Go-MySQL-Driver is licensed under the [Mozilla Public License Version 2.0](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) + +Mozilla summarizes the license scope as follows: +> MPL: The copyleft applies to any files containing MPLed code. + + +That means: + * You can **use** the **unchanged** source code both in private and commercially. + * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0). + * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged**. + +Please read the [MPL 2.0 FAQ](https://www.mozilla.org/en-US/MPL/2.0/FAQ/) if you have further questions regarding the license. + +You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE). + +![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow") + diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go index be41f2e..914e662 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/appengine.go @@ -11,9 +11,15 @@ package mysql import ( + "context" + "net" + "google.golang.org/appengine/cloudsql" ) func init() { - RegisterDial("cloudsql", cloudsql.Dial) + RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { + // XXX: the cloudsql driver still does not export a Context-aware dialer. + return cloudsql.Dial(instance) + }) } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go index 2f61ecd..fec7040 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth.go @@ -234,64 +234,64 @@ func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) erro if err != nil { return err } - return mc.writeAuthSwitchPacket(enc, false) + return mc.writeAuthSwitchPacket(enc) } -func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, bool, error) { +func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { switch plugin { case "caching_sha2_password": authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) - return authResp, false, nil + return authResp, nil case "mysql_old_password": if !mc.cfg.AllowOldPasswords { - return nil, false, ErrOldPassword + return nil, ErrOldPassword } // Note: there are edge cases where this should work but doesn't; // this is currently "wontfix": // https://github.com/go-sql-driver/mysql/issues/184 - authResp := scrambleOldPassword(authData[:8], mc.cfg.Passwd) - return authResp, true, nil + authResp := append(scrambleOldPassword(authData[:8], mc.cfg.Passwd), 0) + return authResp, nil case "mysql_clear_password": if !mc.cfg.AllowCleartextPasswords { - return nil, false, ErrCleartextPassword + return nil, ErrCleartextPassword } // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil case "mysql_native_password": if !mc.cfg.AllowNativePasswords { - return nil, false, ErrNativePassword + return nil, ErrNativePassword } // https://dev.mysql.com/doc/internals/en/secure-password-authentication.html // Native password authentication only need and will need 20-byte challenge. authResp := scramblePassword(authData[:20], mc.cfg.Passwd) - return authResp, false, nil + return authResp, nil case "sha256_password": if len(mc.cfg.Passwd) == 0 { - return nil, true, nil + return []byte{0}, nil } if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil } pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - return []byte{1}, false, nil + return []byte{1}, nil } // encrypted password enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) - return enc, false, err + return enc, err default: errLog.Print("unknown auth plugin:", plugin) - return nil, false, ErrUnknownPlugin + return nil, ErrUnknownPlugin } } @@ -315,11 +315,11 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { plugin = newPlugin - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { return err } - if err = mc.writeAuthSwitchPacket(authResp, addNUL); err != nil { + if err = mc.writeAuthSwitchPacket(authResp); err != nil { return err } @@ -352,7 +352,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { case cachingSha2PasswordPerformFullAuthentication: if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - err = mc.writeAuthSwitchPacket([]byte(mc.cfg.Passwd), true) + err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0)) if err != nil { return err } @@ -360,13 +360,15 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - data := mc.buf.takeSmallBuffer(4 + 1) + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + return err + } data[4] = cachingSha2PasswordRequestPublicKey mc.writePacket(data) // parse public key - data, err := mc.readPacket() - if err != nil { + if data, err = mc.readPacket(); err != nil { return err } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go new file mode 100644 index 0000000..1920ef3 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/auth_test.go @@ -0,0 +1,1330 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "testing" +) + +var testPubKey = []byte("-----BEGIN PUBLIC KEY-----\n" + + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAol0Z8G8U+25Btxk/g/fm\n" + + "UAW/wEKjQCTjkibDE4B+qkuWeiumg6miIRhtilU6m9BFmLQSy1ltYQuu4k17A4tQ\n" + + "rIPpOQYZges/qsDFkZh3wyK5jL5WEFVdOasf6wsfszExnPmcZS4axxoYJfiuilrN\n" + + "hnwinBAqfi3S0sw5MpSI4Zl1AbOrHG4zDI62Gti2PKiMGyYDZTS9xPrBLbN95Kby\n" + + "FFclQLEzA9RJcS1nHFsWtRgHjGPhhjCQxEm9NQ1nePFhCfBfApyfH1VM2VCOQum6\n" + + "Ci9bMuHWjTjckC84mzF99kOxOWVU7mwS6gnJqBzpuz8t3zq8/iQ2y7QrmZV+jTJP\n" + + "WQIDAQAB\n" + + "-----END PUBLIC KEY-----\n") + +var testPubKeyRSA *rsa.PublicKey + +func init() { + block, _ := pem.Decode(testPubKey) + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + panic(err) + } + testPubKeyRSA = pub.(*rsa.PublicKey) +} + +func TestScrambleOldPass(t *testing.T) { + scramble := []byte{9, 8, 7, 6, 5, 4, 3, 2} + vectors := []struct { + pass string + out string + }{ + {" pass", "47575c5a435b4251"}, + {"pass ", "47575c5a435b4251"}, + {"123\t456", "575c47505b5b5559"}, + {"C0mpl!ca ted#PASS123", "5d5d554849584a45"}, + } + for _, tuple := range vectors { + ours := scrambleOldPassword(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed old password %q", tuple.pass) + } + } +} + +func TestScrambleSHA256Pass(t *testing.T) { + scramble := []byte{10, 47, 74, 111, 75, 73, 34, 48, 88, 76, 114, 74, 37, 13, 3, 80, 82, 2, 23, 21} + vectors := []struct { + pass string + out string + }{ + {"secret", "f490e76f66d9d86665ce54d98c78d0acfe2fb0b08b423da807144873d30b312c"}, + {"secret2", "abc3934a012cf342e876071c8ee202de51785b430258a7a0138bc79c4d800bc6"}, + } + for _, tuple := range vectors { + ours := scrambleSHA256Password(scramble, tuple.pass) + if tuple.out != fmt.Sprintf("%x", ours) { + t.Errorf("Failed SHA256 password %q", tuple.pass) + } + } +} + +func TestAuthFastCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{102, 32, 5, 35, 143, 161, 140, 241, 171, 232, 56, + 139, 43, 14, 107, 196, 249, 170, 147, 60, 220, 204, 120, 178, 214, 15, + 184, 150, 26, 61, 57, 235} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 3, // Fast Auth Success + 7, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{90, 105, 74, 126, 30, 48, 37, 56, 3, 23, 115, 127, 69, + 22, 41, 84, 32, 123, 43, 118} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // pub key response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{1, 0, 0, 3, 2, 0, 1, 0, 5}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "caching_sha2_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{171, 201, 138, 146, 89, 159, 11, 170, 0, 67, 165, + 49, 175, 94, 218, 68, 177, 109, 110, 86, 34, 33, 44, 190, 67, 240, 70, + 110, 40, 139, 124, 41} + if writtenAuthRespLen != 32 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 2, 0, 0, 2, 1, 4, // Perform Full Authentication + } + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastCleartextPasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthFastCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + mc.cfg.AllowCleartextPasswords = true + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_clear_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordNotAllowed(t *testing.T) { + _, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.AllowNativePasswords = false + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + _, err := mc.auth(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthFastNativePassword(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{53, 177, 140, 159, 251, 189, 127, 53, 109, 252, + 172, 50, 211, 192, 240, 164, 26, 48, 207, 45} + if writtenAuthRespLen != 20 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{70, 114, 92, 94, 1, 38, 11, 116, 63, 114, 23, 101, 126, + 103, 26, 95, 81, 17, 24, 21} + plugin := "mysql_native_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + if writtenAuthRespLen != 0 { + t.Fatalf("unexpected written auth response (%d bytes): %v", + writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{0} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{1} + if writtenAuthRespLen != 1 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (pub key response) + conn.data = append([]byte{byte(1 + len(testPubKey)), 1, 0, 2, 1}, testPubKey...) + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.HasPrefix(conn.written, []byte{0, 1, 0, 3}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthFastSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} + +func TestAuthFastSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "secret" + + // hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, + 62, 94, 83, 80, 52, 85} + plugin := "sha256_password" + + // send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + + // unset TLS config to prevent the actual establishment of a TLS wrapper + mc.cfg.tls = nil + + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{115, 101, 99, 114, 101, 116, 0} + if writtenAuthRespLen != 7 || !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("unexpected written auth response (%d bytes): %v", writtenAuthRespLen, writtenAuthResp) + } + conn.written = nil + + // auth response (OK) + conn.data = []byte{7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0} + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + if !bytes.Equal(conn.written, []byte{}) { + t.Errorf("unexpected written data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordCached(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 6, 1}, testPubKey...), + + // OK + {7, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 4 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Pub Key Request + 1, 0, 0, 5, 2, + + // 3. Packet: Encrypted Password + 0, 1, 0, 7, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + conn.queuedReplies = [][]byte{ + // Perform Full Authentication + {2, 0, 0, 4, 1, 4}, + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCachingSHA256PasswordFullSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, + 115, 104, 97, 50, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 101, + 11, 26, 18, 94, 97, 22, 72, 2, 46, 70, 106, 29, 55, 45, 94, 76, 90, 84, + 50, 0} + + // auth response + conn.queuedReplies = [][]byte{ + {2, 0, 0, 4, 1, 4}, // Perform Full Authentication + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, // OK + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{ + // 1. Packet: Hash + 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, + 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, + 153, 9, 130, + + // 2. Packet: Cleartext password + 7, 0, 0, 5, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + conn.maxReads = 1 + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrCleartextPassword { + t.Errorf("expected ErrCleartextPassword, got %v", err) + } +} + +func TestAuthSwitchCleartextPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchCleartextPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowCleartextPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{22, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 99, 108, + 101, 97, 114, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = false + + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + conn.maxReads = 1 + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrNativePassword { + t.Errorf("expected ErrNativePassword, got %v", err) + } +} + +func TestAuthSwitchNativePassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{20, 0, 0, 3, 202, 41, 195, 164, 34, 226, 49, 103, + 21, 211, 167, 199, 227, 116, 8, 48, 57, 71, 149, 146} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchNativePasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowNativePasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{44, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 110, 97, + 116, 105, 118, 101, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 96, + 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, 48, 31, 89, 39, 55, + 31, 0} + + // auth response + conn.queuedReplies = [][]byte{{7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{96, 71, 63, 8, 1, 58, 75, 12, 69, 95, 66, 60, 117, 31, + 48, 31, 89, 39, 55, 31} + plugin := "caching_sha2_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{0, 0, 0, 3} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchOldPasswordNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +// Same to TestAuthSwitchOldPasswordNotAllowed, but use OldAuthSwitch request. +func TestOldAuthSwitchNotAllowed(t *testing.T) { + conn, mc := newRWMockConn(2) + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + conn.maxReads = 1 + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + err := mc.handleAuthResult(authData, plugin) + if err != ErrOldPassword { + t.Errorf("expected ErrOldPassword, got %v", err) + } +} + +func TestAuthSwitchOldPassword(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPassword, but use OldAuthSwitch request. +func TestOldAuthSwitch(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "secret" + + // OldAuthSwitch request + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{9, 0, 0, 3, 86, 83, 83, 79, 74, 78, 65, 66, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} +func TestAuthSwitchOldPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{41, 0, 0, 2, 254, 109, 121, 115, 113, 108, 95, 111, 108, + 100, 95, 112, 97, 115, 115, 119, 111, 114, 100, 0, 95, 84, 103, 43, 61, + 49, 123, 61, 91, 50, 40, 113, 35, 84, 96, 101, 92, 123, 121, 107, 0} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +// Same to TestAuthSwitchOldPasswordEmpty, but use OldAuthSwitch request. +func TestOldAuthSwitchPasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.AllowOldPasswords = true + mc.cfg.Passwd = "" + + // OldAuthSwitch request. + conn.data = []byte{1, 0, 0, 2, 0xfe} + + // auth response + conn.queuedReplies = [][]byte{{8, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0}} + conn.maxReads = 2 + + authData := []byte{95, 84, 103, 43, 61, 49, 123, 61, 91, 50, 40, 113, 35, + 84, 96, 101, 92, 123, 121, 107} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReply := []byte{1, 0, 0, 3, 0} + if !bytes.Equal(conn.written, expectedReply) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordEmpty(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Empty Password + 1, 0, 0, 3, 0, + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSA(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // Pub Key Response + append([]byte{byte(1 + len(testPubKey)), 1, 0, 4, 1}, testPubKey...), + + // OK + {7, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 3 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Pub Key Request + 1, 0, 0, 3, 1, + + // 2. Packet: Encrypted Password + 0, 1, 0, 5, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordRSAWithKey(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + mc.cfg.pubKey = testPubKeyRSA + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Encrypted Password + 0, 1, 0, 3, // [changing bytes] + } + if !bytes.HasPrefix(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} + +func TestAuthSwitchSHA256PasswordSecure(t *testing.T) { + conn, mc := newRWMockConn(2) + mc.cfg.Passwd = "secret" + + // Hack to make the caching_sha2_password plugin believe that the connection + // is secure + mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + + // auth switch request + conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, + 115, 115, 119, 111, 114, 100, 0, 78, 82, 62, 40, 100, 1, 59, 31, 44, 69, + 33, 112, 8, 81, 51, 96, 65, 82, 16, 114, 0} + + conn.queuedReplies = [][]byte{ + // OK + {7, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0}, + } + conn.maxReads = 2 + + authData := []byte{123, 87, 15, 84, 20, 58, 37, 121, 91, 117, 51, 24, 19, + 47, 43, 9, 41, 112, 67, 110} + plugin := "mysql_native_password" + + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } + + expectedReplyPrefix := []byte{ + // 1. Packet: Cleartext Password + 7, 0, 0, 3, 115, 101, 99, 114, 101, 116, 0, + } + if !bytes.Equal(conn.written, expectedReplyPrefix) { + t.Errorf("got unexpected data: %v", conn.written) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go new file mode 100644 index 0000000..3e25a3b --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/benchmark_test.go @@ -0,0 +1,373 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "fmt" + "math" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +type TB testing.B + +func (tb *TB) check(err error) { + if err != nil { + tb.Fatal(err) + } +} + +func (tb *TB) checkDB(db *sql.DB, err error) *sql.DB { + tb.check(err) + return db +} + +func (tb *TB) checkRows(rows *sql.Rows, err error) *sql.Rows { + tb.check(err) + return rows +} + +func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt { + tb.check(err) + return stmt +} + +func initDB(b *testing.B, queries ...string) *sql.DB { + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + for _, query := range queries { + if _, err := db.Exec(query); err != nil { + b.Fatalf("error on %q: %v", query, err) + } + } + return db +} + +const concurrencyLevel = 10 + +func BenchmarkQuery(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + var got string + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Errorf("query = %q; want one", got) + wg.Done() + return + } + } + }() + } +} + +func BenchmarkExec(b *testing.B) { + tb := (*TB)(b) + b.StopTimer() + b.ReportAllocs() + db := tb.checkDB(sql.Open("mysql", dsn)) + db.SetMaxIdleConns(concurrencyLevel) + defer db.Close() + + stmt := tb.checkStmt(db.Prepare("DO 1")) + defer stmt.Close() + + remain := int64(b.N) + var wg sync.WaitGroup + wg.Add(concurrencyLevel) + defer wg.Wait() + b.StartTimer() + + for i := 0; i < concurrencyLevel; i++ { + go func() { + for { + if atomic.AddInt64(&remain, -1) < 0 { + wg.Done() + return + } + + if _, err := stmt.Exec(); err != nil { + b.Fatal(err.Error()) + } + } + }() + } +} + +// data, but no db writes +var roundtripSample []byte + +func initRoundtripBenchmarks() ([]byte, int, int) { + if roundtripSample == nil { + roundtripSample = []byte(strings.Repeat("0123456789abcdef", 1024*1024)) + } + return roundtripSample, 16, len(roundtripSample) +} + +func BenchmarkRoundtripTxt(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + sampleString := string(sample) + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + b.StartTimer() + var result string + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sampleString[0:length] + rows := tb.checkRows(db.Query(`SELECT "` + test + `"`)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if result != test { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkRoundtripBin(b *testing.B) { + b.StopTimer() + sample, min, max := initRoundtripBenchmarks() + b.ReportAllocs() + tb := (*TB)(b) + db := tb.checkDB(sql.Open("mysql", dsn)) + defer db.Close() + stmt := tb.checkStmt(db.Prepare("SELECT ?")) + defer stmt.Close() + b.StartTimer() + var result sql.RawBytes + for i := 0; i < b.N; i++ { + length := min + i + if length > max { + length = max + } + test := sample[0:length] + rows := tb.checkRows(stmt.Query(test)) + if !rows.Next() { + rows.Close() + b.Fatalf("crashed") + } + err := rows.Scan(&result) + if err != nil { + rows.Close() + b.Fatalf("crashed") + } + if !bytes.Equal(result, test) { + rows.Close() + b.Errorf("mismatch") + } + rows.Close() + } +} + +func BenchmarkInterpolation(b *testing.B) { + mc := &mysqlConn{ + cfg: &Config{ + InterpolateParams: true, + Loc: time.UTC, + }, + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + buf: newBuffer(nil), + } + + args := []driver.Value{ + int64(42424242), + float64(math.Pi), + false, + time.Unix(1423411542, 807015000), + []byte("bytes containing special chars ' \" \a \x00"), + "string containing special chars ' \" \a \x00", + } + q := "SELECT ?, ?, ?, ?, ?, ?" + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := mc.interpolateParams(q, args) + if err != nil { + b.Fatal(err) + } + } +} + +func benchmarkQueryContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "SELECT val FROM foo WHERE id=?")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + var got string + for pb.Next() { + tb.check(stmt.QueryRow(1).Scan(&got)) + if got != "one" { + b.Fatalf("query = %q; want one", got) + } + } + }) +} + +func BenchmarkQueryContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +func benchmarkExecContext(b *testing.B, db *sql.DB, p int) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db.SetMaxIdleConns(p * runtime.GOMAXPROCS(0)) + + tb := (*TB)(b) + stmt := tb.checkStmt(db.PrepareContext(ctx, "DO 1")) + defer stmt.Close() + + b.SetParallelism(p) + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := stmt.ExecContext(ctx); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkExecContext(b *testing.B) { + db := initDB(b, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", + `INSERT INTO foo VALUES (1, "one")`, + `INSERT INTO foo VALUES (2, "two")`, + ) + defer db.Close() + for _, p := range []int{1, 2, 3, 4} { + b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { + benchmarkQueryContext(b, db, p) + }) + } +} + +// BenchmarkQueryRawBytes benchmarks fetching 100 blobs using sql.RawBytes. +// "size=" means size of each blobs. +func BenchmarkQueryRawBytes(b *testing.B) { + var sizes []int = []int{100, 1000, 2000, 4000, 8000, 12000, 16000, 32000, 64000, 256000} + db := initDB(b, + "DROP TABLE IF EXISTS bench_rawbytes", + "CREATE TABLE bench_rawbytes (id INT PRIMARY KEY, val LONGBLOB)", + ) + defer db.Close() + + blob := make([]byte, sizes[len(sizes)-1]) + for i := range blob { + blob[i] = 42 + } + for i := 0; i < 100; i++ { + _, err := db.Exec("INSERT INTO bench_rawbytes VALUES (?, ?)", i, blob) + if err != nil { + b.Fatal(err) + } + } + + for _, s := range sizes { + b.Run(fmt.Sprintf("size=%v", s), func(b *testing.B) { + db.SetMaxIdleConns(0) + db.SetMaxIdleConns(1) + b.ReportAllocs() + b.ResetTimer() + + for j := 0; j < b.N; j++ { + rows, err := db.Query("SELECT LEFT(val, ?) as v FROM bench_rawbytes", s) + if err != nil { + b.Fatal(err) + } + nrows := 0 + for rows.Next() { + var buf sql.RawBytes + err := rows.Scan(&buf) + if err != nil { + b.Fatal(err) + } + if len(buf) != s { + b.Fatalf("size mismatch: expected %v, got %v", s, len(buf)) + } + nrows++ + } + rows.Close() + if nrows != 100 { + b.Fatalf("numbers of rows mismatch: expected %v, got %v", 100, nrows) + } + } + }) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go index eb4748b..0774c5c 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -15,47 +15,69 @@ import ( ) const defaultBufSize = 4096 +const maxCachedBufSize = 256 * 1024 // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. +// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { - buf []byte + buf []byte // buf is a byte buffer who's length and capacity are equal. nc net.Conn idx int length int timeout time.Duration + dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer + flipcnt uint // flipccnt is the current buffer counter for double-buffering } +// newBuffer allocates and returns a new buffer. func newBuffer(nc net.Conn) buffer { - var b [defaultBufSize]byte + fg := make([]byte, defaultBufSize) return buffer{ - buf: b[:], - nc: nc, + buf: fg, + nc: nc, + dbuf: [2][]byte{fg, nil}, } } +// flip replaces the active buffer with the background buffer +// this is a delayed flip that simply increases the buffer counter; +// the actual flip will be performed the next time we call `buffer.fill` +func (b *buffer) flip() { + b.flipcnt += 1 +} + // fill reads into the buffer until at least _need_ bytes are in it func (b *buffer) fill(need int) error { n := b.length + // fill data into its double-buffering target: if we've called + // flip on this buffer, we'll be copying to the background buffer, + // and then filling it with network data; otherwise we'll just move + // the contents of the current buffer to the front before filling it + dest := b.dbuf[b.flipcnt&1] - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) - } - - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { + // grow buffer if necessary to fit the whole packet. + if need > len(dest) { // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf + dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) + + // if the allocated buffer is not too large, move it to backing storage + // to prevent extra allocations on applications that perform large reads + if len(dest) <= maxCachedBufSize { + b.dbuf[b.flipcnt&1] = dest + } } + // if we're filling the fg buffer, move the existing data to the start of it. + // if we're filling the bg buffer, copy over the data + if n > 0 { + copy(dest[:n], b.buf[b.idx:]) + } + + b.buf = dest b.idx = 0 for { @@ -105,43 +127,56 @@ func (b *buffer) readNext(need int) ([]byte, error) { return b.buf[offset:b.idx], nil } -// returns a buffer with the requested size. +// takeBuffer returns a buffer with the requested size. // If possible, a slice from the existing buffer is returned. // Otherwise a bigger buffer is made. // Only one buffer (total) can be used at a time. -func (b *buffer) takeBuffer(length int) []byte { +func (b *buffer) takeBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } // test (cheap) general case first - if length <= defaultBufSize || length <= cap(b.buf) { - return b.buf[:length] + if length <= cap(b.buf) { + return b.buf[:length], nil } if length < maxPacketSize { b.buf = make([]byte, length) - return b.buf + return b.buf, nil } - return make([]byte, length) + + // buffer is larger than we want to store. + return make([]byte, length), nil } -// shortcut which can be used if the requested buffer is guaranteed to be -// smaller than defaultBufSize +// takeSmallBuffer is shortcut which can be used if length is +// known to be smaller than defaultBufSize. // Only one buffer (total) can be used at a time. -func (b *buffer) takeSmallBuffer(length int) []byte { +func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } - return b.buf[:length] + return b.buf[:length], nil } // takeCompleteBuffer returns the complete existing buffer. // This can be used if the necessary buffer size is unknown. +// cap and len of the returned buffer will be equal. // Only one buffer (total) can be used at a time. -func (b *buffer) takeCompleteBuffer() []byte { +func (b *buffer) takeCompleteBuffer() ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } - return b.buf + return b.buf, nil +} + +// store stores buf, an updated buffer, if its suitable to do so. +func (b *buffer) store(buf []byte) error { + if b.length > 0 { + return ErrBusyBuffer + } else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) { + b.buf = buf[:cap(buf)] + } + return nil } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go index 136c9e4..8d2b556 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/collations.go @@ -8,183 +8,190 @@ package mysql -const defaultCollation = "utf8_general_ci" +const defaultCollation = "utf8mb4_general_ci" const binaryCollation = "binary" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. +// +// ucs2, utf16, and utf32 can't be used for connection charset. +// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset +// They are commented out to reduce this map. var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + //"ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + //"utf16_general_ci": 54, + //"utf16_bin": 55, + //"utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + //"utf32_general_ci": 60, + //"utf32_bin": 61, + //"utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "utf8_tolower_ci": 76, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + //"ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + //"utf16_unicode_ci": 101, + //"utf16_icelandic_ci": 102, + //"utf16_latvian_ci": 103, + //"utf16_romanian_ci": 104, + //"utf16_slovenian_ci": 105, + //"utf16_polish_ci": 106, + //"utf16_estonian_ci": 107, + //"utf16_spanish_ci": 108, + //"utf16_swedish_ci": 109, + //"utf16_turkish_ci": 110, + //"utf16_czech_ci": 111, + //"utf16_danish_ci": 112, + //"utf16_lithuanian_ci": 113, + //"utf16_slovak_ci": 114, + //"utf16_spanish2_ci": 115, + //"utf16_roman_ci": 116, + //"utf16_persian_ci": 117, + //"utf16_esperanto_ci": 118, + //"utf16_hungarian_ci": 119, + //"utf16_sinhala_ci": 120, + //"utf16_german2_ci": 121, + //"utf16_croatian_ci": 122, + //"utf16_unicode_520_ci": 123, + //"utf16_vietnamese_ci": 124, + //"ucs2_unicode_ci": 128, + //"ucs2_icelandic_ci": 129, + //"ucs2_latvian_ci": 130, + //"ucs2_romanian_ci": 131, + //"ucs2_slovenian_ci": 132, + //"ucs2_polish_ci": 133, + //"ucs2_estonian_ci": 134, + //"ucs2_spanish_ci": 135, + //"ucs2_swedish_ci": 136, + //"ucs2_turkish_ci": 137, + //"ucs2_czech_ci": 138, + //"ucs2_danish_ci": 139, + //"ucs2_lithuanian_ci": 140, + //"ucs2_slovak_ci": 141, + //"ucs2_spanish2_ci": 142, + //"ucs2_roman_ci": 143, + //"ucs2_persian_ci": 144, + //"ucs2_esperanto_ci": 145, + //"ucs2_hungarian_ci": 146, + //"ucs2_sinhala_ci": 147, + //"ucs2_german2_ci": 148, + //"ucs2_croatian_ci": 149, + //"ucs2_unicode_520_ci": 150, + //"ucs2_vietnamese_ci": 151, + //"ucs2_general_mysql500_ci": 159, + //"utf32_unicode_ci": 160, + //"utf32_icelandic_ci": 161, + //"utf32_latvian_ci": 162, + //"utf32_romanian_ci": 163, + //"utf32_slovenian_ci": 164, + //"utf32_polish_ci": 165, + //"utf32_estonian_ci": 166, + //"utf32_spanish_ci": 167, + //"utf32_swedish_ci": 168, + //"utf32_turkish_ci": 169, + //"utf32_czech_ci": 170, + //"utf32_danish_ci": 171, + //"utf32_lithuanian_ci": 172, + //"utf32_slovak_ci": 173, + //"utf32_spanish2_ci": 174, + //"utf32_roman_ci": 175, + //"utf32_persian_ci": 176, + //"utf32_esperanto_ci": 177, + //"utf32_hungarian_ci": 178, + //"utf32_sinhala_ci": 179, + //"utf32_german2_ci": 180, + //"utf32_croatian_ci": 181, + //"utf32_unicode_520_ci": 182, + //"utf32_vietnamese_ci": 183, "utf8_unicode_ci": 192, "utf8_icelandic_ci": 193, "utf8_latvian_ci": 194, @@ -234,18 +241,25 @@ var collations = map[string]byte{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, + "gb18030_unicode_520_ci": 250, + "utf8mb4_0900_ai_ci": 255, } // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. var unsafeCollations = map[string]bool{ - "big5_chinese_ci": true, - "sjis_japanese_ci": true, - "gbk_chinese_ci": true, - "big5_bin": true, - "gb2312_bin": true, - "gbk_bin": true, - "sjis_bin": true, - "cp932_japanese_ci": true, - "cp932_bin": true, + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, + "gb18030_chinese_ci": true, + "gb18030_bin": true, + "gb18030_unicode_520_ci": true, } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go new file mode 100644 index 0000000..cc47aa5 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -0,0 +1,53 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows,!appengine + +package mysql + +import ( + "errors" + "io" + "net" + "syscall" +) + +var errUnexpectedRead = errors.New("unexpected read from socket") + +func connCheck(c net.Conn) error { + var ( + n int + err error + buff [1]byte + ) + + sconn, ok := c.(syscall.Conn) + if !ok { + return nil + } + rc, err := sconn.SyscallConn() + if err != nil { + return err + } + rerr := rc.Read(func(fd uintptr) bool { + n, err = syscall.Read(int(fd), buff[:]) + return true + }) + switch { + case rerr != nil: + return rerr + case n == 0 && err == nil: + return io.EOF + case n > 0: + return errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + return nil + default: + return err + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go new file mode 100644 index 0000000..fd01f64 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -0,0 +1,17 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows appengine + +package mysql + +import "net" + +func connCheck(c net.Conn) error { + return nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go new file mode 100644 index 0000000..b7234b0 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/conncheck_test.go @@ -0,0 +1,38 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10,!windows + +package mysql + +import ( + "testing" + "time" +) + +func TestStaleConnectionChecks(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("SET @@SESSION.wait_timeout = 2") + + if err := dbt.db.Ping(); err != nil { + dbt.Fatal(err) + } + + // wait for MySQL to close our connection + time.Sleep(3 * time.Second) + + tx, err := dbt.db.Begin() + if err != nil { + dbt.Fatal(err) + } + + if err := tx.Rollback(); err != nil { + dbt.Fatal(err) + } + }) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go index 911be20..565a548 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection.go @@ -19,19 +19,10 @@ import ( "time" ) -// a copy of context.Context for Go 1.7 and earlier -type mysqlContext interface { - Done() <-chan struct{} - Err() error - - // defined in context.Context, but not used in this driver: - // Deadline() (deadline time.Time, ok bool) - // Value(key interface{}) interface{} -} - type mysqlConn struct { buf buffer netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. affectedRows uint64 insertId uint64 cfg *Config @@ -42,10 +33,11 @@ type mysqlConn struct { status statusFlag sequence uint8 parseTime bool + reset bool // set when the Go SQL package calls ResetSession // for context support (Go 1.8+) watching bool - watcher chan<- mysqlContext + watcher chan<- context.Context closech chan struct{} finished chan<- struct{} canceled atomicError // set non-nil if conn is canceled @@ -192,10 +184,10 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - buf := mc.buf.takeCompleteBuffer() - if buf == nil { + buf, err := mc.buf.takeCompleteBuffer() + if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return "", ErrInvalidConn } buf = buf[:0] @@ -221,6 +213,9 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin switch v := arg.(type) { case int64: buf = strconv.AppendInt(buf, v, 10) + case uint64: + // Handle uint64 explicitly because our custom ConvertValue emits unsigned values + buf = strconv.AppendUint(buf, v, 10) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: @@ -475,7 +470,7 @@ func (mc *mysqlConn) Ping(ctx context.Context) (err error) { defer mc.finish() if err = mc.writeCommandPacket(comPing); err != nil { - return + return mc.markBadConn(err) } return mc.readResultOK() @@ -595,33 +590,32 @@ func (mc *mysqlConn) watchCancel(ctx context.Context) error { mc.cleanup() return nil } + // When ctx is already cancelled, don't watch it. + if err := ctx.Err(); err != nil { + return err + } + // When ctx is not cancellable, don't watch it. if ctx.Done() == nil { return nil } - - mc.watching = true - select { - default: - case <-ctx.Done(): - return ctx.Err() - } + // When watcher is not alive, can't watch it. if mc.watcher == nil { return nil } + mc.watching = true mc.watcher <- ctx - return nil } func (mc *mysqlConn) startWatcher() { - watcher := make(chan mysqlContext, 1) + watcher := make(chan context.Context, 1) mc.watcher = watcher finished := make(chan struct{}) mc.finished = finished go func() { for { - var ctx mysqlContext + var ctx context.Context select { case ctx = <-watcher: case <-mc.closech: @@ -650,5 +644,6 @@ func (mc *mysqlConn) ResetSession(ctx context.Context) error { if mc.closed.IsSet() { return driver.ErrBadConn } + mc.reset = true return nil } diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go new file mode 100644 index 0000000..19c17ff --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connection_test.go @@ -0,0 +1,175 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "errors" + "net" + "testing" +) + +func TestInterpolateParams(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42), "gopher"}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT 42+'gopher'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + +func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?+?", []driver.Value{int64(42)}) + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +// We don't support placeholder in string literal for now. +// https://github.com/go-sql-driver/mysql/pull/490 +func TestInterpolateParamsPlaceholderInString(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT 'abc?xyz',?", []driver.Value{int64(42)}) + // When InterpolateParams support string literal, this should return `"SELECT 'abc?xyz', 42` + if err != driver.ErrSkip { + t.Errorf("Expected err=driver.ErrSkip, got err=%#v, q=%#v", err, q) + } +} + +func TestInterpolateParamsUint64(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + q, err := mc.interpolateParams("SELECT ?", []driver.Value{uint64(42)}) + if err != nil { + t.Errorf("Expected err=nil, got err=%#v, q=%#v", err, q) + } + if q != "SELECT 42" { + t.Errorf("Expected uint64 interpolation to work, got q=%#v", q) + } +} + +func TestCheckNamedValue(t *testing.T) { + value := driver.NamedValue{Value: ^uint64(0)} + x := &mysqlConn{} + err := x.CheckNamedValue(&value) + + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if value.Value != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", value.Value, value.Value) + } +} + +// TestCleanCancel tests passed context is cancelled at start. +// No packet should be sent. Connection should keep current status. +func TestCleanCancel(t *testing.T) { + mc := &mysqlConn{ + closech: make(chan struct{}), + } + mc.startWatcher() + defer mc.cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + for i := 0; i < 3; i++ { // Repeat same behavior + err := mc.Ping(ctx) + if err != context.Canceled { + t.Errorf("expected context.Canceled, got %#v", err) + } + + if mc.closed.IsSet() { + t.Error("expected mc is not closed, closed actually") + } + + if mc.watching { + t.Error("expected watching is false, but true") + } + } +} + +func TestPingMarkBadConnection(t *testing.T) { + nc := badConnection{err: errors.New("boom")} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + } + + err := ms.Ping(context.Background()) + + if err != driver.ErrBadConn { + t.Errorf("expected driver.ErrBadConn, got %#v", err) + } +} + +func TestPingErrInvalidConn(t *testing.T) { + nc := badConnection{err: errors.New("failed to write"), n: 10} + ms := &mysqlConn{ + netConn: nc, + buf: newBuffer(nc), + maxAllowedPacket: defaultMaxAllowedPacket, + closech: make(chan struct{}), + } + + err := ms.Ping(context.Background()) + + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %#v", err) + } +} + +type badConnection struct { + n int + err error + net.Conn +} + +func (bc badConnection) Write(b []byte) (n int, err error) { + return bc.n, bc.err +} + +func (bc badConnection) Close() error { + return nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go new file mode 100644 index 0000000..5aaaba4 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/connector.go @@ -0,0 +1,143 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "net" +) + +type connector struct { + cfg *Config // immutable private copy. +} + +// Connect implements driver.Connector interface. +// Connect returns a connection to the database. +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + cfg: c.cfg, + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + mc.netConn, err = dial(ctx, mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + } + + if err != nil { + if nerr, ok := err.(net.Error); ok && nerr.Temporary() { + errLog.Print("net.Error from Dial()': ", nerr.Error()) + return nil, driver.ErrBadConn + } + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +// Driver implements driver.Connector interface. +// Driver returns &MySQLDriver{}. +func (c *connector) Driver() driver.Driver { + return &MySQLDriver{} +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go index ba12978..1f9decf 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver.go @@ -17,6 +17,7 @@ package mysql import ( + "context" "database/sql" "database/sql/driver" "net" @@ -29,135 +30,54 @@ type MySQLDriver struct{} // DialFunc is a function which can be used to establish the network connection. // Custom dial functions must be registered with RegisterDial +// +// Deprecated: users should register a DialContextFunc instead type DialFunc func(addr string) (net.Conn, error) +// DialContextFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDialContext +type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error) + var ( dialsLock sync.RWMutex - dials map[string]DialFunc + dials map[string]DialContextFunc ) -// RegisterDial registers a custom dial function. It can then be used by the +// RegisterDialContext registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { +// The current context for the connection and its address is passed to the dial function. +func RegisterDialContext(net string, dial DialContextFunc) { dialsLock.Lock() defer dialsLock.Unlock() if dials == nil { - dials = make(map[string]DialFunc) + dials = make(map[string]DialContextFunc) } dials[net] = dial } +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +// +// Deprecated: users should call RegisterDialContext instead +func RegisterDial(network string, dial DialFunc) { + RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) { + return dial(addr) + }) +} + // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how -// the DSN string is formated +// the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxAllowedPacket: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - closech: make(chan struct{}), - } - mc.cfg, err = ParseDSN(dsn) + cfg, err := ParseDSN(dsn) if err != nil { return nil, err } - mc.parseTime = mc.cfg.ParseTime - - // Connect to Server - dialsLock.RLock() - dial, ok := dials[mc.cfg.Net] - dialsLock.RUnlock() - if ok { - mc.netConn, err = dial(mc.cfg.Addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.Timeout} - mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) + c := &connector{ + cfg: cfg, } - if err != nil { - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - // Call startWatcher for context support (From Go 1.8) - mc.startWatcher() - - mc.buf = newBuffer(mc.netConn) - - // Set I/O timeouts - mc.buf.timeout = mc.cfg.ReadTimeout - mc.writeTimeout = mc.cfg.WriteTimeout - - // Reading Handshake Initialization Packet - authData, plugin, err := mc.readHandshakePacket() - if err != nil { - mc.cleanup() - return nil, err - } - if plugin == "" { - plugin = defaultAuthPlugin - } - - // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) - if err != nil { - // try the default auth plugin, if using the requested plugin failed - errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) - plugin = defaultAuthPlugin - authResp, addNUL, err = mc.auth(authData, plugin) - if err != nil { - mc.cleanup() - return nil, err - } - } - if err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin); err != nil { - mc.cleanup() - return nil, err - } - - // Handle response to auth packet, switch methods if possible - if err = mc.handleAuthResult(authData, plugin); err != nil { - // Authentication failed and MySQL has already closed the connection - // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). - // Do not send COM_QUIT, just cleanup and return the error. - mc.cleanup() - return nil, err - } - - if mc.cfg.MaxAllowedPacket > 0 { - mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket - } else { - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxAllowedPacket = stringToInt(maxap) - 1 - } - if mc.maxAllowedPacket < maxPacketSize { - mc.maxWriteSize = mc.maxAllowedPacket - } - - // Handle DSN Params - err = mc.handleParams() - if err != nil { - mc.Close() - return nil, err - } - - return mc, nil + return c.Connect(context.Background()) } func init() { diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go new file mode 100644 index 0000000..eb5a8fe --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110.go @@ -0,0 +1,37 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "database/sql/driver" +) + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go new file mode 100644 index 0000000..19a0e59 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go @@ -0,0 +1,137 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.10 + +package mysql + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "net" + "testing" + "time" +) + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go new file mode 100644 index 0000000..3dee1ba --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/driver_test.go @@ -0,0 +1,2996 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "context" + "crypto/tls" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "net" + "net/url" + "os" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Ensure that all the driver interfaces are implemented +var ( + _ driver.Rows = &binaryRows{} + _ driver.Rows = &textRows{} +) + +var ( + user string + pass string + prot string + addr string + dbname string + dsn string + netAddr string + available bool +) + +var ( + tDate = time.Date(2012, 6, 14, 0, 0, 0, 0, time.UTC) + sDate = "2012-06-14" + tDateTime = time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC) + sDateTime = "2011-11-20 21:27:37" + tDate0 = time.Time{} + sDate0 = "0000-00-00" + sDateTime0 = "0000-00-00 00:00:00" +) + +// See https://github.com/go-sql-driver/mysql/wiki/Testing +func init() { + // get environment variables + env := func(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue + } + user = env("MYSQL_TEST_USER", "root") + pass = env("MYSQL_TEST_PASS", "") + prot = env("MYSQL_TEST_PROT", "tcp") + addr = env("MYSQL_TEST_ADDR", "localhost:3306") + dbname = env("MYSQL_TEST_DBNAME", "gotest") + netAddr = fmt.Sprintf("%s(%s)", prot, addr) + dsn = fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, pass, netAddr, dbname) + c, err := net.Dial(prot, addr) + if err == nil { + available = true + c.Close() + } +} + +type DBTest struct { + *testing.T + db *sql.DB +} + +type netErrorMock struct { + temporary bool + timeout bool +} + +func (e netErrorMock) Temporary() bool { + return e.temporary +} + +func (e netErrorMock) Timeout() bool { + return e.timeout +} + +func (e netErrorMock) Error() string { + return fmt.Sprintf("mock net error. Temporary: %v, Timeout %v", e.temporary, e.timeout) +} + +func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn += "&multiStatements=true" + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + } +} + +func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + db.Exec("DROP TABLE IF EXISTS test") + + dsn2 := dsn + "&interpolateParams=true" + var db2 *sql.DB + if _, err := ParseDSN(dsn2); err != errInvalidDSNUnsafeCollation { + db2, err = sql.Open("mysql", dsn2) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db2.Close() + } + + dsn3 := dsn + "&multiStatements=true" + var db3 *sql.DB + if _, err := ParseDSN(dsn3); err != errInvalidDSNUnsafeCollation { + db3, err = sql.Open("mysql", dsn3) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db3.Close() + } + + dbt := &DBTest{t, db} + dbt2 := &DBTest{t, db2} + dbt3 := &DBTest{t, db3} + for _, test := range tests { + test(dbt) + dbt.db.Exec("DROP TABLE IF EXISTS test") + if db2 != nil { + test(dbt2) + dbt2.db.Exec("DROP TABLE IF EXISTS test") + } + if db3 != nil { + test(dbt3) + dbt3.db.Exec("DROP TABLE IF EXISTS test") + } + } +} + +func (dbt *DBTest) fail(method, query string, err error) { + if len(query) > 300 { + query = "[query too large to print]" + } + dbt.Fatalf("error on %s %s: %s", method, query, err.Error()) +} + +func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) { + res, err := dbt.db.Exec(query, args...) + if err != nil { + dbt.fail("exec", query, err) + } + return res +} + +func (dbt *DBTest) mustQuery(query string, args ...interface{}) (rows *sql.Rows) { + rows, err := dbt.db.Query(query, args...) + if err != nil { + dbt.fail("query", query, err) + } + return rows +} + +func maybeSkip(t *testing.T, err error, skipErrno uint16) { + mySQLErr, ok := err.(*MySQLError) + if !ok { + return + } + + if mySQLErr.Number == skipErrno { + t.Skipf("skipping test for error: %v", err) + } +} + +func TestEmptyQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // just a comment, no query + rows := dbt.mustQuery("--") + defer rows.Close() + // will hang before #255 + if rows.Next() { + dbt.Errorf("next on rows must be false") + } + }) +} + +func TestCRUD(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + + // Test for unexpected data + var out bool + rows := dbt.mustQuery("SELECT * FROM test") + if rows.Next() { + dbt.Error("unexpected data in empty table") + } + rows.Close() + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + id, err := res.LastInsertId() + if err != nil { + dbt.Fatalf("res.LastInsertId() returned error: %s", err.Error()) + } + if id != 0 { + dbt.Fatalf("expected InsertId 0, got %d", id) + } + + // Read + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if true != out { + dbt.Errorf("true != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Update + res = dbt.mustExec("UPDATE test SET value = ? WHERE value = ?", false, true) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check Update + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if false != out { + dbt.Errorf("false != %t", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + // Delete + res = dbt.mustExec("DELETE FROM test WHERE value = ?", false) + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Check for unexpected rows + res = dbt.mustExec("DELETE FROM test") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 0 { + dbt.Fatalf("expected 0 affected row, got %d", count) + } + }) +} + +func TestMultiQuery(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE `test` (`id` int(11) NOT NULL, `value` int(11) NOT NULL) ") + + // Create Data + res := dbt.mustExec("INSERT INTO test VALUES (1, 1)") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Update + res = dbt.mustExec("UPDATE test SET value = 3 WHERE id = 1; UPDATE test SET value = 4 WHERE id = 1; UPDATE test SET value = 5 WHERE id = 1;") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 1 { + dbt.Fatalf("expected 1 affected row, got %d", count) + } + + // Read + var out int + rows := dbt.mustQuery("SELECT value FROM test WHERE id=1;") + if rows.Next() { + rows.Scan(&out) + if 5 != out { + dbt.Errorf("5 != %d", out) + } + + if rows.Next() { + dbt.Error("unexpected data") + } + } else { + dbt.Error("no data") + } + rows.Close() + + }) +} + +func TestInt(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [5]string{"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT"} + in := int64(42) + var out int64 + var rows *sql.Rows + + // SIGNED + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // UNSIGNED ZEROFILL + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + " ZEROFILL)") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s ZEROFILL: %d != %d", v, in, out) + } + } else { + dbt.Errorf("%s ZEROFILL: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat32(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + in := float32(42.23) + var out float32 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %g != %g", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (42.23)") + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestFloat64Placeholder(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [2]string{"FLOAT", "DOUBLE"} + var expected float64 = 42.23 + var out float64 + var rows *sql.Rows + for _, v := range types { + dbt.mustExec("CREATE TABLE test (id int, value " + v + ")") + dbt.mustExec("INSERT INTO test VALUES (1, 42.23)") + rows = dbt.mustQuery("SELECT value FROM test WHERE id = ?", 1) + if rows.Next() { + rows.Scan(&out) + if expected != out { + dbt.Errorf("%s: %g != %g", v, expected, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + dbt.mustExec("DROP TABLE IF EXISTS test") + } + }) +} + +func TestString(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + types := [6]string{"CHAR(255)", "VARCHAR(255)", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"} + in := "κόσμε üöäßñóùéàâÿœ'îë Árvíztűrő いろはにほへとちりぬるを イロハニホヘト דג סקרן чащах น่าฟังเอย" + var out string + var rows *sql.Rows + + for _, v := range types { + dbt.mustExec("CREATE TABLE test (value " + v + ") CHARACTER SET utf8") + + dbt.mustExec("INSERT INTO test VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Errorf("%s: %s != %s", v, in, out) + } + } else { + dbt.Errorf("%s: no data", v) + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + } + + // BLOB + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + + id := 2 + in = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. " + + "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + + "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." + dbt.mustExec("INSERT INTO test VALUES (?, ?)", id, in) + + err := dbt.db.QueryRow("SELECT value FROM test WHERE id = ?", id).Scan(&out) + if err != nil { + dbt.Fatalf("Error on BLOB-Query: %s", err.Error()) + } else if out != in { + dbt.Errorf("BLOB: %s != %s", in, out) + } + }) +} + +func TestRawBytes(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + v1 := []byte("aaa") + v2 := []byte("bbb") + rows := dbt.mustQuery("SELECT ?, ?", v1, v2) + defer rows.Close() + if rows.Next() { + var o1, o2 sql.RawBytes + if err := rows.Scan(&o1, &o2); err != nil { + dbt.Errorf("Got error: %v", err) + } + if !bytes.Equal(v1, o1) { + dbt.Errorf("expected %v, got %v", v1, o1) + } + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + // https://github.com/go-sql-driver/mysql/issues/765 + // Appending to RawBytes shouldn't overwrite next RawBytes. + o1 = append(o1, "xyzzy"...) + if !bytes.Equal(v2, o2) { + dbt.Errorf("expected %v, got %v", v2, o2) + } + } else { + dbt.Errorf("no data") + } + }) +} + +type testValuer struct { + value string +} + +func (tv testValuer) Value() (driver.Value, error) { + return tv.value, nil +} + +func TestValuer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuer{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE test (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO test VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM test") + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + rows.Close() + + dbt.mustExec("DROP TABLE IF EXISTS test") + }) +} + +type testValuerWithValidation struct { + value string +} + +func (tv testValuerWithValidation) Value() (driver.Value, error) { + if len(tv.value) == 0 { + return nil, fmt.Errorf("Invalid string valuer. Value must not be empty") + } + + return tv.value, nil +} + +func TestValuerWithValidation(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + in := testValuerWithValidation{"a_value"} + var out string + var rows *sql.Rows + + dbt.mustExec("CREATE TABLE testValuer (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO testValuer VALUES (?)", in) + + rows = dbt.mustQuery("SELECT value FROM testValuer") + defer rows.Close() + + if rows.Next() { + rows.Scan(&out) + if in.value != out { + dbt.Errorf("Valuer: %v != %s", in, out) + } + } else { + dbt.Errorf("Valuer: no data") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", testValuerWithValidation{""}); err == nil { + dbt.Errorf("Failed to check valuer error") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", nil); err != nil { + dbt.Errorf("Failed to check nil") + } + + if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", map[string]bool{}); err == nil { + dbt.Errorf("Failed to check not valuer") + } + + dbt.mustExec("DROP TABLE IF EXISTS testValuer") + }) +} + +type timeTests struct { + dbtype string + tlayout string + tests []timeTest +} + +type timeTest struct { + s string // leading "!": do not use t as value in queries + t time.Time +} + +type timeMode byte + +func (t timeMode) String() string { + switch t { + case binaryString: + return "binary:string" + case binaryTime: + return "binary:time.Time" + case textString: + return "text:string" + } + panic("unsupported timeMode") +} + +func (t timeMode) Binary() bool { + switch t { + case binaryString, binaryTime: + return true + } + return false +} + +const ( + binaryString timeMode = iota + binaryTime + textString +) + +func (t timeTest) genQuery(dbtype string, mode timeMode) string { + var inner string + if mode.Binary() { + inner = "?" + } else { + inner = `"%s"` + } + return `SELECT cast(` + inner + ` as ` + dbtype + `)` +} + +func (t timeTest) run(dbt *DBTest, dbtype, tlayout string, mode timeMode) { + var rows *sql.Rows + query := t.genQuery(dbtype, mode) + switch mode { + case binaryString: + rows = dbt.mustQuery(query, t.s) + case binaryTime: + rows = dbt.mustQuery(query, t.t) + case textString: + query = fmt.Sprintf(query, t.s) + rows = dbt.mustQuery(query) + default: + panic("unsupported mode") + } + defer rows.Close() + var err error + if !rows.Next() { + err = rows.Err() + if err == nil { + err = fmt.Errorf("no data") + } + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + var dst interface{} + err = rows.Scan(&dst) + if err != nil { + dbt.Errorf("%s [%s]: %s", dbtype, mode, err) + return + } + switch val := dst.(type) { + case []uint8: + str := string(val) + if str == t.s { + return + } + if mode.Binary() && dbtype == "DATETIME" && len(str) == 26 && str[:19] == t.s { + // a fix mainly for TravisCI: + // accept full microsecond resolution in result for DATETIME columns + // where the binary protocol was used + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, str, + ) + case time.Time: + if val == t.t { + return + } + dbt.Errorf("%s [%s] to string: expected %q, got %q", + dbtype, mode, + t.s, val.Format(tlayout), + ) + default: + fmt.Printf("%#v\n", []interface{}{dbtype, tlayout, mode, t.s, t.t}) + dbt.Errorf("%s [%s]: unhandled type %T (is '%v')", + dbtype, mode, + val, val, + ) + } +} + +func TestDateTime(t *testing.T) { + afterTime := func(t time.Time, d string) time.Time { + dur, err := time.ParseDuration(d) + if err != nil { + panic(err) + } + return t.Add(dur) + } + // NOTE: MySQL rounds DATETIME(x) up - but that's not included in the tests + format := "2006-01-02 15:04:05.999999" + t0 := time.Time{} + tstr0 := "0000-00-00 00:00:00.000000" + testcases := []timeTests{ + {"DATE", format[:10], []timeTest{ + {t: time.Date(2011, 11, 20, 0, 0, 0, 0, time.UTC)}, + {t: t0, s: tstr0[:10]}, + }}, + {"DATETIME", format[:19], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(0)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 0, time.UTC)}, + {t: t0, s: tstr0[:19]}, + }}, + {"DATETIME(1)", format[:21], []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 100000000, time.UTC)}, + {t: t0, s: tstr0[:21]}, + }}, + {"DATETIME(6)", format, []timeTest{ + {t: time.Date(2011, 11, 20, 21, 27, 37, 123456000, time.UTC)}, + {t: t0, s: tstr0}, + }}, + {"TIME", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(0)", format[11:19], []timeTest{ + {t: afterTime(t0, "12345s")}, + {s: "!-12:34:56"}, + {s: "!-838:59:59"}, + {s: "!838:59:59"}, + {t: t0, s: tstr0[11:19]}, + }}, + {"TIME(1)", format[11:21], []timeTest{ + {t: afterTime(t0, "12345600ms")}, + {s: "!-12:34:56.7"}, + {s: "!-838:59:58.9"}, + {s: "!838:59:58.9"}, + {t: t0, s: tstr0[11:21]}, + }}, + {"TIME(6)", format[11:], []timeTest{ + {t: afterTime(t0, "1234567890123000ns")}, + {s: "!-12:34:56.789012"}, + {s: "!-838:59:58.999999"}, + {s: "!838:59:58.999999"}, + {t: t0, s: tstr0[11:]}, + }}, + } + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + microsecsSupported := false + zeroDateSupported := false + var rows *sql.Rows + var err error + rows, err = dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`) + if err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + rows, err = dbt.db.Query(`SELECT cast("0000-00-00" as DATE) = "0000-00-00"`) + if err == nil { + rows.Scan(&zeroDateSupported) + rows.Close() + } + for _, setups := range testcases { + if t := setups.dbtype; !microsecsSupported && t[len(t)-1:] == ")" { + // skip fractional second tests if unsupported by server + continue + } + for _, setup := range setups.tests { + allowBinTime := true + if setup.s == "" { + // fill time string wherever Go can reliable produce it + setup.s = setup.t.Format(setups.tlayout) + } else if setup.s[0] == '!' { + // skip tests using setup.t as source in queries + allowBinTime = false + // fix setup.s - remove the "!" + setup.s = setup.s[1:] + } + if !zeroDateSupported && setup.s == tstr0[:len(setup.s)] { + // skip disallowed 0000-00-00 date + continue + } + setup.run(dbt, setups.dbtype, setups.tlayout, textString) + setup.run(dbt, setups.dbtype, setups.tlayout, binaryString) + if allowBinTime { + setup.run(dbt, setups.dbtype, setups.tlayout, binaryTime) + } + } + } + }) + } +} + +func TestTimestampMicros(t *testing.T) { + format := "2006-01-02 15:04:05.999999" + f0 := format[:19] + f1 := format[:21] + f6 := format[:26] + runTests(t, dsn, func(dbt *DBTest) { + // check if microseconds are supported. + // Do not use timestamp(x) for that check - before 5.5.6, x would mean display width + // and not precision. + // Se last paragraph at http://dev.mysql.com/doc/refman/5.6/en/fractional-seconds.html + microsecsSupported := false + if rows, err := dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`); err == nil { + rows.Scan(µsecsSupported) + rows.Close() + } + if !microsecsSupported { + // skip test + return + } + _, err := dbt.db.Exec(` + CREATE TABLE test ( + value0 TIMESTAMP NOT NULL DEFAULT '` + f0 + `', + value1 TIMESTAMP(1) NOT NULL DEFAULT '` + f1 + `', + value6 TIMESTAMP(6) NOT NULL DEFAULT '` + f6 + `' + )`, + ) + if err != nil { + dbt.Error(err) + } + defer dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("INSERT INTO test SET value0=?, value1=?, value6=?", f0, f1, f6) + var res0, res1, res6 string + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Errorf("test contained no selectable values") + } + err = rows.Scan(&res0, &res1, &res6) + if err != nil { + dbt.Error(err) + } + if res0 != f0 { + dbt.Errorf("expected %q, got %q", f0, res0) + } + if res1 != f1 { + dbt.Errorf("expected %q, got %q", f1, res1) + } + if res6 != f6 { + dbt.Errorf("expected %q, got %q", f6, res6) + } + }) +} + +func TestNULL(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + nullStmt, err := dbt.db.Prepare("SELECT NULL") + if err != nil { + dbt.Fatal(err) + } + defer nullStmt.Close() + + nonNullStmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + defer nonNullStmt.Close() + + // NullBool + var nb sql.NullBool + // Invalid + if err = nullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if nb.Valid { + dbt.Error("valid NullBool which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nb); err != nil { + dbt.Fatal(err) + } + if !nb.Valid { + dbt.Error("invalid NullBool which should be valid") + } else if nb.Bool != true { + dbt.Errorf("Unexpected NullBool value: %t (should be true)", nb.Bool) + } + + // NullFloat64 + var nf sql.NullFloat64 + // Invalid + if err = nullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if nf.Valid { + dbt.Error("valid NullFloat64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&nf); err != nil { + dbt.Fatal(err) + } + if !nf.Valid { + dbt.Error("invalid NullFloat64 which should be valid") + } else if nf.Float64 != float64(1) { + dbt.Errorf("unexpected NullFloat64 value: %f (should be 1.0)", nf.Float64) + } + + // NullInt64 + var ni sql.NullInt64 + // Invalid + if err = nullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if ni.Valid { + dbt.Error("valid NullInt64 which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ni); err != nil { + dbt.Fatal(err) + } + if !ni.Valid { + dbt.Error("invalid NullInt64 which should be valid") + } else if ni.Int64 != int64(1) { + dbt.Errorf("unexpected NullInt64 value: %d (should be 1)", ni.Int64) + } + + // NullString + var ns sql.NullString + // Invalid + if err = nullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if ns.Valid { + dbt.Error("valid NullString which should be invalid") + } + // Valid + if err = nonNullStmt.QueryRow().Scan(&ns); err != nil { + dbt.Fatal(err) + } + if !ns.Valid { + dbt.Error("invalid NullString which should be valid") + } else if ns.String != `1` { + dbt.Error("unexpected NullString value:" + ns.String + " (should be `1`)") + } + + // nil-bytes + var b []byte + // Read nil + if err = nullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil []byte which should be nil") + } + // Read non-nil + if err = nonNullStmt.QueryRow().Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil []byte which should be non-nil") + } + // Insert nil + b = nil + success := false + if err = dbt.db.QueryRow("SELECT ? IS NULL", b).Scan(&success); err != nil { + dbt.Fatal(err) + } + if !success { + dbt.Error("inserting []byte(nil) as NULL failed") + } + // Check input==output with input==nil + b = nil + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b != nil { + dbt.Error("non-nil echo from nil input") + } + // Check input==output with input!=nil + b = []byte("") + if err = dbt.db.QueryRow("SELECT ?", b).Scan(&b); err != nil { + dbt.Fatal(err) + } + if b == nil { + dbt.Error("nil echo from non-nil input") + } + + // Insert NULL + dbt.mustExec("CREATE TABLE test (dummmy1 int, value int, dummy2 int)") + + dbt.mustExec("INSERT INTO test VALUES (?, ?, ?)", 1, nil, 2) + + var out interface{} + rows := dbt.mustQuery("SELECT * FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if out != nil { + dbt.Errorf("%v != nil", out) + } + } else { + dbt.Error("no data") + } + }) +} + +func TestUint64(t *testing.T) { + const ( + u0 = uint64(0) + uall = ^u0 + uhigh = uall >> 1 + utop = ^uhigh + s0 = int64(0) + sall = ^s0 + shigh = int64(uhigh) + stop = ^shigh + ) + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare(`SELECT ?, ?, ? ,?, ?, ?, ?, ?`) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + row := stmt.QueryRow( + u0, uhigh, utop, uall, + s0, shigh, stop, sall, + ) + + var ua, ub, uc, ud uint64 + var sa, sb, sc, sd int64 + + err = row.Scan(&ua, &ub, &uc, &ud, &sa, &sb, &sc, &sd) + if err != nil { + dbt.Fatal(err) + } + switch { + case ua != u0, + ub != uhigh, + uc != utop, + ud != uall, + sa != s0, + sb != shigh, + sc != stop, + sd != sall: + dbt.Fatal("unexpected result value") + } + }) +} + +func TestLongData(t *testing.T) { + runTests(t, dsn+"&maxAllowedPacket=0", func(dbt *DBTest) { + var maxAllowedPacketSize int + err := dbt.db.QueryRow("select @@max_allowed_packet").Scan(&maxAllowedPacketSize) + if err != nil { + dbt.Fatal(err) + } + maxAllowedPacketSize-- + + // don't get too ambitious + if maxAllowedPacketSize > 1<<25 { + maxAllowedPacketSize = 1 << 25 + } + + dbt.mustExec("CREATE TABLE test (value LONGBLOB)") + + in := strings.Repeat(`a`, maxAllowedPacketSize+1) + var out string + var rows *sql.Rows + + // Long text data + const nonDataQueryLen = 28 // length query w/o value + inS := in[:maxAllowedPacketSize-nonDataQueryLen] + dbt.mustExec("INSERT INTO test VALUES('" + inS + "')") + rows = dbt.mustQuery("SELECT value FROM test") + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if inS != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(inS), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + dbt.Fatalf("LONGBLOB: no data") + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Long binary data + dbt.mustExec("INSERT INTO test VALUES(?)", in) + rows = dbt.mustQuery("SELECT value FROM test WHERE 1=?", 1) + defer rows.Close() + if rows.Next() { + rows.Scan(&out) + if in != out { + dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(in), len(out)) + } + if rows.Next() { + dbt.Error("LONGBLOB: unexpexted row") + } + } else { + if err = rows.Err(); err != nil { + dbt.Fatalf("LONGBLOB: no data (err: %s)", err.Error()) + } else { + dbt.Fatal("LONGBLOB: no data (err: )") + } + } + }) +} + +func TestLoadData(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + verifyLoadDataResult := func() { + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + dbt.Fatal(err.Error()) + } + + i := 0 + values := [4]string{ + "a string", + "a string containing a \t", + "a string containing a \n", + "a string containing both \t\n", + } + + var id int + var value string + + for rows.Next() { + i++ + err = rows.Scan(&id, &value) + if err != nil { + dbt.Fatal(err.Error()) + } + if i != id { + dbt.Fatalf("%d != %d", i, id) + } + if values[i-1] != value { + dbt.Fatalf("%q != %q", values[i-1], value) + } + } + err = rows.Err() + if err != nil { + dbt.Fatal(err.Error()) + } + + if i != 4 { + dbt.Fatalf("rows count mismatch. Got %d, want 4", i) + } + } + + dbt.db.Exec("DROP TABLE IF EXISTS test") + dbt.mustExec("CREATE TABLE test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") + + // Local File + file, err := ioutil.TempFile("", "gotest") + defer os.Remove(file.Name()) + if err != nil { + dbt.Fatal(err) + } + RegisterLocalFile(file.Name()) + + // Try first with empty file + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + var count int + err = dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&count) + if err != nil { + dbt.Fatal(err.Error()) + } + if count != 0 { + dbt.Fatalf("unexpected row count: got %d, want 0", count) + } + + // Then fille File with data and try to load it + file.WriteString("1\ta string\n2\ta string containing a \\t\n3\ta string containing a \\n\n4\ta string containing both \\t\\n\n") + file.Close() + dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) + verifyLoadDataResult() + + // Try with non-existing file + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent file didn't fail") + } else if err.Error() != "local file 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + + // Empty table + dbt.mustExec("TRUNCATE TABLE test") + + // Reader + RegisterReaderHandler("test", func() io.Reader { + file, err = os.Open(file.Name()) + if err != nil { + dbt.Fatal(err) + } + return file + }) + dbt.mustExec("LOAD DATA LOCAL INFILE 'Reader::test' INTO TABLE test") + verifyLoadDataResult() + // negative test + _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'Reader::doesnotexist' INTO TABLE test") + if err == nil { + dbt.Fatal("load non-existent Reader didn't fail") + } else if err.Error() != "Reader 'doesnotexist' is not registered" { + dbt.Fatal(err.Error()) + } + }) +} + +func TestFoundRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 affected rows, got %d", count) + } + }) + runTests(t, dsn+"&clientFoundRows=true", func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + count, err := res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 2 { + dbt.Fatalf("Expected 2 matched rows, got %d", count) + } + res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + count, err = res.RowsAffected() + if err != nil { + dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) + } + if count != 3 { + dbt.Fatalf("Expected 3 matched rows, got %d", count) + } + }) +} + +func TestTLS(t *testing.T) { + tlsTestReq := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + if err == ErrNoTLS { + dbt.Skip("server does not support TLS") + } else { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + rows := dbt.mustQuery("SHOW STATUS LIKE 'Ssl_cipher'") + defer rows.Close() + + var variable, value *sql.RawBytes + for rows.Next() { + if err := rows.Scan(&variable, &value); err != nil { + dbt.Fatal(err.Error()) + } + + if (*value == nil) || (len(*value) == 0) { + dbt.Fatalf("no Cipher") + } else { + dbt.Logf("Cipher: %s", *value) + } + } + } + tlsTestOpt := func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.Fatalf("error on Ping: %s", err.Error()) + } + } + + runTests(t, dsn+"&tls=preferred", tlsTestOpt) + runTests(t, dsn+"&tls=skip-verify", tlsTestReq) + + // Verify that registering / using a custom cfg works + RegisterTLSConfig("custom-skip-verify", &tls.Config{ + InsecureSkipVerify: true, + }) + runTests(t, dsn+"&tls=custom-skip-verify", tlsTestReq) +} + +func TestReuseClosedConnection(t *testing.T) { + // this test does not use sql.database, it uses the driver directly + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + md := &MySQLDriver{} + conn, err := md.Open(dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + stmt, err := conn.Prepare("DO 1") + if err != nil { + t.Fatalf("error preparing statement: %s", err.Error()) + } + _, err = stmt.Exec(nil) + if err != nil { + t.Fatalf("error executing statement: %s", err.Error()) + } + err = conn.Close() + if err != nil { + t.Fatalf("error closing connection: %s", err.Error()) + } + + defer func() { + if err := recover(); err != nil { + t.Errorf("panic after reusing a closed connection: %v", err) + } + }() + _, err = stmt.Exec(nil) + if err != nil && err != driver.ErrBadConn { + t.Errorf("unexpected error '%s', expected '%s'", + err.Error(), driver.ErrBadConn.Error()) + } +} + +func TestCharset(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mustSetCharset := func(charsetParam, expected string) { + runTests(t, dsn+"&"+charsetParam, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT @@character_set_connection") + defer rows.Close() + + if !rows.Next() { + dbt.Fatalf("error getting connection charset: %s", rows.Err()) + } + + var got string + rows.Scan(&got) + + if got != expected { + dbt.Fatalf("expected connection charset %s but got %s", expected, got) + } + }) + } + + // non utf8 test + mustSetCharset("charset=ascii", "ascii") + + // when the first charset is invalid, use the second + mustSetCharset("charset=none,utf8", "utf8") + + // when the first charset is valid, use it + mustSetCharset("charset=ascii,utf8", "ascii") + mustSetCharset("charset=utf8,ascii", "utf8") +} + +func TestFailingCharset(t *testing.T) { + runTests(t, dsn+"&charset=none", func(dbt *DBTest) { + // run query to really establish connection... + _, err := dbt.db.Exec("SELECT 1") + if err == nil { + dbt.db.Close() + t.Fatalf("connection must not succeed without a valid charset") + } + }) +} + +func TestCollation(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + defaultCollation := "utf8mb4_general_ci" + testCollations := []string{ + "", // do not set + defaultCollation, // driver default + "latin1_general_ci", + "binary", + "utf8_unicode_ci", + "cp1257_bin", + } + + for _, collation := range testCollations { + var expected, tdsn string + if collation != "" { + tdsn = dsn + "&collation=" + collation + expected = collation + } else { + tdsn = dsn + expected = defaultCollation + } + + runTests(t, tdsn, func(dbt *DBTest) { + var got string + if err := dbt.db.QueryRow("SELECT @@collation_connection").Scan(&got); err != nil { + dbt.Fatal(err) + } + + if got != expected { + dbt.Fatalf("expected connection collation %s but got %s", expected, got) + } + }) + } +} + +func TestColumnsWithAlias(t *testing.T) { + runTests(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1 AS A") + defer rows.Close() + cols, _ := rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A" { + t.Fatalf("expected column name \"A\", got \"%s\"", cols[0]) + } + + rows = dbt.mustQuery("SELECT * FROM (SELECT 1 AS one) AS A") + defer rows.Close() + cols, _ = rows.Columns() + if len(cols) != 1 { + t.Fatalf("expected 1 column, got %d", len(cols)) + } + if cols[0] != "A.one" { + t.Fatalf("expected column name \"A.one\", got \"%s\"", cols[0]) + } + }) +} + +func TestRawBytesResultExceedsBuffer(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // defaultBufSize from buffer.go + expected := strings.Repeat("abc", defaultBufSize) + + rows := dbt.mustQuery("SELECT '" + expected + "'") + defer rows.Close() + if !rows.Next() { + dbt.Error("expected result, got none") + } + var result sql.RawBytes + rows.Scan(&result) + if expected != string(result) { + dbt.Error("result did not match expected value") + } + }) +} + +func TestTimezoneConversion(t *testing.T) { + zones := []string{"UTC", "US/Central", "US/Pacific", "Local"} + + // Regression test for timezone handling + tzTest := func(dbt *DBTest) { + // Create table + dbt.mustExec("CREATE TABLE test (ts TIMESTAMP)") + + // Insert local time into database (should be converted) + usCentral, _ := time.LoadLocation("US/Central") + reftime := time.Date(2014, 05, 30, 18, 03, 17, 0, time.UTC).In(usCentral) + dbt.mustExec("INSERT INTO test VALUE (?)", reftime) + + // Retrieve time from DB + rows := dbt.mustQuery("SELECT ts FROM test") + defer rows.Close() + if !rows.Next() { + dbt.Fatal("did not get any rows out") + } + + var dbTime time.Time + err := rows.Scan(&dbTime) + if err != nil { + dbt.Fatal("Err", err) + } + + // Check that dates match + if reftime.Unix() != dbTime.Unix() { + dbt.Errorf("times do not match.\n") + dbt.Errorf(" Now(%v)=%v\n", usCentral, reftime) + dbt.Errorf(" Now(UTC)=%v\n", dbTime) + } + } + + for _, tz := range zones { + runTests(t, dsn+"&parseTime=true&loc="+url.QueryEscape(tz), tzTest) + } +} + +// Special cases + +func TestRowsClose(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows, err := dbt.db.Query("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + err = rows.Close() + if err != nil { + dbt.Fatal(err) + } + + if rows.Next() { + dbt.Fatal("unexpected row after rows.Close()") + } + + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + }) +} + +// dangling statements +// http://code.google.com/p/go/issues/detail?id=3865 +func TestCloseStmtBeforeRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1") + if err != nil { + dbt.Fatal(err) + } + + rows, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows.Close() + + err = stmt.Close() + if err != nil { + dbt.Fatal(err) + } + + if !rows.Next() { + dbt.Fatal("getting row failed") + } else { + err = rows.Err() + if err != nil { + dbt.Fatal(err) + } + + var out bool + err = rows.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + }) +} + +// It is valid to have multiple Rows for the same Stmt +// http://code.google.com/p/go/issues/detail?id=3734 +func TestStmtMultiRows(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + stmt, err := dbt.db.Prepare("SELECT 1 UNION SELECT 0") + if err != nil { + dbt.Fatal(err) + } + + rows1, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows1.Close() + + rows2, err := stmt.Query() + if err != nil { + stmt.Close() + dbt.Fatal(err) + } + defer rows2.Close() + + var out bool + + // 1 + if !rows1.Next() { + dbt.Fatal("first rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + if !rows2.Next() { + dbt.Fatal("first rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != true { + dbt.Errorf("true != %t", out) + } + } + + // 2 + if !rows1.Next() { + dbt.Fatal("second rows1.Next failed") + } else { + err = rows1.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows1.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows1.Next() { + dbt.Fatal("unexpected row on rows1") + } + err = rows1.Close() + if err != nil { + dbt.Fatal(err) + } + } + + if !rows2.Next() { + dbt.Fatal("second rows2.Next failed") + } else { + err = rows2.Err() + if err != nil { + dbt.Fatal(err) + } + + err = rows2.Scan(&out) + if err != nil { + dbt.Fatalf("error on rows.Scan(): %s", err.Error()) + } + if out != false { + dbt.Errorf("false != %t", out) + } + + if rows2.Next() { + dbt.Fatal("unexpected row on rows2") + } + err = rows2.Close() + if err != nil { + dbt.Fatal(err) + } + } + }) +} + +// Regression test for +// * more than 32 NULL parameters (issue 209) +// * more parameters than fit into the buffer (issue 201) +// * parameters * 64 > max_allowed_packet (issue 734) +func TestPreparedManyCols(t *testing.T) { + numParams := 65535 + runTests(t, dsn, func(dbt *DBTest) { + query := "SELECT ?" + strings.Repeat(",?", numParams-1) + stmt, err := dbt.db.Prepare(query) + if err != nil { + dbt.Fatal(err) + } + defer stmt.Close() + + // create more parameters than fit into the buffer + // which will take nil-values + params := make([]interface{}, numParams) + rows, err := stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + + // Create 0byte string which we can't send via STMT_LONG_DATA. + for i := 0; i < numParams; i++ { + params[i] = "" + } + rows, err = stmt.Query(params...) + if err != nil { + dbt.Fatal(err) + } + rows.Close() + }) +} + +func TestConcurrent(t *testing.T) { + if enabled, _ := readBool(os.Getenv("MYSQL_TEST_CONCURRENT")); !enabled { + t.Skip("MYSQL_TEST_CONCURRENT env var not set") + } + + runTests(t, dsn, func(dbt *DBTest) { + var max int + err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + dbt.Logf("testing up to %d concurrent connections \r\n", max) + + var remaining, succeeded int32 = int32(max), 0 + + var wg sync.WaitGroup + wg.Add(max) + + var fatalError string + var once sync.Once + fatalf := func(s string, vals ...interface{}) { + once.Do(func() { + fatalError = fmt.Sprintf(s, vals...) + }) + } + + for i := 0; i < max; i++ { + go func(id int) { + defer wg.Done() + + tx, err := dbt.db.Begin() + atomic.AddInt32(&remaining, -1) + + if err != nil { + if err.Error() != "Error 1040: Too many connections" { + fatalf("error on conn %d: %s", id, err.Error()) + } + return + } + + // keep the connection busy until all connections are open + for remaining > 0 { + if _, err = tx.Exec("DO 1"); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + } + + if err = tx.Commit(); err != nil { + fatalf("error on conn %d: %s", id, err.Error()) + return + } + + // everything went fine with this connection + atomic.AddInt32(&succeeded, 1) + }(i) + } + + // wait until all conections are open + wg.Wait() + + if fatalError != "" { + dbt.Fatal(fatalError) + } + + dbt.Logf("reached %d concurrent connections\r\n", succeeded) + }) +} + +func testDialError(t *testing.T, dialErr error, expectErr error) { + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + return nil, dialErr + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + _, err = db.Exec("DO 1") + if err != expectErr { + t.Fatalf("was expecting %s. Got: %s", dialErr, err) + } +} + +func TestDialUnknownError(t *testing.T) { + testErr := fmt.Errorf("test") + testDialError(t, testErr, testErr) +} + +func TestDialNonRetryableNetErr(t *testing.T) { + testErr := netErrorMock{} + testDialError(t, testErr, testErr) +} + +func TestDialTemporaryNetErr(t *testing.T) { + testErr := netErrorMock{temporary: true} + testDialError(t, testErr, driver.ErrBadConn) +} + +// Tests custom dial functions +func TestCustomDial(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + // our custom dial function which justs wraps net.Dial here + RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + if _, err = db.Exec("DO 1"); err != nil { + t.Fatalf("connection failed: %s", err.Error()) + } +} + +func TestSQLInjection(t *testing.T) { + createTest := func(arg string) func(dbt *DBTest) { + return func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (?)", 1) + + var v int + // NULL can't be equal to anything, the idea here is to inject query so it returns row + // This test verifies that escapeQuotes and escapeBackslash are working properly + err := dbt.db.QueryRow("SELECT v FROM test WHERE NULL = ?", arg).Scan(&v) + if err == sql.ErrNoRows { + return // success, sql injection failed + } else if err == nil { + dbt.Errorf("sql injection successful with arg: %s", arg) + } else { + dbt.Errorf("error running query with arg: %s; err: %s", arg, err.Error()) + } + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, createTest("1 OR 1=1")) + runTests(t, testdsn, createTest("' OR '1'='1")) + } +} + +// Test if inserted data is correctly retrieved after being escaped +func TestInsertRetrieveEscapedData(t *testing.T) { + testData := func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v VARCHAR(255))") + + // All sequences that are escaped by escapeQuotes and escapeBackslash + v := "foo \x00\n\r\x1a\"'\\" + dbt.mustExec("INSERT INTO test VALUES (?)", v) + + var out string + err := dbt.db.QueryRow("SELECT v FROM test").Scan(&out) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + if out != v { + dbt.Errorf("%q != %q", out, v) + } + } + + dsns := []string{ + dsn, + dsn + "&sql_mode='NO_BACKSLASH_ESCAPES'", + } + for _, testdsn := range dsns { + runTests(t, testdsn, testData) + } +} + +func TestUnixSocketAuthFail(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Save the current logger so we can restore it. + oldLogger := errLog + + // Set a new logger so we can capture its output. + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + newLogger := log.New(buffer, "prefix: ", 0) + SetLogger(newLogger) + + // Restore the logger. + defer SetLogger(oldLogger) + + // Make a new DSN that uses the MySQL socket file and a bad password, which + // we can make by simply appending any character to the real password. + badPass := pass + "x" + socket := "" + if prot == "unix" { + socket = addr + } else { + // Get socket file from MySQL. + err := dbt.db.QueryRow("SELECT @@socket").Scan(&socket) + if err != nil { + t.Fatalf("error on SELECT @@socket: %s", err.Error()) + } + } + t.Logf("socket: %s", socket) + badDSN := fmt.Sprintf("%s:%s@unix(%s)/%s?timeout=30s", user, badPass, socket, dbname) + db, err := sql.Open("mysql", badDSN) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + // Connect to MySQL for real. This will cause an auth failure. + err = db.Ping() + if err == nil { + t.Error("expected Ping() to return an error") + } + + // The driver should not log anything. + if actual := buffer.String(); actual != "" { + t.Errorf("expected no output, got %q", actual) + } + }) +} + +// See Issue #422 +func TestInterruptBySignal(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + DROP PROCEDURE IF EXISTS test_signal; + CREATE PROCEDURE test_signal(ret INT) + BEGIN + SELECT ret; + SIGNAL SQLSTATE + '45001' + SET + MESSAGE_TEXT = "an error", + MYSQL_ERRNO = 45001; + END + `) + defer dbt.mustExec("DROP PROCEDURE test_signal") + + var val int + + // text protocol + rows, err := dbt.db.Query("CALL test_signal(42)") + if err != nil { + dbt.Fatalf("error on text query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + + // binary protocol + rows, err = dbt.db.Query("CALL test_signal(?)", 42) + if err != nil { + dbt.Fatalf("error on binary query: %s", err.Error()) + } + for rows.Next() { + if err := rows.Scan(&val); err != nil { + dbt.Error(err) + } else if val != 42 { + dbt.Errorf("expected val to be 42") + } + } + rows.Close() + }) +} + +func TestColumnsReusesSlice(t *testing.T) { + rows := mysqlRows{ + rs: resultSet{ + columns: []mysqlField{ + { + tableName: "test", + name: "A", + }, + { + tableName: "test", + name: "B", + }, + }, + }, + } + + allocs := testing.AllocsPerRun(1, func() { + cols := rows.Columns() + + if len(cols) != 2 { + t.Fatalf("expected 2 columns, got %d", len(cols)) + } + }) + + if allocs != 0 { + t.Fatalf("expected 0 allocations, got %d", int(allocs)) + } + + if rows.rs.columnNames == nil { + t.Fatalf("expected columnNames to be set, got nil") + } +} + +func TestRejectReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read-only. We didn't set the `rejectReadOnly` + // option, so any writes after this should fail. + _, err := dbt.db.Exec("SET SESSION TRANSACTION READ ONLY") + // Error 1193: Unknown system variable 'TRANSACTION' => skip test, + // MySQL server version is too old + maybeSkip(t, err, 1193) + if _, err := dbt.db.Exec("DROP TABLE test"); err == nil { + t.Fatalf("writing to DB in read-only session without " + + "rejectReadOnly did not error") + } + // Set the session back to read-write so runTests() can properly clean + // up the table `test`. + dbt.mustExec("SET SESSION TRANSACTION READ WRITE") + }) + + // Enable the `rejectReadOnly` option. + runTests(t, dsn+"&rejectReadOnly=true", func(dbt *DBTest) { + // Create Table + dbt.mustExec("CREATE TABLE test (value BOOL)") + // Set the session to read only. Any writes after this should error on + // a driver.ErrBadConn, and cause `database/sql` to initiate a new + // connection. + dbt.mustExec("SET SESSION TRANSACTION READ ONLY") + // This would error, but `database/sql` should automatically retry on a + // new connection which is not read-only, and eventually succeed. + dbt.mustExec("DROP TABLE test") + }) +} + +func TestPing(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + if err := dbt.db.Ping(); err != nil { + dbt.fail("Ping", "Ping", err) + } + }) +} + +// See Issue #799 +func TestEmptyPassword(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + dsn := fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, "", netAddr, dbname) + db, err := sql.Open("mysql", dsn) + if err == nil { + defer db.Close() + err = db.Ping() + } + + if pass == "" { + if err != nil { + t.Fatal(err.Error()) + } + } else { + if err == nil { + t.Fatal("expected authentication error") + } + if !strings.HasPrefix(err.Error(), "Error 1045") { + t.Fatal(err.Error()) + } + } +} + +// static interface implementation checks of mysqlConn +var ( + _ driver.ConnBeginTx = &mysqlConn{} + _ driver.ConnPrepareContext = &mysqlConn{} + _ driver.ExecerContext = &mysqlConn{} + _ driver.Pinger = &mysqlConn{} + _ driver.QueryerContext = &mysqlConn{} +) + +// static interface implementation checks of mysqlStmt +var ( + _ driver.StmtExecContext = &mysqlStmt{} + _ driver.StmtQueryContext = &mysqlStmt{} +) + +// Ensure that all the driver interfaces are implemented +var ( + // _ driver.RowsColumnTypeLength = &binaryRows{} + // _ driver.RowsColumnTypeLength = &textRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &binaryRows{} + _ driver.RowsColumnTypeDatabaseTypeName = &textRows{} + _ driver.RowsColumnTypeNullable = &binaryRows{} + _ driver.RowsColumnTypeNullable = &textRows{} + _ driver.RowsColumnTypePrecisionScale = &binaryRows{} + _ driver.RowsColumnTypePrecisionScale = &textRows{} + _ driver.RowsColumnTypeScanType = &binaryRows{} + _ driver.RowsColumnTypeScanType = &textRows{} + _ driver.RowsNextResultSet = &binaryRows{} + _ driver.RowsNextResultSet = &textRows{} +) + +func TestMultiResultSet(t *testing.T) { + type result struct { + values [][]int + columns []string + } + + // checkRows is a helper test function to validate rows containing 3 result + // sets with specific values and columns. The basic query would look like this: + // + // SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + // SELECT 0 UNION SELECT 1; + // SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + // + // to distinguish test cases the first string argument is put in front of + // every error or fatal message. + checkRows := func(desc string, rows *sql.Rows, dbt *DBTest) { + expected := []result{ + { + values: [][]int{{1, 2}, {3, 4}}, + columns: []string{"col1", "col2"}, + }, + { + values: [][]int{{1, 2, 3}, {4, 5, 6}}, + columns: []string{"col1", "col2", "col3"}, + }, + } + + var res1 result + for rows.Next() { + var res [2]int + if err := rows.Scan(&res[0], &res[1]); err != nil { + dbt.Fatal(err) + } + res1.values = append(res1.values, res[:]) + } + + cols, err := rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res1.columns = cols + + if !reflect.DeepEqual(expected[0], res1) { + dbt.Error(desc, "want =", expected[0], "got =", res1) + } + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + // ignoring one result set + + if !rows.NextResultSet() { + dbt.Fatal(desc, "expected next result set") + } + + var res2 result + cols, err = rows.Columns() + if err != nil { + dbt.Fatal(desc, err) + } + res2.columns = cols + + for rows.Next() { + var res [3]int + if err := rows.Scan(&res[0], &res[1], &res[2]); err != nil { + dbt.Fatal(desc, err) + } + res2.values = append(res2.values, res[:]) + } + + if !reflect.DeepEqual(expected[1], res2) { + dbt.Error(desc, "want =", expected[1], "got =", res2) + } + + if rows.NextResultSet() { + dbt.Error(desc, "unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error(desc, err) + } + } + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery(`DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6;`) + defer rows.Close() + checkRows("query: ", rows, dbt) + }) + + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + queries := []string{ + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + DO 1; + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + DO 1; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + ` + DROP PROCEDURE IF EXISTS test_mrss; + CREATE PROCEDURE test_mrss() + BEGIN + SELECT 1 AS col1, 2 AS col2 UNION SELECT 3, 4; + SELECT 0 UNION SELECT 1; + SELECT 1 AS col1, 2 AS col2, 3 AS col3 UNION SELECT 4, 5, 6; + END + `, + } + + defer dbt.mustExec("DROP PROCEDURE IF EXISTS test_mrss") + + for i, query := range queries { + dbt.mustExec(query) + + stmt, err := dbt.db.Prepare("CALL test_mrss()") + if err != nil { + dbt.Fatalf("%v (i=%d)", err, i) + } + defer stmt.Close() + + for j := 0; j < 2; j++ { + rows, err := stmt.Query() + if err != nil { + dbt.Fatalf("%v (i=%d) (j=%d)", err, i, j) + } + checkRows(fmt.Sprintf("prepared stmt query (i=%d) (j=%d): ", i, j), rows, dbt) + } + } + }) +} + +func TestMultiResultSetNoSelect(t *testing.T) { + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("DO 1; DO 2;") + defer rows.Close() + + if rows.Next() { + dbt.Error("unexpected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +// tests if rows are set in a proper state if some results were ignored before +// calling rows.NextResultSet. +func TestSkipResults(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + rows := dbt.mustQuery("SELECT 1, 2") + defer rows.Close() + + if !rows.Next() { + dbt.Error("expected row") + } + + if rows.NextResultSet() { + dbt.Error("unexpected next result set") + } + + if err := rows.Err(); err != nil { + dbt.Error("expected nil; got ", err) + } + }) +} + +func TestPingContext(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := dbt.db.PingContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (1)"); err == nil { + dbt.Error("expected error") + } else if err.Error() != "context canceled" { + dbt.Fatalf("unexpected error: %s", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + + // Context is already canceled, so error should come before execution. + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (1)"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + + // The second insert query will fail, so the table has no changes. + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelQueryRow(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + dbt.mustExec("INSERT INTO test VALUES (1), (2), (3)") + ctx, cancel := context.WithCancel(context.Background()) + + rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM test") + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + // the first row will be succeed. + var v int + if !rows.Next() { + dbt.Fatalf("unexpected end") + } + if err := rows.Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + + cancel() + // make sure the driver receives the cancel request. + time.Sleep(100 * time.Millisecond) + + if rows.Next() { + dbt.Errorf("expected end, but not") + } + if err := rows.Err(); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelPrepare(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := dbt.db.PrepareContext(ctx, "SELECT 1"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextCancelStmtExec(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.ExecContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query to be done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelStmtQuery(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + if err != nil { + dbt.Fatalf("unexpected error: %v", err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(250*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := stmt.QueryContext(ctx); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Wait for the INSERT query has done. + time.Sleep(time.Second) + + // Check how many times the query is executed. + var v int + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + dbt.Fatalf("%s", err.Error()) + } + if v != 1 { // TODO: need to kill the query, and v should be 0. + dbt.Skipf("[WARN] expected val to be 1, got %d", v) + } + }) +} + +func TestContextCancelBegin(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + tx, err := dbt.db.BeginTx(ctx, nil) + if err != nil { + dbt.Fatal(err) + } + + // Delay execution for just a bit until db.ExecContext has begun. + defer time.AfterFunc(100*time.Millisecond, cancel).Stop() + + // This query will be canceled. + startTime := time.Now() + if _, err := tx.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + if d := time.Since(startTime); d > 500*time.Millisecond { + dbt.Errorf("too long execution time: %s", d) + } + + // Transaction is canceled, so expect an error. + switch err := tx.Commit(); err { + case sql.ErrTxDone: + // because the transaction has already been rollbacked. + // the database/sql package watches ctx + // and rollbacks when ctx is canceled. + case context.Canceled: + // the database/sql package rollbacks on another goroutine, + // so the transaction may not be rollbacked depending on goroutine scheduling. + default: + dbt.Errorf("expected sql.ErrTxDone or context.Canceled, got %v", err) + } + + // Context is canceled, so cannot begin a transaction. + if _, err := dbt.db.BeginTx(ctx, nil); err != context.Canceled { + dbt.Errorf("expected context.Canceled, got %v", err) + } + }) +} + +func TestContextBeginIsolationLevel(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx1, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelRepeatableRead, + }) + if err != nil { + dbt.Fatal(err) + } + + tx2, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelReadCommitted, + }) + if err != nil { + dbt.Fatal(err) + } + + _, err = tx1.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if err != nil { + dbt.Fatal(err) + } + + var v int + row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Because writer transaction wasn't commited yet, it should be available + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + err = tx1.Commit() + if err != nil { + dbt.Fatal(err) + } + + row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + // Data written by writer transaction is already commited, it should be selectable + if v != 1 { + dbt.Errorf("expected val to be 1, got %d", v) + } + tx2.Commit() + }) +} + +func TestContextBeginReadOnly(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (v INTEGER)") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tx, err := dbt.db.BeginTx(ctx, &sql.TxOptions{ + ReadOnly: true, + }) + if _, ok := err.(*MySQLError); ok { + dbt.Skip("It seems that your MySQL does not support READ ONLY transactions") + return + } else if err != nil { + dbt.Fatal(err) + } + + // INSERT queries fail in a READ ONLY transaction. + _, err = tx.ExecContext(ctx, "INSERT INTO test VALUES (1)") + if _, ok := err.(*MySQLError); !ok { + dbt.Errorf("expected MySQLError, got %v", err) + } + + // SELECT queries can be executed. + var v int + row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + if err := row.Scan(&v); err != nil { + dbt.Fatal(err) + } + if v != 0 { + dbt.Errorf("expected val to be 0, got %d", v) + } + + if err := tx.Commit(); err != nil { + dbt.Fatal(err) + } + }) +} + +func TestRowsColumnTypes(t *testing.T) { + niNULL := sql.NullInt64{Int64: 0, Valid: false} + ni0 := sql.NullInt64{Int64: 0, Valid: true} + ni1 := sql.NullInt64{Int64: 1, Valid: true} + ni42 := sql.NullInt64{Int64: 42, Valid: true} + nfNULL := sql.NullFloat64{Float64: 0.0, Valid: false} + nf0 := sql.NullFloat64{Float64: 0.0, Valid: true} + nf1337 := sql.NullFloat64{Float64: 13.37, Valid: true} + nt0 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 0, time.UTC), Valid: true} + nt1 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 100000000, time.UTC), Valid: true} + nt2 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 110000000, time.UTC), Valid: true} + nt6 := NullTime{Time: time.Date(2006, 01, 02, 15, 04, 05, 111111000, time.UTC), Valid: true} + nd1 := NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} + nd2 := NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} + ndNULL := NullTime{Time: time.Time{}, Valid: false} + rbNULL := sql.RawBytes(nil) + rb0 := sql.RawBytes("0") + rb42 := sql.RawBytes("42") + rbTest := sql.RawBytes("Test") + rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 + rbx0 := sql.RawBytes("\x00") + rbx42 := sql.RawBytes("\x42") + + var columns = []struct { + name string + fieldType string // type used when creating table schema + databaseTypeName string // actual type used by MySQL + scanType reflect.Type + nullable bool + precision int64 // 0 if not ok + scale int64 + valuesIn [3]string + valuesOut [3]interface{} + }{ + {"bit8null", "BIT(8)", "BIT", scanTypeRawBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]interface{}{rbx0, rbNULL, rbx42}}, + {"boolnull", "BOOL", "TINYINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "true", "0"}, [3]interface{}{niNULL, ni1, ni0}}, + {"bool", "BOOL NOT NULL", "TINYINT", scanTypeInt8, false, 0, 0, [3]string{"1", "0", "FALSE"}, [3]interface{}{int8(1), int8(0), int8(0)}}, + {"intnull", "INTEGER", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"smallint", "SMALLINT NOT NULL", "SMALLINT", scanTypeInt16, false, 0, 0, [3]string{"0", "-32768", "32767"}, [3]interface{}{int16(0), int16(-32768), int16(32767)}}, + {"smallintnull", "SMALLINT", "SMALLINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int3null", "INT(3)", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, + {"int7", "INT(7) NOT NULL", "INT", scanTypeInt32, false, 0, 0, [3]string{"0", "-1337", "42"}, [3]interface{}{int32(0), int32(-1337), int32(42)}}, + {"mediumintnull", "MEDIUMINT", "MEDIUMINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "42", "NULL"}, [3]interface{}{ni0, ni42, niNULL}}, + {"bigint", "BIGINT NOT NULL", "BIGINT", scanTypeInt64, false, 0, 0, [3]string{"0", "65535", "-42"}, [3]interface{}{int64(0), int64(65535), int64(-42)}}, + {"bigintnull", "BIGINT", "BIGINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "1", "42"}, [3]interface{}{niNULL, ni1, ni42}}, + {"tinyuint", "TINYINT UNSIGNED NOT NULL", "TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, + {"smalluint", "SMALLINT UNSIGNED NOT NULL", "SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, + {"biguint", "BIGINT UNSIGNED NOT NULL", "BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, + {"uint13", "INT(13) UNSIGNED NOT NULL", "INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, + {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float32(0), float32(42), float32(13.37)}}, + {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"double", "DOUBLE NOT NULL", "DOUBLE", scanTypeFloat64, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float64(0), float64(42), float64(13.37)}}, + {"doublenull", "DOUBLE", "DOUBLE", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, + {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), sql.RawBytes("13.370000"), sql.RawBytes("1234.123456")}}, + {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeRawBytes, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), rbNULL, sql.RawBytes("1234.123456")}}, + {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), sql.RawBytes("13.3700"), sql.RawBytes("1234.1235")}}, + {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, + {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, + {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, + {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, + {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, + {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, + {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, + {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, + {"date", "DATE", "DATE", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02'", "NULL", "'2006-03-04'"}, [3]interface{}{nd1, ndNULL, nd2}}, + {"year", "YEAR NOT NULL", "YEAR", scanTypeUint16, false, 0, 0, [3]string{"2006", "2000", "1994"}, [3]interface{}{uint16(2006), uint16(2000), uint16(1994)}}, + } + + schema := "" + values1 := "" + values2 := "" + values3 := "" + for _, column := range columns { + schema += fmt.Sprintf("`%s` %s, ", column.name, column.fieldType) + values1 += column.valuesIn[0] + ", " + values2 += column.valuesIn[1] + ", " + values3 += column.valuesIn[2] + ", " + } + schema = schema[:len(schema)-2] + values1 = values1[:len(values1)-2] + values2 = values2[:len(values2)-2] + values3 = values3[:len(values3)-2] + + dsns := []string{ + dsn + "&parseTime=true", + dsn + "&parseTime=false", + } + for _, testdsn := range dsns { + runTests(t, testdsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (" + schema + ")") + dbt.mustExec("INSERT INTO test VALUES (" + values1 + "), (" + values2 + "), (" + values3 + ")") + + rows, err := dbt.db.Query("SELECT * FROM test") + if err != nil { + t.Fatalf("Query: %v", err) + } + + tt, err := rows.ColumnTypes() + if err != nil { + t.Fatalf("ColumnTypes: %v", err) + } + + if len(tt) != len(columns) { + t.Fatalf("unexpected number of columns: expected %d, got %d", len(columns), len(tt)) + } + + types := make([]reflect.Type, len(tt)) + for i, tp := range tt { + column := columns[i] + + // Name + name := tp.Name() + if name != column.name { + t.Errorf("column name mismatch %s != %s", name, column.name) + continue + } + + // DatabaseTypeName + databaseTypeName := tp.DatabaseTypeName() + if databaseTypeName != column.databaseTypeName { + t.Errorf("databasetypename name mismatch for column %q: %s != %s", name, databaseTypeName, column.databaseTypeName) + continue + } + + // ScanType + scanType := tp.ScanType() + if scanType != column.scanType { + if scanType == nil { + t.Errorf("scantype is null for column %q", name) + } else { + t.Errorf("scantype mismatch for column %q: %s != %s", name, scanType.Name(), column.scanType.Name()) + } + continue + } + types[i] = scanType + + // Nullable + nullable, ok := tp.Nullable() + if !ok { + t.Errorf("nullable not ok %q", name) + continue + } + if nullable != column.nullable { + t.Errorf("nullable mismatch for column %q: %t != %t", name, nullable, column.nullable) + } + + // Length + // length, ok := tp.Length() + // if length != column.length { + // if !ok { + // t.Errorf("length not ok for column %q", name) + // } else { + // t.Errorf("length mismatch for column %q: %d != %d", name, length, column.length) + // } + // continue + // } + + // Precision and Scale + precision, scale, ok := tp.DecimalSize() + if precision != column.precision { + if !ok { + t.Errorf("precision not ok for column %q", name) + } else { + t.Errorf("precision mismatch for column %q: %d != %d", name, precision, column.precision) + } + continue + } + if scale != column.scale { + if !ok { + t.Errorf("scale not ok for column %q", name) + } else { + t.Errorf("scale mismatch for column %q: %d != %d", name, scale, column.scale) + } + continue + } + } + + values := make([]interface{}, len(tt)) + for i := range values { + values[i] = reflect.New(types[i]).Interface() + } + i := 0 + for rows.Next() { + err = rows.Scan(values...) + if err != nil { + t.Fatalf("failed to scan values in %v", err) + } + for j := range values { + value := reflect.ValueOf(values[j]).Elem().Interface() + if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { + if columns[j].scanType == scanTypeRawBytes { + t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) + } else { + t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) + } + } + } + i++ + } + if i != 3 { + t.Errorf("expected 3 rows, got %d", i) + } + + if err := rows.Close(); err != nil { + t.Errorf("error closing rows: %s", err) + } + }) + } +} + +func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (value VARCHAR(255))") + dbt.db.Exec("INSERT INTO test VALUES (?)", (*testValuer)(nil)) + // This test will panic on the INSERT if ConvertValue() does not check for typed nil before calling Value() + }) +} + +// TestRawBytesAreNotModified checks for a race condition that arises when a query context +// is canceled while a user is calling rows.Scan. This is a more stringent test than the one +// proposed in https://github.com/golang/go/issues/23519. Here we're explicitly using +// `sql.RawBytes` to check the contents of our internal buffers are not modified after an implicit +// call to `Rows.Close`, so Context cancellation should **not** invalidate the backing buffers. +func TestRawBytesAreNotModified(t *testing.T) { + const blob = "abcdefghijklmnop" + const contextRaceIterations = 20 + const blobSize = defaultBufSize * 3 / 4 // Second row overwrites first row. + const insertRows = 4 + + var sqlBlobs = [2]string{ + strings.Repeat(blob, blobSize/len(blob)), + strings.Repeat(strings.ToUpper(blob), blobSize/len(blob)), + } + + runTests(t, dsn, func(dbt *DBTest) { + dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + for i := 0; i < insertRows; i++ { + dbt.mustExec("INSERT INTO test VALUES (?, ?)", i+1, sqlBlobs[i&1]) + } + + for i := 0; i < contextRaceIterations; i++ { + func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rows, err := dbt.db.QueryContext(ctx, `SELECT id, value FROM test`) + if err != nil { + t.Fatal(err) + } + + var b int + var raw sql.RawBytes + for rows.Next() { + if err := rows.Scan(&b, &raw); err != nil { + t.Fatal(err) + } + + before := string(raw) + // Ensure cancelling the query does not corrupt the contents of `raw` + cancel() + time.Sleep(time.Microsecond * 100) + after := string(raw) + + if before != after { + t.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) + } + } + rows.Close() + }() + } + }) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go index be014ba..1d9b4ab 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -14,6 +14,7 @@ import ( "crypto/tls" "errors" "fmt" + "math/big" "net" "net/url" "sort" @@ -72,6 +73,26 @@ func NewConfig() *Config { } } +func (cfg *Config) Clone() *Config { + cp := *cfg + if cp.tls != nil { + cp.tls = cfg.tls.Clone() + } + if len(cp.Params) > 0 { + cp.Params = make(map[string]string, len(cfg.Params)) + for k, v := range cfg.Params { + cp.Params[k] = v + } + } + if cfg.pubKey != nil { + cp.pubKey = &rsa.PublicKey{ + N: new(big.Int).Set(cfg.pubKey.N), + E: cfg.pubKey.E, + } + } + return &cp +} + func (cfg *Config) normalize() error { if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { return errInvalidDSNUnsafeCollation @@ -92,17 +113,35 @@ func (cfg *Config) normalize() error { default: return errors.New("default addr for network '" + cfg.Net + "' unknown") } - } else if cfg.Net == "tcp" { cfg.Addr = ensureHavePort(cfg.Addr) } - if cfg.tls != nil { - if cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.Addr) - if err == nil { - cfg.tls.ServerName = host - } + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.tls = &tls.Config{} + case "skip-verify", "preferred": + cfg.tls = &tls.Config{InsecureSkipVerify: true} + default: + cfg.tls = getTLSConfigClone(cfg.TLSConfig) + if cfg.tls == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } + } + + if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + + if cfg.ServerPubKey != "" { + cfg.pubKey = getServerPubKey(cfg.ServerPubKey) + if cfg.pubKey == nil { + return errors.New("invalid value / unknown server pub key name: " + cfg.ServerPubKey) } } @@ -531,13 +570,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { if err != nil { return fmt.Errorf("invalid value for server pub key name: %v", err) } - - if pubKey := getServerPubKey(name); pubKey != nil { - cfg.ServerPubKey = name - cfg.pubKey = pubKey - } else { - return errors.New("invalid value / unknown server pub key name: " + name) - } + cfg.ServerPubKey = name // Strict mode case "strict": @@ -556,25 +589,17 @@ func parseDSNParams(cfg *Config, params string) (err error) { if isBool { if boolValue { cfg.TLSConfig = "true" - cfg.tls = &tls.Config{} } else { cfg.TLSConfig = "false" } - } else if vl := strings.ToLower(value); vl == "skip-verify" { + } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { cfg.TLSConfig = vl - cfg.tls = &tls.Config{InsecureSkipVerify: true} } else { name, err := url.QueryUnescape(value) if err != nil { return fmt.Errorf("invalid value for TLS config name: %v", err) } - - if tlsConfig := getTLSConfigClone(name); tlsConfig != nil { - cfg.TLSConfig = name - cfg.tls = tlsConfig - } else { - return errors.New("invalid value / unknown config name: " + name) - } + cfg.TLSConfig = name } // I/O write Timeout diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go new file mode 100644 index 0000000..50dc293 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/dsn_test.go @@ -0,0 +1,415 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "crypto/tls" + "fmt" + "net/url" + "reflect" + "testing" + "time" +) + +var testDSNs = []struct { + in string + out *Config +}{{ + "username:password@protocol(address)/dbname?param=value", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, +}, { + "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, +}, { + "user@unix(/path/to/socket)/dbname?charset=utf8", + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, +}, { + "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, +}, { + "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, +}, { + "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, +}, { + "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "@/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "/", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "user:p@/ssword@/", + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "unix/?arg=%2Fsome%2Fpath.ext", + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(127.0.0.1)/dbname", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, { + "tcp(de:ad:be:ef::ca:fe)/dbname", + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, +}, +} + +func TestDSNParser(t *testing.T) { + for i, tst := range testDSNs { + cfg, err := ParseDSN(tst.in) + if err != nil { + t.Error(err.Error()) + } + + // pointer not static + cfg.tls = nil + + if !reflect.DeepEqual(cfg, tst.out) { + t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) + } + } +} + +func TestDSNParserInvalid(t *testing.T) { + var invalidDSNs = []string{ + "@net(addr/", // no closing brace + "@tcp(/", // no closing brace + "tcp(/", // no closing brace + "(/", // no closing brace + "net(addr)//", // unescaped + "User:pass@tcp(1.2.3.4:3306)", // no trailing slash + "net()/", // unknown default addr + //"/dbname?arg=/some/unescaped/path", + } + + for i, tst := range invalidDSNs { + if _, err := ParseDSN(tst); err == nil { + t.Errorf("invalid DSN #%d. (%s) didn't error!", i, tst) + } + } +} + +func TestDSNReformat(t *testing.T) { + for i, tst := range testDSNs { + dsn1 := tst.in + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Error(err.Error()) + continue + } + cfg1.tls = nil // pointer not static + res1 := fmt.Sprintf("%+v", cfg1) + + dsn2 := cfg1.FormatDSN() + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Error(err.Error()) + continue + } + cfg2.tls = nil // pointer not static + res2 := fmt.Sprintf("%+v", cfg2) + + if res1 != res2 { + t.Errorf("%d. %q does not match %q", i, res2, res1) + } + } +} + +func TestDSNServerPubKey(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + tst := baseDSN + "testKey" + cfg, err := ParseDSN(tst) + if err != nil { + t.Error(err.Error()) + } + + if cfg.ServerPubKey != "testKey" { + t.Errorf("unexpected cfg.ServerPubKey value: %v", cfg.ServerPubKey) + } + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } + + // Key is missing + tst = baseDSN + "invalid_name" + cfg, err = ParseDSN(tst) + if err == nil { + t.Errorf("invalid name in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } +} + +func TestDSNServerPubKeyQueryEscape(t *testing.T) { + const name = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?serverPubKey=" + url.QueryEscape(name) + + RegisterServerPubKey(name, testPubKeyRSA) + defer DeregisterServerPubKey(name) + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + + if cfg.pubKey != testPubKeyRSA { + t.Error("pub key pointer doesn't match") + } +} + +func TestDSNWithCustomTLS(t *testing.T) { + baseDSN := "User:password@tcp(localhost:5555)/dbname?tls=" + tlsCfg := tls.Config{} + + RegisterTLSConfig("utils_test", &tlsCfg) + defer DeregisterTLSConfig("utils_test") + + // Custom TLS is missing + tst := baseDSN + "invalid_tls" + cfg, err := ParseDSN(tst) + if err == nil { + t.Errorf("invalid custom TLS in DSN (%s) but did not error. Got config: %#v", tst, cfg) + } + + tst = baseDSN + "utils_test" + + // Custom TLS with a server name + name := "foohost" + tlsCfg.ServerName = name + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, tst) + } + + // Custom TLS without a server name + name = "localhost" + tlsCfg.ServerName = "" + cfg, err = ParseDSN(tst) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct ServerName (%s) parsing DSN (%s).", name, tst) + } else if tlsCfg.ServerName != "" { + t.Errorf("tlsCfg was mutated ServerName (%s) should be empty parsing DSN (%s).", name, tst) + } +} + +func TestDSNTLSConfig(t *testing.T) { + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true" + + cfg, err := ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + dsn = "tcp(example.com)/?tls=true" + cfg, err = ParseDSN(dsn) + if err != nil { + t.Error(err.Error()) + } + if cfg.tls == nil { + t.Error("cfg.tls should not be nil") + } + if cfg.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host without port)", expectedServerName, cfg.tls.ServerName) + } +} + +func TestDSNWithCustomTLSQueryEscape(t *testing.T) { + const configKey = "&%!:" + dsn := "User:password@tcp(localhost:5555)/dbname?tls=" + url.QueryEscape(configKey) + name := "foohost" + tlsCfg := tls.Config{ServerName: name} + + RegisterTLSConfig(configKey, &tlsCfg) + defer DeregisterTLSConfig(configKey) + + cfg, err := ParseDSN(dsn) + + if err != nil { + t.Error(err.Error()) + } else if cfg.tls.ServerName != name { + t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, dsn) + } +} + +func TestDSNUnsafeCollation(t *testing.T) { + _, err := ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=true") + if err != errInvalidDSNUnsafeCollation { + t.Errorf("expected %v, got %v", errInvalidDSNUnsafeCollation, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci&interpolateParams=false") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=gbk_chinese_ci") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=ascii_bin&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=latin1_german1_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } + + _, err = ParseDSN("/dbname?collation=utf8mb4_general_ci&interpolateParams=true") + if err != nil { + t.Errorf("expected %v, got %v", nil, err) + } +} + +func TestParamsAreSorted(t *testing.T) { + expected := "/dbname?interpolateParams=true&foobar=baz&quux=loo" + cfg := NewConfig() + cfg.DBName = "dbname" + cfg.InterpolateParams = true + cfg.Params = map[string]string{ + "quux": "loo", + "foobar": "baz", + } + actual := cfg.FormatDSN() + if actual != expected { + t.Errorf("generic Config.Params were not sorted: want %#v, got %#v", expected, actual) + } +} + +func TestCloneConfig(t *testing.T) { + RegisterServerPubKey("testKey", testPubKeyRSA) + defer DeregisterServerPubKey("testKey") + + expectedServerName := "example.com" + dsn := "tcp(example.com:1234)/?tls=true&foobar=baz&serverPubKey=testKey" + cfg, err := ParseDSN(dsn) + if err != nil { + t.Fatal(err.Error()) + } + + cfg2 := cfg.Clone() + if cfg == cfg2 { + t.Errorf("Config.Clone did not create a separate config struct") + } + + if cfg2.tls.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + } + + cfg2.tls.ServerName = "example2.com" + if cfg.tls.ServerName == cfg2.tls.ServerName { + t.Errorf("changed cfg.tls.Server name should not propagate to original Config") + } + + if _, ok := cfg2.Params["foobar"]; !ok { + t.Errorf("cloned Config is missing custom params") + } + + delete(cfg2.Params, "foobar") + + if _, ok := cfg.Params["foobar"]; !ok { + t.Errorf("custom params in cloned Config should not propagate to original Config") + } + + if !reflect.DeepEqual(cfg.pubKey, cfg2.pubKey) { + t.Errorf("public key in Config should be identical") + } +} + +func TestNormalizeTLSConfig(t *testing.T) { + tt := []struct { + tlsConfig string + want *tls.Config + }{ + {"", nil}, + {"false", nil}, + {"true", &tls.Config{ServerName: "myserver"}}, + {"skip-verify", &tls.Config{InsecureSkipVerify: true}}, + {"preferred", &tls.Config{InsecureSkipVerify: true}}, + {"test_tls_config", &tls.Config{ServerName: "myServerName"}}, + } + + RegisterTLSConfig("test_tls_config", &tls.Config{ServerName: "myServerName"}) + defer func() { DeregisterTLSConfig("test_tls_config") }() + + for _, tc := range tt { + t.Run(tc.tlsConfig, func(t *testing.T) { + cfg := &Config{ + Addr: "myserver:3306", + TLSConfig: tc.tlsConfig, + } + + cfg.normalize() + + if cfg.tls == nil { + if tc.want != nil { + t.Fatal("wanted a tls config but got nil instead") + } + return + } + + if cfg.tls.ServerName != tc.want.ServerName { + t.Errorf("tls.ServerName doesn't match (want: '%s', got: '%s')", + tc.want.ServerName, cfg.tls.ServerName) + } + if cfg.tls.InsecureSkipVerify != tc.want.InsecureSkipVerify { + t.Errorf("tls.InsecureSkipVerify doesn't match (want: %T, got :%T)", + tc.want.InsecureSkipVerify, cfg.tls.InsecureSkipVerify) + } + }) + } +} + +func BenchmarkParseDSN(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + for _, tst := range testDSNs { + if _, err := ParseDSN(tst.in); err != nil { + b.Error(err.Error()) + } + } + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go new file mode 100644 index 0000000..96f9126 --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/errors_test.go @@ -0,0 +1,42 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "log" + "testing" +) + +func TestErrorsSetLogger(t *testing.T) { + previous := errLog + defer func() { + errLog = previous + }() + + // set up logger + const expected = "prefix: test\n" + buffer := bytes.NewBuffer(make([]byte, 0, 64)) + logger := log.New(buffer, "prefix: ", 0) + + // print + SetLogger(logger) + errLog.Print("test") + + // check result + if actual := buffer.String(); actual != expected { + t.Errorf("expected %q, got %q", expected, actual) + } +} + +func TestErrorsStrictIgnoreNotes(t *testing.T) { + runTests(t, dsn+"&sql_notes=false", func(dbt *DBTest) { + dbt.mustExec("DROP TABLE IF EXISTS does_not_exist") + }) +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go index 170aaa0..30b3352 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets.go @@ -51,7 +51,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { mc.sequence++ // packets with length 0 terminate a previous packet which is a - // multiple of (2^24)−1 bytes long + // multiple of (2^24)-1 bytes long if pktLen == 0 { // there was no previous packet if prevData == nil { @@ -96,6 +96,35 @@ func (mc *mysqlConn) writePacket(data []byte) error { return ErrPktTooLarge } + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.reset { + mc.reset = false + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + // If this connection has a ReadTimeout which we've been setting on + // reads, reset it to its default value before we attempt a non-blocking + // read, otherwise the scheduler will just time us out before we can read + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Time{}) + } + if err == nil { + err = connCheck(conn) + } + if err != nil { + errLog.Print("closing bad idle connection: ", err) + mc.Close() + return driver.ErrBadConn + } + } + for { var size int if pktLen >= maxPacketSize { @@ -194,7 +223,11 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro return nil, "", ErrOldProtocol } if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - return nil, "", ErrNoTLS + if mc.cfg.TLSConfig == "preferred" { + mc.cfg.tls = nil + } else { + return nil, "", ErrNoTLS + } } pos += 2 @@ -243,7 +276,7 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro // Client Authentication Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse -func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, plugin string) error { +func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error { // Adjust client flags based on server support clientFlags := clientProtocol41 | clientSecureConn | @@ -269,7 +302,8 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // encode length of the auth plugin data var authRespLEIBuf [9]byte - authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(authResp))) + authRespLen := len(authResp) + authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen)) if len(authRespLEI) > 1 { // if the length can not be written in 1 byte, it must be written as a // length encoded integer @@ -277,9 +311,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, } pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1 - if addNUL { - pktLen++ - } // To specify a db name if n := len(mc.cfg.DBName); n > 0 { @@ -288,10 +319,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, } // Calculate packet length and get buffer with that size - data := mc.buf.takeSmallBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -330,6 +361,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, if err := tlsConn.Handshake(); err != nil { return err } + mc.rawConn = mc.netConn mc.netConn = tlsConn mc.buf.nc = tlsConn } @@ -350,10 +382,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // Auth Data [length encoded integer] pos += copy(data[pos:], authRespLEI) pos += copy(data[pos:], authResp) - if addNUL { - data[pos] = 0x00 - pos++ - } // Databasename [null terminated string] if len(mc.cfg.DBName) > 0 { @@ -364,30 +392,24 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, pos += copy(data[pos:], plugin) data[pos] = 0x00 + pos++ // Send Auth packet - return mc.writePacket(data) + return mc.writePacket(data[:pos]) } // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte, addNUL bool) error { +func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { pktLen := 4 + len(authData) - if addNUL { - pktLen++ - } - data := mc.buf.takeSmallBuffer(pktLen) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } // Add the auth data [EOF] copy(data[4:], authData) - if addNUL { - data[pktLen-1] = 0x00 - } - return mc.writePacket(data) } @@ -399,10 +421,10 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -418,10 +440,10 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { mc.sequence = 0 pktLen := 1 + len(arg) - data := mc.buf.takeBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -439,10 +461,10 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1 + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -479,7 +501,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { return data[1:], "", err case iEOF: - if len(data) < 1 { + if len(data) == 1 { // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest return nil, "mysql_old_password", nil } @@ -895,7 +917,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { const minPktLen = 4 + 1 + 4 + 1 + 4 mc := stmt.mc - // Determine threshould dynamically to avoid packet size shortage. + // Determine threshold dynamically to avoid packet size shortage. longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1) if longDataSize < 64 { longDataSize = 64 @@ -905,15 +927,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { mc.sequence = 0 var data []byte + var err error if len(args) == 0 { - data = mc.buf.takeBuffer(minPktLen) + data, err = mc.buf.takeBuffer(minPktLen) } else { - data = mc.buf.takeCompleteBuffer() + data, err = mc.buf.takeCompleteBuffer() + // In this case the len(data) == cap(data) which is used to optimise the flow below. } - if data == nil { + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -939,7 +963,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { pos := minPktLen var nullMask []byte - if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { + if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= cap(data) { // buffer has to be extended but we don't know by how much so // we depend on append after all data with known sizes fit. // We stop at that because we deal with a lot of columns here @@ -948,10 +972,11 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { copy(tmp[:pos], data[:pos]) data = tmp nullMask = data[pos : pos+maskLen] + // No need to clean nullMask as make ensures that. pos += maskLen } else { nullMask = data[pos : pos+maskLen] - for i := 0; i < maskLen; i++ { + for i := range nullMask { nullMask[i] = 0 } pos += maskLen @@ -996,6 +1021,22 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { ) } + case uint64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x80 // type is unsigned + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + case float64: paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 @@ -1088,7 +1129,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In that case we must build the data packet with the new values buffer if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) - mc.buf.buf = data + if err = mc.buf.store(data); err != nil { + errLog.Print(err) + return errBadConnNoWrite + } } pos += len(paramValues) diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go new file mode 100644 index 0000000..b61e4db --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/packets_test.go @@ -0,0 +1,336 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "errors" + "net" + "testing" + "time" +) + +var ( + errConnClosed = errors.New("connection is closed") + errConnTooManyReads = errors.New("too many reads") + errConnTooManyWrites = errors.New("too many writes") +) + +// struct to mock a net.Conn for testing purposes +type mockConn struct { + laddr net.Addr + raddr net.Addr + data []byte + written []byte + queuedReplies [][]byte + closed bool + read int + reads int + writes int + maxReads int + maxWrites int +} + +func (m *mockConn) Read(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.reads++ + if m.maxReads > 0 && m.reads > m.maxReads { + return 0, errConnTooManyReads + } + + n = copy(b, m.data) + m.read += n + m.data = m.data[n:] + return +} +func (m *mockConn) Write(b []byte) (n int, err error) { + if m.closed { + return 0, errConnClosed + } + + m.writes++ + if m.maxWrites > 0 && m.writes > m.maxWrites { + return 0, errConnTooManyWrites + } + + n = len(b) + m.written = append(m.written, b...) + + if n > 0 && len(m.queuedReplies) > 0 { + m.data = m.queuedReplies[0] + m.queuedReplies = m.queuedReplies[1:] + } + return +} +func (m *mockConn) Close() error { + m.closed = true + return nil +} +func (m *mockConn) LocalAddr() net.Addr { + return m.laddr +} +func (m *mockConn) RemoteAddr() net.Addr { + return m.raddr +} +func (m *mockConn) SetDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetReadDeadline(t time.Time) error { + return nil +} +func (m *mockConn) SetWriteDeadline(t time.Time) error { + return nil +} + +// make sure mockConn implements the net.Conn interface +var _ net.Conn = new(mockConn) + +func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: NewConfig(), + netConn: conn, + closech: make(chan struct{}), + maxAllowedPacket: defaultMaxAllowedPacket, + sequence: sequence, + } + return conn, mc +} + +func TestReadPacketSingleByte(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 1 { + t.Fatalf("unexpected packet length: expected %d, got %d", 1, len(packet)) + } + if packet[0] != 0xff { + t.Fatalf("unexpected packet content: expected %x, got %x", 0xff, packet[0]) + } +} + +func TestReadPacketWrongSequenceID(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + // too low sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} + conn.maxReads = 1 + mc.sequence = 1 + _, err := mc.readPacket() + if err != ErrPktSync { + t.Errorf("expected ErrPktSync, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // too high sequence id + conn.data = []byte{0x01, 0x00, 0x00, 0x42, 0xff} + _, err = mc.readPacket() + if err != ErrPktSyncMul { + t.Errorf("expected ErrPktSyncMul, got %v", err) + } +} + +func TestReadPacketSplit(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + } + + data := make([]byte, maxPacketSize*2+4*3) + const pkt2ofs = maxPacketSize + 4 + const pkt3ofs = 2 * (maxPacketSize + 4) + + // case 1: payload has length maxPacketSize + data = data[:pkt2ofs+4] + + // 1st packet has maxPacketSize length and sequence id 0 + // ff ff ff 00 ... + data[0] = 0xff + data[1] = 0xff + data[2] = 0xff + + // mark the payload start and end of 1st packet so that we can check if the + // content was correctly appended + data[4] = 0x11 + data[maxPacketSize+3] = 0x22 + + // 2nd packet has payload length 0 and squence id 1 + // 00 00 00 01 + data[pkt2ofs+3] = 0x01 + + conn.data = data + conn.maxReads = 3 + packet, err := mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize-1] != 0x22 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x22, packet[maxPacketSize-1]) + } + + // case 2: payload has length which is a multiple of maxPacketSize + data = data[:cap(data)] + + // 2nd packet now has maxPacketSize length + data[pkt2ofs] = 0xff + data[pkt2ofs+1] = 0xff + data[pkt2ofs+2] = 0xff + + // mark the payload start and end of the 2nd packet + data[pkt2ofs+4] = 0x33 + data[pkt2ofs+maxPacketSize+3] = 0x44 + + // 3rd packet has payload length 0 and squence id 2 + // 00 00 00 02 + data[pkt3ofs+3] = 0x02 + + conn.data = data + conn.reads = 0 + conn.maxReads = 5 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != 2*maxPacketSize { + t.Fatalf("unexpected packet length: expected %d, got %d", 2*maxPacketSize, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[2*maxPacketSize-1] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[2*maxPacketSize-1]) + } + + // case 3: payload has a length larger maxPacketSize, which is not an exact + // multiple of it + data = data[:pkt2ofs+4+42] + data[pkt2ofs] = 0x2a + data[pkt2ofs+1] = 0x00 + data[pkt2ofs+2] = 0x00 + data[pkt2ofs+4+41] = 0x44 + + conn.data = data + conn.reads = 0 + conn.maxReads = 4 + mc.sequence = 0 + packet, err = mc.readPacket() + if err != nil { + t.Fatal(err) + } + if len(packet) != maxPacketSize+42 { + t.Fatalf("unexpected packet length: expected %d, got %d", maxPacketSize+42, len(packet)) + } + if packet[0] != 0x11 { + t.Fatalf("unexpected payload start: expected %x, got %x", 0x11, packet[0]) + } + if packet[maxPacketSize+41] != 0x44 { + t.Fatalf("unexpected payload end: expected %x, got %x", 0x44, packet[maxPacketSize+41]) + } +} + +func TestReadPacketFail(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + closech: make(chan struct{}), + } + + // illegal empty (stand-alone) packet + conn.data = []byte{0x00, 0x00, 0x00, 0x00} + conn.maxReads = 1 + _, err := mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read header + conn.closed = true + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } + + // reset + conn.closed = false + conn.reads = 0 + mc.sequence = 0 + mc.buf = newBuffer(conn) + + // fail to read body + conn.maxReads = 1 + _, err = mc.readPacket() + if err != ErrInvalidConn { + t.Errorf("expected ErrInvalidConn, got %v", err) + } +} + +// https://github.com/go-sql-driver/mysql/pull/801 +// not-NUL terminated plugin_name in init packet +func TestRegression801(t *testing.T) { + conn := new(mockConn) + mc := &mysqlConn{ + buf: newBuffer(conn), + cfg: new(Config), + sequence: 42, + closech: make(chan struct{}), + } + + conn.data = []byte{72, 0, 0, 42, 10, 53, 46, 53, 46, 56, 0, 165, 0, 0, 0, + 60, 70, 63, 58, 68, 104, 34, 97, 0, 223, 247, 33, 2, 0, 15, 128, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 120, 114, 47, 85, 75, 109, 99, 51, 77, + 50, 64, 0, 109, 121, 115, 113, 108, 95, 110, 97, 116, 105, 118, 101, 95, + 112, 97, 115, 115, 119, 111, 114, 100} + conn.maxReads = 1 + + authData, pluginName, err := mc.readHandshakePacket() + if err != nil { + t.Fatalf("got error: %v", err) + } + + if pluginName != "mysql_native_password" { + t.Errorf("expected plugin name 'mysql_native_password', got '%s'", pluginName) + } + + expectedAuthData := []byte{60, 70, 63, 58, 68, 104, 34, 97, 98, 120, 114, + 47, 85, 75, 109, 99, 51, 77, 50, 64} + if !bytes.Equal(authData, expectedAuthData) { + t.Errorf("expected authData '%v', got '%v'", expectedAuthData, authData) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go index d3b1e28..888bdb5 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/rows.go @@ -111,6 +111,13 @@ func (rows *mysqlRows) Close() (err error) { return err } + // flip the buffer for this connection if we need to drain it. + // note that for a successful query (i.e. one where rows.next() + // has been called until it returns false), `rows.mc` will be nil + // by the time the user calls `(*Rows).Close`, so we won't reach this + // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 + mc.buf.flip() + // Remove unread packets from stream if !rows.rs.done { err = mc.readUntilEOF() diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go index ce7fe4c..f7e3709 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "reflect" - "strconv" ) type mysqlStmt struct { @@ -164,14 +163,8 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint(), nil case reflect.Float32, reflect.Float64: return rv.Float(), nil case reflect.Bool: diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go new file mode 100644 index 0000000..4b9914f --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/statement_test.go @@ -0,0 +1,126 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "testing" +) + +func TestConvertDerivedString(t *testing.T) { + type derived string + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Derived string type not convertible", err) + } + + if output != "value" { + t.Fatalf("Derived string type not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedByteSlice(t *testing.T) { + type derived []uint8 + + output, err := converter{}.ConvertValue(derived("value")) + if err != nil { + t.Fatal("Byte slice not convertible", err) + } + + if bytes.Compare(output.([]byte), []byte("value")) != 0 { + t.Fatalf("Byte slice not converted, got %#v %T", output, output) + } +} + +func TestConvertDerivedUnsupportedSlice(t *testing.T) { + type derived []int + + _, err := converter{}.ConvertValue(derived{1}) + if err == nil || err.Error() != "unsupported type mysql.derived, a slice of int" { + t.Fatal("Unexpected error", err) + } +} + +func TestConvertDerivedBool(t *testing.T) { + type derived bool + + output, err := converter{}.ConvertValue(derived(true)) + if err != nil { + t.Fatal("Derived bool type not convertible", err) + } + + if output != true { + t.Fatalf("Derived bool type not converted, got %#v %T", output, output) + } +} + +func TestConvertPointer(t *testing.T) { + str := "value" + + output, err := converter{}.ConvertValue(&str) + if err != nil { + t.Fatal("Pointer type not convertible", err) + } + + if output != "value" { + t.Fatalf("Pointer type not converted, got %#v %T", output, output) + } +} + +func TestConvertSignedIntegers(t *testing.T) { + values := []interface{}{ + int8(-42), + int16(-42), + int32(-42), + int64(-42), + int(-42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != int64(-42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } +} + +func TestConvertUnsignedIntegers(t *testing.T) { + values := []interface{}{ + uint8(42), + uint16(42), + uint32(42), + uint64(42), + uint(42), + } + + for _, value := range values { + output, err := converter{}.ConvertValue(value) + if err != nil { + t.Fatalf("%T type not convertible %s", value, err) + } + + if output != uint64(42) { + t.Fatalf("%T type not converted, got %#v %T", value, output, output) + } + } + + output, err := converter{}.ConvertValue(^uint64(0)) + if err != nil { + t.Fatal("uint64 high-bit not convertible", err) + } + + if output != ^uint64(0) { + t.Fatalf("uint64 high-bit converted, got %#v %T", output, output) + } +} diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go index cb3650b..cfa10e9 100644 --- a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils.go @@ -56,7 +56,7 @@ var ( // db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") // func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { return fmt.Errorf("key '%s' is reserved", key) } @@ -684,7 +684,7 @@ type atomicBool struct { value uint32 } -// IsSet returns wether the current boolean value is true +// IsSet returns whether the current boolean value is true func (ab *atomicBool) IsSet() bool { return atomic.LoadUint32(&ab.value) > 0 } @@ -698,7 +698,7 @@ func (ab *atomicBool) Set(value bool) { } } -// TrySet sets the value of the bool and returns wether the value changed +// TrySet sets the value of the bool and returns whether the value changed func (ab *atomicBool) TrySet(value bool) bool { if value { return atomic.SwapUint32(&ab.value, 1) == 0 diff --git a/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go new file mode 100644 index 0000000..8951a7a --- /dev/null +++ b/vendor/github.com/siddontang/go-mysql/vendor/github.com/go-sql-driver/mysql/utils_test.go @@ -0,0 +1,334 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "database/sql" + "database/sql/driver" + "encoding/binary" + "testing" + "time" +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} + +func TestLengthEncodedInteger(t *testing.T) { + var integerTests = []struct { + num uint64 + encoded []byte + }{ + {0x0000000000000000, []byte{0x00}}, + {0x0000000000000012, []byte{0x12}}, + {0x00000000000000fa, []byte{0xfa}}, + {0x0000000000000100, []byte{0xfc, 0x00, 0x01}}, + {0x0000000000001234, []byte{0xfc, 0x34, 0x12}}, + {0x000000000000ffff, []byte{0xfc, 0xff, 0xff}}, + {0x0000000000010000, []byte{0xfd, 0x00, 0x00, 0x01}}, + {0x0000000000123456, []byte{0xfd, 0x56, 0x34, 0x12}}, + {0x0000000000ffffff, []byte{0xfd, 0xff, 0xff, 0xff}}, + {0x0000000001000000, []byte{0xfe, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}}, + {0x123456789abcdef0, []byte{0xfe, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12}}, + {0xffffffffffffffff, []byte{0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + } + + for _, tst := range integerTests { + num, isNull, numLen := readLengthEncodedInteger(tst.encoded) + if isNull { + t.Errorf("%x: expected %d, got NULL", tst.encoded, tst.num) + } + if num != tst.num { + t.Errorf("%x: expected %d, got %d", tst.encoded, tst.num, num) + } + if numLen != len(tst.encoded) { + t.Errorf("%x: expected size %d, got %d", tst.encoded, len(tst.encoded), numLen) + } + encoded := appendLengthEncodedInteger(nil, num) + if !bytes.Equal(encoded, tst.encoded) { + t.Errorf("%v: expected %x, got %x", num, tst.encoded, encoded) + } + } +} + +func TestFormatBinaryDateTime(t *testing.T) { + rawDate := [11]byte{} + binary.LittleEndian.PutUint16(rawDate[:2], 1978) // years + rawDate[2] = 12 // months + rawDate[3] = 30 // days + rawDate[4] = 15 // hours + rawDate[5] = 46 // minutes + rawDate[6] = 23 // seconds + binary.LittleEndian.PutUint32(rawDate[7:], 987654) // microseconds + expect := func(expected string, inlen, outlen uint8) { + actual, _ := formatBinaryDateTime(rawDate[:inlen], outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for length in %d, out %d", + expected, actual, inlen, outlen, + ) + } + } + expect("0000-00-00", 0, 10) + expect("0000-00-00 00:00:00", 0, 19) + expect("1978-12-30", 4, 10) + expect("1978-12-30 15:46:23", 7, 19) + expect("1978-12-30 15:46:23.987654", 11, 26) +} + +func TestFormatBinaryTime(t *testing.T) { + expect := func(expected string, src []byte, outlen uint8) { + actual, _ := formatBinaryTime(src, outlen) + bytes, ok := actual.([]byte) + if !ok { + t.Errorf("formatBinaryDateTime must return []byte, was %T", actual) + } + if string(bytes) != expected { + t.Errorf( + "expected %q, got %q for src=%q and outlen=%d", + expected, actual, src, outlen) + } + } + + // binary format: + // sign (0: positive, 1: negative), days(4), hours, minutes, seconds, micro(4) + + // Zeros + expect("00:00:00", []byte{}, 8) + expect("00:00:00.0", []byte{}, 10) + expect("00:00:00.000000", []byte{}, 15) + + // Without micro(4) + expect("12:34:56", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("-12:34:56", []byte{1, 0, 0, 0, 0, 12, 34, 56}, 8) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56}, 11) + expect("24:34:56", []byte{0, 1, 0, 0, 0, 0, 34, 56}, 8) + expect("-99:34:56", []byte{1, 4, 0, 0, 0, 3, 34, 56}, 8) + expect("103079215103:34:56", []byte{0, 255, 255, 255, 255, 23, 34, 56}, 8) + + // With micro(4) + expect("12:34:56.00", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 11) + expect("12:34:56.000099", []byte{0, 0, 0, 0, 0, 12, 34, 56, 99, 0, 0, 0}, 15) +} + +func TestEscapeBackslash(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesBackslash([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringBackslash([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\\0bar", "foo\x00bar") + expect("foo\\nbar", "foo\nbar") + expect("foo\\rbar", "foo\rbar") + expect("foo\\Zbar", "foo\x1abar") + expect("foo\\\"bar", "foo\"bar") + expect("foo\\\\bar", "foo\\bar") + expect("foo\\'bar", "foo'bar") +} + +func TestEscapeQuotes(t *testing.T) { + expect := func(expected, value string) { + actual := string(escapeBytesQuotes([]byte{}, []byte(value))) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + + actual = string(escapeStringQuotes([]byte{}, value)) + if actual != expected { + t.Errorf( + "expected %s, got %s", + expected, actual, + ) + } + } + + expect("foo\x00bar", "foo\x00bar") // not affected + expect("foo\nbar", "foo\nbar") // not affected + expect("foo\rbar", "foo\rbar") // not affected + expect("foo\x1abar", "foo\x1abar") // not affected + expect("foo''bar", "foo'bar") // affected + expect("foo\"bar", "foo\"bar") // not affected +} + +func TestAtomicBool(t *testing.T) { + var ab atomicBool + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(true) + if ab.value != 1 { + t.Fatal("Set(true) did not set value to 1") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(false) + if ab.value != 0 { + t.Fatal("Set(false) did not set value to 0") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab.Set(false) + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + if ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to fail") + } + if !ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to succeed") + } + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + + ab.Set(true) + if !ab.IsSet() { + t.Fatal("Expected value to be true") + } + if ab.TrySet(true) { + t.Fatal("Expected TrySet(true) to fail") + } + if !ab.TrySet(false) { + t.Fatal("Expected TrySet(false) to succeed") + } + if ab.IsSet() { + t.Fatal("Expected value to be false") + } + + ab._noCopy.Lock() // we've "tested" it ¯\_(ツ)_/¯ +} + +func TestAtomicError(t *testing.T) { + var ae atomicError + if ae.Value() != nil { + t.Fatal("Expected value to be nil") + } + + ae.Set(ErrMalformPkt) + if v := ae.Value(); v != ErrMalformPkt { + if v == nil { + t.Fatal("Value is still nil") + } + t.Fatal("Error did not match") + } + ae.Set(ErrPktSync) + if ae.Value() == ErrMalformPkt { + t.Fatal("Error still matches old error") + } + if v := ae.Value(); v != ErrPktSync { + t.Fatal("Error did not match") + } +} + +func TestIsolationLevelMapping(t *testing.T) { + data := []struct { + level driver.IsolationLevel + expected string + }{ + { + level: driver.IsolationLevel(sql.LevelReadCommitted), + expected: "READ COMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelRepeatableRead), + expected: "REPEATABLE READ", + }, + { + level: driver.IsolationLevel(sql.LevelReadUncommitted), + expected: "READ UNCOMMITTED", + }, + { + level: driver.IsolationLevel(sql.LevelSerializable), + expected: "SERIALIZABLE", + }, + } + + for i, td := range data { + if actual, err := mapIsolationLevel(td.level); actual != td.expected || err != nil { + t.Fatal(i, td.expected, actual, err) + } + } + + // check unsupported mapping + expectedErr := "mysql: unsupported isolation level: 7" + actual, err := mapIsolationLevel(driver.IsolationLevel(sql.LevelLinearizable)) + if actual != "" || err == nil { + t.Fatal("Expected error on unsupported isolation level") + } + if err.Error() != expectedErr { + t.Fatalf("Expected error to be %q, got %q", expectedErr, err) + } +} From 84fe34b22c9a80a265ffbea043ef1918ceed7eef Mon Sep 17 00:00:00 2001 From: Zach Moazeni Date: Mon, 10 Jun 2019 09:24:03 -0400 Subject: [PATCH 113/176] Bump default-retries to 3 There are some legitimate retries that can occur during testing. Namely `logic.ExpectProcess()` (in `applier.go`). We'll look for a process that does exist, but timing-wise doesn't have the `state` or `info` columns populated. Without this, the test will fail abruptly. --- localtests/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localtests/test.sh b/localtests/test.sh index 17b60e9..1078965 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -152,7 +152,7 @@ test_single() { --serve-socket-file=/tmp/gh-ost.test.sock \ --initially-drop-socket-file \ --test-on-replica \ - --default-retries=1 \ + --default-retries=3 \ --chunk-size=10 \ --verbose \ --debug \ From 43ea21c3bb1b6b9b5e50038278e7bebbe5d0f096 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 14:54:00 +0300 Subject: [PATCH 114/176] Initial GitHub CI/CD GitHub CI/CD is [a thing](https://github.com/features/actions)! We will attempt: - CI: build + unit test - replica tests: running `gh-ost` tests on multiple MySQL versions --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ec8978d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: CI + +on: [pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + + - name: Set up Go 1.12 + uses: actions/setup-go@v1 + with: + version: 1.12 + id: go + + - name: Build + run: script/cibuild From 6cfa76d1399238fb6350125e1b20613247cb50c2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 14:58:37 +0300 Subject: [PATCH 115/176] adding replica-tests --- .github/workflows/replica-tests.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/replica-tests.yml diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml new file mode 100644 index 0000000..5fbff05 --- /dev/null +++ b/.github/workflows/replica-tests.yml @@ -0,0 +1,20 @@ +name: CI + +on: [pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + + - name: Set up Go 1.12 + uses: actions/setup-go@v1 + with: + version: 1.12 + id: go + + - name: Build + run: script/cibuild-gh-ost-replica-tests From 18612b661ac30e3ff45237ba104af251308a0a8f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 14:59:22 +0300 Subject: [PATCH 116/176] name of test --- .github/workflows/replica-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index 5fbff05..ac2535b 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -1,4 +1,4 @@ -name: CI +name: Replica tests on: [pull_request] From e618682ff9355093c6b7ce7625aec3057e352891 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:04:04 +0300 Subject: [PATCH 117/176] go1.12 --- .github/workflows/replica-tests.yml | 4 ++-- script/ensure-go-installed | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index ac2535b..d775131 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -1,4 +1,4 @@ -name: Replica tests +name: migration tests on: [pull_request] @@ -16,5 +16,5 @@ jobs: version: 1.12 id: go - - name: Build + - name: migration tests run: script/cibuild-gh-ost-replica-tests diff --git a/script/ensure-go-installed b/script/ensure-go-installed index a511e0e..c9c2ae8 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,20 +1,20 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.9.2 -SUPPORTED_GO_VERSIONS='go1.[89]' +PREFERRED_GO_VERSION=go1.12.6 +SUPPORTED_GO_VERSIONS='go1.1[234]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=8b4f6ae6deae1150d2e341d02c247fd18a99af387516540eeb84702ffd76d3a1 +GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815 GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz -GO_PKG_LINUX_SHA=de874549d9a8d8d8062be05808509c09a88a248e77ec14eb77453530829ac02b +GO_PKG_LINUX_SHA=dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" cd $ROOTDIR # If Go isn't installed globally, setup environment variables for local install. if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" ]; then - GODIR="$ROOTDIR/.vendor/go19" + GODIR="$ROOTDIR/.vendor/golocal" if [ $(uname -s) = "Darwin" ]; then export GOROOT="$GODIR/usr/local/go" @@ -32,12 +32,12 @@ if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" cd "$GODIR"; if [ $(uname -s) = "Darwin" ]; then - curl -L -O https://storage.googleapis.com/golang/$GO_PKG_DARWIN + curl -L -O https://dl.google.com/go/$GO_PKG_DARWIN shasum -a256 $GO_PKG_DARWIN | grep $GO_PKG_DARWIN_SHA xar -xf $GO_PKG_DARWIN cpio -i < com.googlecode.go.pkg/Payload else - curl -L -O https://storage.googleapis.com/golang/$GO_PKG_LINUX + curl -L -O https://dl.google.com/go/$GO_PKG_LINUX shasum -a256 $GO_PKG_LINUX | grep $GO_PKG_LINUX_SHA tar xf $GO_PKG_LINUX fi From b881bff1e8f3801ed00a727bb9bafe42c7299c46 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:06:03 +0300 Subject: [PATCH 118/176] fixed linter issue --- go/logic/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/server.go b/go/logic/server.go index 774c4ab..4c1f581 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -144,7 +144,7 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr switch command { case "help": { - fmt.Fprintln(writer, `available commands: + fmt.Fprint(writer, `available commands: status # Print a detailed status message sup # Print a short status message coordinates # Print the currently inspected coordinates From 106ca5df3caa3a9c972dbc69cf966eb583e72406 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:12:14 +0300 Subject: [PATCH 119/176] debug info --- script/cibuild-gh-ost-replica-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index ab8c689..c312aa9 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -44,9 +44,11 @@ test_mysql_version() { sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master + cat gh-ost-ci-env/bin/gh-ost-test-mysql-master sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica + cat gh-ost-ci-env/bin/gh-ost-test-mysql-replica export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" From 1305ff6e010e40631fb887c3e317acedb7c09af5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:17:48 +0300 Subject: [PATCH 120/176] debug info --- localtests/test.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/localtests/test.sh b/localtests/test.sh index 1078965..0a913ae 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -50,6 +50,22 @@ verify_master_and_replica() { original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" echo "sql_mode on master is ${original_sql_mode}" + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss + gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss + sleep 1 + if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then echo "Cannot verify gh-ost-test-mysql-replica" exit 1 From 90ab2368dd247da73d1cc58c726fc7fc089e86ed Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:21:04 +0300 Subject: [PATCH 121/176] graceful sleep; remove extra debug info --- localtests/test.sh | 17 ++--------------- script/cibuild-gh-ost-replica-tests | 2 -- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/localtests/test.sh b/localtests/test.sh index 0a913ae..d4b3f17 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -50,21 +50,8 @@ verify_master_and_replica() { original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" echo "sql_mode on master is ${original_sql_mode}" - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 - gh-ost-test-mysql-replica -uroot -proot -e "select 1 as root" -ss - gh-ost-test-mysql-replica -e "select 1 as msandbox" -ss - sleep 1 + echo "Gracefully sleeping for 3 seconds while replica is setting up..." + sleep 3 if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then echo "Cannot verify gh-ost-test-mysql-replica" diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c312aa9..ab8c689 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -44,11 +44,9 @@ test_mysql_version() { sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master - cat gh-ost-ci-env/bin/gh-ost-test-mysql-master sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica - cat gh-ost-ci-env/bin/gh-ost-test-mysql-replica export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" From 17e4fe80175c1d00b2665017d07a3571619814d5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:26:29 +0300 Subject: [PATCH 122/176] TravisCI updated to go1.12 ; but we are likely to remove TravisCI anyhow --- .travis.yml | 3 +-- README.md | 2 +- build.sh | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6b68f3..1944f00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,7 @@ language: go go: - - "1.9" - - "1.10" + - "1.12.x" os: - linux diff --git a/README.md b/README.md index 008c65b..cd6f592 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d [Download latest release here](https://github.com/github/gh-ost/releases/latest) -`gh-ost` is a Go project; it is built with Go `1.9` and above. To build on your own, use either: +`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either: - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary. - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost` diff --git a/build.sh b/build.sh index df9ac7d..8b6925d 100755 --- a/build.sh +++ b/build.sh @@ -20,9 +20,9 @@ function build { - if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then - echo "go version is too low. Must use 1.9 or above" - exit 1 + if ! go version | egrep -q 'go(1\.1[234])' ; then + echo "go version must be 1.12 or above" + ok=1 fi echo "Building ${osname} binary" From e1c8df744788721ef7b1def8c75b4981f686f0bf Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 11 Aug 2019 15:29:28 +0300 Subject: [PATCH 123/176] travis: declaring mysql service --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1944f00..c1c1891 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ go: os: - linux +services: + - mysql + env: - MYSQL_USER=root - CURRENT_CI_ENV=travis From 143abf05b0f85080cc1e27a6c5f2ed5f1538cef3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 12 Aug 2019 08:03:17 +0300 Subject: [PATCH 124/176] fail when go is < 1.12 --- build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 8b6925d..b7f3a54 100755 --- a/build.sh +++ b/build.sh @@ -18,11 +18,9 @@ function build { GOOS=$3 GOARCH=$4 - - if ! go version | egrep -q 'go(1\.1[234])' ; then echo "go version must be 1.12 or above" - ok=1 + exit 1 fi echo "Building ${osname} binary" From 3362c5d32254bd78e0bfcf305c77cff46e62983c Mon Sep 17 00:00:00 2001 From: Gillian Gunson Date: Mon, 12 Aug 2019 12:40:31 -0700 Subject: [PATCH 125/176] Changing panic message --- go/logic/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/server.go b/go/logic/server.go index 774c4ab..e1919ab 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -341,7 +341,7 @@ help # This message err := fmt.Errorf("User commanded 'panic' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName) return NoPrintStatusRule, err } - err := fmt.Errorf("User commanded 'panic'. I will now panic, without cleanup. PANIC!") + err := fmt.Errorf("User commanded 'panic'. The migration will be aborted without cleanup. Please drop the gh-ost tables before trying again.") this.migrationContext.PanicAbort <- err return NoPrintStatusRule, err } From 6cc1a1a34e27a301fb755ff21646acc9f3f7cdf0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 15 Aug 2019 11:10:48 +0300 Subject: [PATCH 126/176] CI: adapting to internal builds system --- Dockerfile.test | 11 ++++ docker-compose.yml | 7 +++ script/bootstrap | 1 + script/build-deploy-tarball | 35 ++++++++++++ script/cibuild | 16 +----- script/cibuild-gh-ost-build-deploy-tarball | 66 +++++++++++++--------- script/test | 17 ++++++ 7 files changed, 110 insertions(+), 43 deletions(-) create mode 100644 Dockerfile.test create mode 100644 docker-compose.yml create mode 100644 script/build-deploy-tarball create mode 100644 script/test diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..6abc4a0 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,11 @@ +FROM golang:1.12.1 +LABEL maintainer="github@github.com" + +RUN apt-get update +RUN apt-get install -y lsb-release +RUN rm -rf /var/lib/apt/lists/* + +COPY . /go/src/github.com/github/gh-ost +WORKDIR /go/src/github.com/github/gh-ost + +CMD ["script/test"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8a2bd2e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: "3.5" +services: + app: + image: app + build: + context: . + dockerfile: Dockerfile.test diff --git a/script/bootstrap b/script/bootstrap index 6ac885b..573313a 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -4,6 +4,7 @@ set -e # Make sure we have the version of Go we want to depend on, either from the # system or one we grab ourselves. +# If executing from within Dockerfile then this assumption is inherently true, since we use a `golang` docker image. . script/ensure-go-installed # Since we want to be able to build this outside of GOPATH, we set it diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball new file mode 100644 index 0000000..d36b6c9 --- /dev/null +++ b/script/build-deploy-tarball @@ -0,0 +1,35 @@ +#!/bin/sh + +set -e + +script/build + +# Get a fresh directory and make sure to delete it afterwards +build_dir=tmp/build +rm -rf $build_dir +mkdir -p $build_dir +trap "rm -rf $build_dir" EXIT + +commit_sha=$(git rev-parse HEAD) + +if [ $(uname -s) = "Darwin" ]; then + build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" +else + build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" +fi + +tarball=$build_dir/${commit_sha}-${build_arch}.tar + +# Create the tarball +tar cvf $tarball --mode="ugo=rx" bin/ + +# Compress it and copy it to the directory for the CI to upload it +gzip $tarball +mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost +cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ + +### HACK HACK HACK ### +# blame @carlosmn and @mattr- +# Allow builds on stretch to also be used for jessie +jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-stretch-/-jessie-/) +cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" diff --git a/script/cibuild b/script/cibuild index 7e757b5..e609b7a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,17 +1,3 @@ #!/bin/bash -set -e - -. script/bootstrap - -echo "Verifying code is formatted via 'gofmt -s -w go/'" -gofmt -s -w go/ -git diff --exit-code --quiet - -echo "Building" -script/build - -cd .gopath/src/github.com/github/gh-ost - -echo "Running unit tests" -go test ./go/... +script/test diff --git a/script/cibuild-gh-ost-build-deploy-tarball b/script/cibuild-gh-ost-build-deploy-tarball index 692b42b..a852ad6 100755 --- a/script/cibuild-gh-ost-build-deploy-tarball +++ b/script/cibuild-gh-ost-build-deploy-tarball @@ -1,37 +1,47 @@ -#!/bin/sh +#!/bin/bash -set -e +output_fold() { + # Exit early if no label provided + if [ -z "$1" ]; then + echo "output_fold(): requires a label argument." + return + fi -script/cibuild + exit_value=0 # exit_value is used to record exit status of the given command + label=$1 # human-readable label describing what's being folded up + shift 1 # having retrieved the output_fold()-specific arguments, strip them off $@ -# Get a fresh directory and make sure to delete it afterwards -build_dir=tmp/build -rm -rf $build_dir -mkdir -p $build_dir -trap "rm -rf $build_dir" EXIT + # Only echo the tags when in CI_MODE + if [ "$CI_MODE" ]; then + echo "%%%FOLD {$label}%%%" + fi -commit_sha=$(git rev-parse HEAD) + # run the remaining arguments. If the command exits non-0, the `||` will + # prevent the `-e` flag from seeing the failure exit code, and we'll see + # the second echo execute + "$@" || exit_value=$? -if [ $(uname -s) = "Darwin" ]; then - build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" -else - build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" -fi + # Only echo the tags when in CI_MODE + if [ "$CI_MODE" ]; then + echo "%%%END FOLD%%%" + fi -tarball=$build_dir/${commit_sha}-${build_arch}.tar + # preserve the exit code from the subcommand. + return $exit_value +} -# Create the tarball -tar cvf $tarball --mode="ugo=rx" bin/ +function cleanup() { + echo + echo "%%%FOLD {Shutting down services...}%%%" + docker-compose down + echo "%%%END FOLD%%%" +} -# Compress it and copy it to the directory for the CI to upload it -gzip $tarball -mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost -cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ +trap cleanup EXIT -### HACK HACK HACK ### -# Blame @carlosmn. In the good way. -# We don't have any jessie machines for building, but a pure-Go binary depends -# on a version of libc and ld which are widely available, so we can copy the -# tarball over with jessie in its name so we can deploy it on jessie machines. -jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-precise-/-jessie-/) -cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" +export CI_MODE=true + +output_fold "Bootstrapping container..." docker-compose build +output_fold "Running tests..." docker-compose run --rm app + +docker-compose run -e BUILD_ARTIFACT_DIR=$BUILD_ARTIFACT_DIR -v $BUILD_ARTIFACT_DIR:$BUILD_ARTIFACT_DIR app script/build-deploy-tarball diff --git a/script/test b/script/test new file mode 100644 index 0000000..7e757b5 --- /dev/null +++ b/script/test @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +. script/bootstrap + +echo "Verifying code is formatted via 'gofmt -s -w go/'" +gofmt -s -w go/ +git diff --exit-code --quiet + +echo "Building" +script/build + +cd .gopath/src/github.com/github/gh-ost + +echo "Running unit tests" +go test ./go/... From 36b0adb1b8a06aa490f00518348e5c4dcf37ce42 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 15 Aug 2019 11:14:10 +0300 Subject: [PATCH 127/176] executable scripts --- script/build-deploy-tarball | 0 script/test | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 script/build-deploy-tarball mode change 100644 => 100755 script/test diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball old mode 100644 new mode 100755 diff --git a/script/test b/script/test old mode 100644 new mode 100755 From 78698899880b45c6e93a715b190317b55d07eabc Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 6 Oct 2019 17:08:35 +0300 Subject: [PATCH 128/176] context, status and hooks: progressPct and CurrentLag --- go/base/context.go | 5 +++++ go/logic/hooks.go | 3 +++ go/logic/migrator.go | 7 +++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index f12cd43..84e6264 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -174,6 +174,7 @@ type MigrationContext struct { pointOfInterestTime time.Time pointOfInterestTimeMutex *sync.Mutex CurrentLag int64 + CurrentProgress uint64 // math.Float64bits([f=0..100]) ThrottleHTTPStatusCode int64 controlReplicasLagResult mysql.ReplicationLagResult TotalRowsCopied int64 @@ -428,6 +429,10 @@ func (this *MigrationContext) MarkRowCopyEndTime() { this.RowCopyEndTime = time.Now() } +func (this *MigrationContext) GetCurrentLagDuration() time.Duration { + return time.Duration(atomic.LoadInt64(&this.CurrentLag)) +} + // GetTotalRowsCopied returns the accurate number of rows being copied (affected) // This is not exactly the same as the rows being iterated via chunks, but potentially close enough func (this *MigrationContext) GetTotalRowsCopied() int64 { diff --git a/go/logic/hooks.go b/go/logic/hooks.go index be11130..b56b0b4 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -8,6 +8,7 @@ package logic import ( "fmt" + "math" "os" "os/exec" "path/filepath" @@ -63,6 +64,8 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.GetApplierHostname())) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) + env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", math.Float64frombits(atomic.LoadUint64(&this.migrationContext.CurrentProgress)))) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 86de5ac..985bef1 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -895,6 +895,8 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { } else { progressPct = 100.0 * float64(totalRowsCopied) / float64(rowsEstimate) } + // we take the opportunity to update migration context with progressPct + atomic.StoreUint64(&this.migrationContext.CurrentProgress, math.Float64bits(progressPct)) // Before status, let's see if we should print a nice reminder for what exactly we're doing here. shouldPrintMigrationStatusHint := (elapsedSeconds%600 == 0) if rule == ForcePrintStatusAndHintRule { @@ -911,7 +913,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { eta := "N/A" if progressPct >= 100.0 { eta = "due" - } else if progressPct >= 1.0 { + } else if progressPct >= 0.1 { elapsedRowCopySeconds := this.migrationContext.ElapsedRowCopyTime().Seconds() totalExpectedSeconds := elapsedRowCopySeconds * float64(rowsEstimate) / float64(totalRowsCopied) etaSeconds = totalExpectedSeconds - elapsedRowCopySeconds @@ -958,12 +960,13 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+v, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied), len(this.applyEventsQueue), cap(this.applyEventsQueue), base.PrettifyDurationOutput(elapsedTime), base.PrettifyDurationOutput(this.migrationContext.ElapsedRowCopyTime()), currentBinlogCoordinates, + this.migrationContext.GetCurrentLagDuration().Seconds(), state, eta, ) From 8893b2207b4bbadedf80f428dc43de28810dc4b4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 6 Oct 2019 17:12:20 +0300 Subject: [PATCH 129/176] documenting new hook env variables --- doc/hooks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/hooks.md b/doc/hooks.md index 93f500f..4c49c85 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -65,6 +65,8 @@ The following variables are available on all hooks: - `GH_OST_ELAPSED_COPY_SECONDS` - row-copy time (excluding startup, row-count and postpone time) - `GH_OST_ESTIMATED_ROWS` - estimated total rows in table - `GH_OST_COPIED_ROWS` - number of rows copied by `gh-ost` +- `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server +- `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration - `GH_OST_MIGRATED_HOST` - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` From 271c7274a3dbac3f82c8939ccfbbc22ec8bcd964 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 7 Oct 2019 06:59:56 +0300 Subject: [PATCH 130/176] refactor progressPct into migrationContext --- go/base/context.go | 13 ++++++++++++- go/logic/hooks.go | 3 +-- go/logic/migrator.go | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 84e6264..5ebf092 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -7,6 +7,7 @@ package base import ( "fmt" + "math" "os" "regexp" "strings" @@ -174,7 +175,7 @@ type MigrationContext struct { pointOfInterestTime time.Time pointOfInterestTimeMutex *sync.Mutex CurrentLag int64 - CurrentProgress uint64 // math.Float64bits([f=0..100]) + currentProgress uint64 ThrottleHTTPStatusCode int64 controlReplicasLagResult mysql.ReplicationLagResult TotalRowsCopied int64 @@ -433,6 +434,16 @@ func (this *MigrationContext) GetCurrentLagDuration() time.Duration { return time.Duration(atomic.LoadInt64(&this.CurrentLag)) } +func (this *MigrationContext) GetProgressPct() float64 { + return math.Float64frombits(atomic.LoadUint64(&this.currentProgress)) +} + +func (this *MigrationContext) SetProgressPct(progressPct float64) { + atomic.StoreUint64(&this.currentProgress, math.Float64bits(progressPct)) +} + +// math.Float64bits([f=0..100]) + // GetTotalRowsCopied returns the accurate number of rows being copied (affected) // This is not exactly the same as the rows being iterated via chunks, but potentially close enough func (this *MigrationContext) GetTotalRowsCopied() int64 { diff --git a/go/logic/hooks.go b/go/logic/hooks.go index b56b0b4..fa5011e 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -8,7 +8,6 @@ package logic import ( "fmt" - "math" "os" "os/exec" "path/filepath" @@ -65,7 +64,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) - env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", math.Float64frombits(atomic.LoadUint64(&this.migrationContext.CurrentProgress)))) + env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 985bef1..1446a65 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -896,7 +896,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { progressPct = 100.0 * float64(totalRowsCopied) / float64(rowsEstimate) } // we take the opportunity to update migration context with progressPct - atomic.StoreUint64(&this.migrationContext.CurrentProgress, math.Float64bits(progressPct)) + this.migrationContext.SetProgressPct(progressPct) // Before status, let's see if we should print a nice reminder for what exactly we're doing here. shouldPrintMigrationStatusHint := (elapsedSeconds%600 == 0) if rule == ForcePrintStatusAndHintRule { From 21e7ec6b7c005d713eae354f76f6791a9f539e50 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 7 Oct 2019 07:06:11 +0300 Subject: [PATCH 131/176] lag: %+vs notation, e.g. 1s --- go/logic/migrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 1446a65..b0bec68 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -960,7 +960,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+v, State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+vs, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied), len(this.applyEventsQueue), cap(this.applyEventsQueue), From f4676bf463e39a983803c55fe8d790f0ce858951 Mon Sep 17 00:00:00 2001 From: Abeyu M Date: Mon, 7 Oct 2019 11:10:36 -0400 Subject: [PATCH 132/176] implement a logging interface --- go/base/context.go | 21 ++++++ go/base/default_logger.go | 73 +++++++++++++++++++ go/base/utils.go | 3 +- go/cmd/gh-ost/main.go | 64 ++++++++--------- go/logic/applier.go | 129 +++++++++++++++++----------------- go/logic/inspect.go | 71 +++++++++---------- go/logic/migrator.go | 144 +++++++++++++++++++------------------- go/logic/server.go | 13 ++-- go/logic/streamer.go | 9 ++- go/logic/throttler.go | 13 ++-- 10 files changed, 313 insertions(+), 227 deletions(-) create mode 100644 go/base/default_logger.go diff --git a/go/base/context.go b/go/base/context.go index f12cd43..a4318b9 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -7,6 +7,7 @@ package base import ( "fmt" + "github.com/outbrain/golib/log" "os" "regexp" "strings" @@ -214,6 +215,25 @@ type MigrationContext struct { ForceTmpTableName string recentBinlogCoordinates mysql.BinlogCoordinates + + Log Logger +} + +type Logger interface { + Debug(args ...interface{}) + Debugf(format string, args ...interface{}) + Info(args ...interface{}) + Infof(format string, args ...interface{}) + Warning(args ...interface{}) error + Warningf(format string, args ...interface{}) error + Error(args ...interface{}) error + Errorf(format string, args ...interface{}) error + Errore(err error) error + Fatal(args ...interface{}) error + Fatalf(format string, args ...interface{}) error + Fatale(err error) error + SetLevel(level log.LogLevel) + SetPrintStackTrace(printStackTraceFlag bool) } type ContextConfig struct { @@ -248,6 +268,7 @@ func NewMigrationContext() *MigrationContext { pointOfInterestTimeMutex: &sync.Mutex{}, ColumnRenameMap: make(map[string]string), PanicAbort: make(chan error), + Log: NewDefaultLogger(), } } diff --git a/go/base/default_logger.go b/go/base/default_logger.go new file mode 100644 index 0000000..be6b1f2 --- /dev/null +++ b/go/base/default_logger.go @@ -0,0 +1,73 @@ +package base + +import ( + "github.com/outbrain/golib/log" +) + +type simpleLogger struct{} + +func NewDefaultLogger() *simpleLogger { + return &simpleLogger{} +} + +func (*simpleLogger) Debug(args ...interface{}) { + log.Debug(args[0].(string), args[1:]) + return +} + +func (*simpleLogger) Debugf(format string, args ...interface{}) { + log.Debugf(format, args...) + return +} + +func (*simpleLogger) Info(args ...interface{}) { + log.Info(args[0].(string), args[1:]) + return +} + +func (*simpleLogger) Infof(format string, args ...interface{}) { + log.Infof(format, args...) + return +} + +func (*simpleLogger) Warning(args ...interface{}) error { + return log.Warning(args[0].(string), args[1:]) +} + +func (*simpleLogger) Warningf(format string, args ...interface{}) error { + return log.Warningf(format, args...) +} + +func (*simpleLogger) Error(args ...interface{}) error { + return log.Error(args[0].(string), args[1:]) +} + +func (*simpleLogger) Errorf(format string, args ...interface{}) error { + return log.Errorf(format, args...) +} + +func (*simpleLogger) Errore(err error) error { + return log.Errore(err) +} + +func (*simpleLogger) Fatal(args ...interface{}) error { + return log.Fatal(args[0].(string), args[1:]) +} + +func (*simpleLogger) Fatalf(format string, args ...interface{}) error { + return log.Fatalf(format, args...) +} + +func (*simpleLogger) Fatale(err error) error { + return log.Fatale(err) +} + +func (*simpleLogger) SetLevel(level log.LogLevel) { + log.SetLevel(level) + return +} + +func (*simpleLogger) SetPrintStackTrace(printStackTraceFlag bool) { + log.SetPrintStackTrace(printStackTraceFlag) + return +} diff --git a/go/base/utils.go b/go/base/utils.go index 99536f8..1476a21 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -14,7 +14,6 @@ import ( gosql "database/sql" "github.com/github/gh-ost/go/mysql" - "github.com/outbrain/golib/log" ) var ( @@ -86,7 +85,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, } if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) { - log.Infof("connection validated on %+v", connectionConfig.Key) + migrationContext.Log.Infof("connection validated on %+v", connectionConfig.Key) return version, nil } else if extraPort == 0 { return "", fmt.Errorf("Unexpected database port reported: %+v", port) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..6773c08 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -31,7 +31,7 @@ func acceptSignals(migrationContext *base.MigrationContext) { for sig := range c { switch sig { case syscall.SIGHUP: - log.Infof("Received SIGHUP. Reloading configuration") + migrationContext.Log.Infof("Received SIGHUP. Reloading configuration") if err := migrationContext.ReadConfigFile(); err != nil { log.Errore(err) } else { @@ -156,69 +156,69 @@ func main() { return } - log.SetLevel(log.ERROR) + migrationContext.Log.SetLevel(log.ERROR) if *verbose { - log.SetLevel(log.INFO) + migrationContext.Log.SetLevel(log.INFO) } if *debug { - log.SetLevel(log.DEBUG) + migrationContext.Log.SetLevel(log.DEBUG) } if *stack { - log.SetPrintStackTrace(*stack) + migrationContext.Log.SetPrintStackTrace(*stack) } if *quiet { // Override!! - log.SetLevel(log.ERROR) + migrationContext.Log.SetLevel(log.ERROR) } if migrationContext.DatabaseName == "" { - log.Fatalf("--database must be provided and database name must not be empty") + migrationContext.Log.Fatalf("--database must be provided and database name must not be empty") } if migrationContext.OriginalTableName == "" { - log.Fatalf("--table must be provided and table name must not be empty") + migrationContext.Log.Fatalf("--table must be provided and table name must not be empty") } if migrationContext.AlterStatement == "" { - log.Fatalf("--alter must be provided and statement must not be empty") + migrationContext.Log.Fatalf("--alter must be provided and statement must not be empty") } migrationContext.Noop = !(*executeFlag) if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica { - log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") } if migrationContext.AllowedRunningOnMaster && migrationContext.MigrateOnReplica { - log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive") } if migrationContext.MigrateOnReplica && migrationContext.TestOnReplica { - log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive") + migrationContext.Log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive") } if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR { - log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") + migrationContext.Log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") } if migrationContext.TestOnReplicaSkipReplicaStop { if !migrationContext.TestOnReplica { - log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") + migrationContext.Log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") } - log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") + migrationContext.Log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") } if migrationContext.CliMasterUser != "" && migrationContext.AssumeMasterHostname == "" { - log.Fatalf("--master-user requires --assume-master-host") + migrationContext.Log.Fatalf("--master-user requires --assume-master-host") } if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" { - log.Fatalf("--master-password requires --assume-master-host") + migrationContext.Log.Fatalf("--master-password requires --assume-master-host") } if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-ca requires --ssl") + migrationContext.Log.Fatalf("--ssl-ca requires --ssl") } if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-cert requires --ssl") + migrationContext.Log.Fatalf("--ssl-cert requires --ssl") } if migrationContext.TLSKey != "" && !migrationContext.UseTLS { - log.Fatalf("--ssl-key requires --ssl") + migrationContext.Log.Fatalf("--ssl-key requires --ssl") } if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS { - log.Fatalf("--ssl-allow-insecure requires --ssl") + migrationContext.Log.Fatalf("--ssl-allow-insecure requires --ssl") } if *replicationLagQuery != "" { - log.Warningf("--replication-lag-query is deprecated") + migrationContext.Log.Warningf("--replication-lag-query is deprecated") } switch *cutOver { @@ -227,19 +227,19 @@ func main() { case "two-step": migrationContext.CutOverType = base.CutOverTwoStep default: - log.Fatalf("Unknown cut-over: %s", *cutOver) + migrationContext.Log.Fatalf("Unknown cut-over: %s", *cutOver) } if err := migrationContext.ReadConfigFile(); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadThrottleControlReplicaKeys(*throttleControlReplicas); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadMaxLoad(*maxLoad); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.ReadCriticalLoad(*criticalLoad); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if migrationContext.ServeSocketFile == "" { migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName) @@ -248,7 +248,7 @@ func main() { fmt.Println("Password:") bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) if err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } migrationContext.CliPassword = string(bytePassword) } @@ -262,13 +262,13 @@ func main() { migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() if err := migrationContext.SetupTLS(); err != nil { - log.Fatale(err) + migrationContext.Log.Fatale(err) } if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil { - log.Errore(err) + migrationContext.Log.Errore(err) } if err := migrationContext.SetExponentialBackoffMaxInterval(*exponentialBackoffMaxInterval); err != nil { - log.Errore(err) + migrationContext.Log.Errore(err) } log.Infof("starting gh-ost %+v", AppVersion) @@ -278,7 +278,7 @@ func main() { err := migrator.Migrate() if err != nil { migrator.ExecOnFailureHook() - log.Fatale(err) + migrationContext.Log.Fatale(err) } fmt.Fprintf(os.Stdout, "# Done\n") } diff --git a/go/logic/applier.go b/go/logic/applier.go index 5fb795b..926023f 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -16,7 +16,6 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -99,7 +98,7 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.readTableColumns(); err != nil { return err } - log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion) + this.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion) return nil } @@ -110,13 +109,13 @@ func (this *Applier) validateAndReadTimeZone() error { return err } - log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone) + this.migrationContext.Log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone) return nil } // readTableColumns reads table columns on applier func (this *Applier) readTableColumns() (err error) { - log.Infof("Examining table structure on applier") + this.migrationContext.Log.Infof("Examining table structure on applier") this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) if err != nil { return err @@ -157,7 +156,7 @@ func (this *Applier) ValidateOrDropExistingTables() error { } } if len(this.migrationContext.GetOldTableName()) > mysql.MaxTableNameLength { - log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength) + this.migrationContext.Log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength) } if this.tableExists(this.migrationContext.GetOldTableName()) { @@ -175,14 +174,14 @@ func (this *Applier) CreateGhostTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Creating ghost table %s.%s", + this.migrationContext.Log.Infof("Creating ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Ghost table created") + this.migrationContext.Log.Infof("Ghost table created") return nil } @@ -193,15 +192,15 @@ func (this *Applier) AlterGhost() error { sql.EscapeName(this.migrationContext.GetGhostTableName()), this.migrationContext.AlterStatement, ) - log.Infof("Altering ghost table %s.%s", + this.migrationContext.Log.Infof("Altering ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) - log.Debugf("ALTER statement: %s", query) + this.migrationContext.Log.Debugf("ALTER statement: %s", query) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Ghost table altered") + this.migrationContext.Log.Infof("Ghost table altered") return nil } @@ -222,14 +221,14 @@ func (this *Applier) CreateChangelogTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetChangelogTableName()), ) - log.Infof("Creating changelog table %s.%s", + this.migrationContext.Log.Infof("Creating changelog table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetChangelogTableName()), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Changelog table created") + this.migrationContext.Log.Infof("Changelog table created") return nil } @@ -239,14 +238,14 @@ func (this *Applier) dropTable(tableName string) error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) - log.Infof("Dropping table %s.%s", + this.migrationContext.Log.Infof("Dropping table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Table dropped") + this.migrationContext.Log.Infof("Table dropped") return nil } @@ -313,7 +312,7 @@ func (this *Applier) InitiateHeartbeat() { if _, err := this.WriteChangelog("heartbeat", time.Now().Format(time.RFC3339Nano)); err != nil { numSuccessiveFailures++ if numSuccessiveFailures > this.migrationContext.MaxRetries() { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } else { numSuccessiveFailures = 0 @@ -348,14 +347,14 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) { } var result int64 if err := this.db.QueryRow(throttleQuery).Scan(&result); err != nil { - return 0, log.Errore(err) + return 0, this.migrationContext.Log.Errore(err) } return result, nil } // ReadMigrationMinValues returns the minimum values to be iterated on rowcopy func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { - log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) + this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) if err != nil { return err @@ -370,13 +369,13 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error { return err } } - log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) + this.migrationContext.Log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues) return err } // ReadMigrationMaxValues returns the maximum values to be iterated on rowcopy func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { - log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) + this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns) if err != nil { return err @@ -391,7 +390,7 @@ func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error { return err } } - log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) + this.migrationContext.Log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues) return err } @@ -449,7 +448,7 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo return hasFurtherRange, nil } } - log.Debugf("Iteration complete: no further range to iterate") + this.migrationContext.Log.Debugf("Iteration complete: no further range to iterate") return hasFurtherRange, nil } @@ -507,7 +506,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } rowsAffected, _ = sqlResult.RowsAffected() duration = time.Since(startTime) - log.Debugf( + this.migrationContext.Log.Debugf( "Issued INSERT on range: [%s]..[%s]; iteration: %d; chunk-size: %d", this.migrationContext.MigrationIterationRangeMinValues, this.migrationContext.MigrationIterationRangeMaxValues, @@ -522,7 +521,7 @@ func (this *Applier) LockOriginalTable() error { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Locking %s.%s", + this.migrationContext.Log.Infof("Locking %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) @@ -530,18 +529,18 @@ func (this *Applier) LockOriginalTable() error { if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err } - log.Infof("Table locked") + this.migrationContext.Log.Infof("Table locked") return nil } // UnlockTables makes tea. No wait, it unlocks tables. func (this *Applier) UnlockTables() error { query := `unlock /* gh-ost */ tables` - log.Infof("Unlocking tables") + this.migrationContext.Log.Infof("Unlocking tables") if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err } - log.Infof("Tables unlocked") + this.migrationContext.Log.Infof("Tables unlocked") return nil } @@ -555,7 +554,7 @@ func (this *Applier) SwapTablesQuickAndBumpy() error { sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - log.Infof("Renaming original table") + this.migrationContext.Log.Infof("Renaming original table") this.migrationContext.RenameTablesStartTime = time.Now() if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil { return err @@ -565,13 +564,13 @@ func (this *Applier) SwapTablesQuickAndBumpy() error { sql.EscapeName(this.migrationContext.GetGhostTableName()), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming ghost table") + this.migrationContext.Log.Infof("Renaming ghost table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } this.migrationContext.RenameTablesEndTime = time.Now() - log.Infof("Tables renamed") + this.migrationContext.Log.Infof("Tables renamed") return nil } @@ -590,7 +589,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming back both tables") + this.migrationContext.Log.Infof("Renaming back both tables") if _, err := sqlutils.ExecNoPrepare(this.db, query); err == nil { return nil } @@ -601,7 +600,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), ) - log.Infof("Renaming back to ghost table") + this.migrationContext.Log.Infof("Renaming back to ghost table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { renameError = err } @@ -611,11 +610,11 @@ func (this *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Renaming back to original table") + this.migrationContext.Log.Infof("Renaming back to original table") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { renameError = err } - return log.Errore(renameError) + return this.migrationContext.Log.Errore(renameError) } // StopSlaveIOThread is applicable with --test-on-replica; it stops the IO thread, duh. @@ -623,44 +622,44 @@ func (this *Applier) RenameTablesRollback() (renameError error) { // and have them written to the binary log, so that we can then read them via streamer. func (this *Applier) StopSlaveIOThread() error { query := `stop /* gh-ost */ slave io_thread` - log.Infof("Stopping replication IO thread") + this.migrationContext.Log.Infof("Stopping replication IO thread") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Replication IO thread stopped") + this.migrationContext.Log.Infof("Replication IO thread stopped") return nil } // StartSlaveIOThread is applicable with --test-on-replica func (this *Applier) StartSlaveIOThread() error { query := `start /* gh-ost */ slave io_thread` - log.Infof("Starting replication IO thread") + this.migrationContext.Log.Infof("Starting replication IO thread") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Replication IO thread started") + this.migrationContext.Log.Infof("Replication IO thread started") return nil } // StartSlaveSQLThread is applicable with --test-on-replica func (this *Applier) StopSlaveSQLThread() error { query := `stop /* gh-ost */ slave sql_thread` - log.Infof("Verifying SQL thread is stopped") + this.migrationContext.Log.Infof("Verifying SQL thread is stopped") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("SQL thread stopped") + this.migrationContext.Log.Infof("SQL thread stopped") return nil } // StartSlaveSQLThread is applicable with --test-on-replica func (this *Applier) StartSlaveSQLThread() error { query := `start /* gh-ost */ slave sql_thread` - log.Infof("Verifying SQL thread is running") + this.migrationContext.Log.Infof("Verifying SQL thread is running") if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("SQL thread started") + this.migrationContext.Log.Infof("SQL thread started") return nil } @@ -677,7 +676,7 @@ func (this *Applier) StopReplication() error { if err != nil { return err } - log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates) + this.migrationContext.Log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates) return nil } @@ -689,7 +688,7 @@ func (this *Applier) StartReplication() error { if err := this.StartSlaveSQLThread(); err != nil { return err } - log.Infof("Replication started") + this.migrationContext.Log.Infof("Replication started") return nil } @@ -703,7 +702,7 @@ func (this *Applier) ExpectUsedLock(sessionId int64) error { var result int64 query := `select is_used_lock(?)` lockName := this.GetSessionLockName(sessionId) - log.Infof("Checking session lock: %s", lockName) + this.migrationContext.Log.Infof("Checking session lock: %s", lockName) if err := this.db.QueryRow(query, lockName).Scan(&result); err != nil || result != sessionId { return fmt.Errorf("Session lock %s expected to be found but wasn't", lockName) } @@ -738,7 +737,7 @@ func (this *Applier) ExpectProcess(sessionId int64, stateHint, infoHint string) // DropAtomicCutOverSentryTableIfExists checks if the "old" table name // happens to be a cut-over magic table; if so, it drops it. func (this *Applier) DropAtomicCutOverSentryTableIfExists() error { - log.Infof("Looking for magic cut-over table") + this.migrationContext.Log.Infof("Looking for magic cut-over table") tableName := this.migrationContext.GetOldTableName() rowMap := this.showTableStatus(tableName) if rowMap == nil { @@ -748,7 +747,7 @@ func (this *Applier) DropAtomicCutOverSentryTableIfExists() error { if rowMap["Comment"].String != atomicCutOverMagicHint { return fmt.Errorf("Expected magic comment on %s, did not find it", tableName) } - log.Infof("Dropping magic cut-over table") + this.migrationContext.Log.Infof("Dropping magic cut-over table") return this.dropTable(tableName) } @@ -768,14 +767,14 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error { this.migrationContext.TableEngine, atomicCutOverMagicHint, ) - log.Infof("Creating magic cut-over table %s.%s", + this.migrationContext.Log.Infof("Creating magic cut-over table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(tableName), ) if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { return err } - log.Infof("Magic cut-over table created") + this.migrationContext.Log.Infof("Magic cut-over table created") return nil } @@ -804,7 +803,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke lockResult := 0 query := `select get_lock(?, 0)` lockName := this.GetSessionLockName(sessionId) - log.Infof("Grabbing voluntary lock: %s", lockName) + this.migrationContext.Log.Infof("Grabbing voluntary lock: %s", lockName) if err := tx.QueryRow(query, lockName).Scan(&lockResult); err != nil || lockResult != 1 { err := fmt.Errorf("Unable to acquire lock %s", lockName) tableLocked <- err @@ -812,7 +811,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke } tableLockTimeoutSeconds := this.migrationContext.CutOverLockTimeoutSeconds * 2 - log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds) + this.migrationContext.Log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds) query = fmt.Sprintf(`set session lock_wait_timeout:=%d`, tableLockTimeoutSeconds) if _, err := tx.Exec(query); err != nil { tableLocked <- err @@ -830,7 +829,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - log.Infof("Locking %s.%s, %s.%s", + this.migrationContext.Log.Infof("Locking %s.%s, %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.DatabaseName), @@ -841,7 +840,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke tableLocked <- err return err } - log.Infof("Tables locked") + this.migrationContext.Log.Infof("Tables locked") tableLocked <- nil // No error. // From this point on, we are committed to UNLOCK TABLES. No matter what happens, @@ -850,22 +849,22 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke // The cut-over phase will proceed to apply remaining backlog onto ghost table, // and issue RENAME. We wait here until told to proceed. <-okToUnlockTable - log.Infof("Will now proceed to drop magic table and unlock tables") + this.migrationContext.Log.Infof("Will now proceed to drop magic table and unlock tables") // The magic table is here because we locked it. And we are the only ones allowed to drop it. // And in fact, we will: - log.Infof("Dropping magic cut-over table") + this.migrationContext.Log.Infof("Dropping magic cut-over table") query = fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) if _, err := tx.Exec(query); err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) // We DO NOT return here because we must `UNLOCK TABLES`! } // Tables still locked - log.Infof("Releasing lock from %s.%s, %s.%s", + this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), sql.EscapeName(this.migrationContext.DatabaseName), @@ -874,9 +873,9 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke query = `unlock tables` if _, err := tx.Exec(query); err != nil { tableUnlocked <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } - log.Infof("Tables unlocked") + this.migrationContext.Log.Infof("Tables unlocked") tableUnlocked <- nil return nil } @@ -898,7 +897,7 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed } sessionIdChan <- sessionId - log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds) + this.migrationContext.Log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds) query := fmt.Sprintf(`set session lock_wait_timeout:=%d`, this.migrationContext.CutOverLockTimeoutSeconds) if _, err := tx.Exec(query); err != nil { return err @@ -914,13 +913,13 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName), ) - log.Infof("Issuing and expecting this to block: %s", query) + this.migrationContext.Log.Infof("Issuing and expecting this to block: %s", query) if _, err := tx.Exec(query); err != nil { tablesRenamed <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } tablesRenamed <- nil - log.Infof("Tables renamed") + this.migrationContext.Log.Infof("Tables renamed") return nil } @@ -1026,19 +1025,19 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) }() if err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // no error atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, int64(len(dmlEvents))) if this.migrationContext.CountTableRows { atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, totalDelta) } - log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents)) + this.migrationContext.Log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents)) return nil } func (this *Applier) Teardown() { - log.Debugf("Tearing down...") + this.migrationContext.Log.Debugf("Tearing down...") this.db.Close() this.singletonDB.Close() atomic.StoreInt64(&this.finishedMigrating, 1) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31184b0..bc10830 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -17,7 +17,6 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -69,7 +68,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.applyBinlogFormat(); err != nil { return err } - log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion) + this.migrationContext.Log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion) return nil } @@ -137,14 +136,14 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { switch column.Type { case sql.FloatColumnType: { - log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) + this.migrationContext.Log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) uniqueKeyIsValid = false } case sql.JSONColumnType: { // Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code // will remain in place to potentially handle the future case where JSON is supported in indexes. - log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) + this.migrationContext.Log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) uniqueKeyIsValid = false } } @@ -157,17 +156,17 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { if this.migrationContext.UniqueKey == nil { return fmt.Errorf("No shared unique key can be found after ALTER! Bailing out") } - log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name) + this.migrationContext.Log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name) if this.migrationContext.UniqueKey.HasNullable { if this.migrationContext.NullableUniqueKeyAllowed { - log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey) + this.migrationContext.Log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey) } else { return fmt.Errorf("Chosen key (%s) has nullable columns. Bailing out. To force this operation to continue, supply --allow-nullable-unique-key flag. Only do so if you are certain there are no actual NULL values in this key. As long as there aren't, migration should be fine. NULL values in columns of this key will corrupt migration's data", this.migrationContext.UniqueKey) } } this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) - log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) + this.migrationContext.Log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) // By fact that a non-empty unique key exists we also know the shared columns are non-empty // This additional step looks at which columns are unsigned. We could have merged this within @@ -250,19 +249,19 @@ func (this *Inspector) validateGrants() error { this.migrationContext.HasSuperPrivilege = foundSuper if foundAll { - log.Infof("User has ALL privileges") + this.migrationContext.Log.Infof("User has ALL privileges") return nil } if foundSuper && foundReplicationSlave && foundDBAll { - log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) return nil } if foundReplicationClient && foundReplicationSlave && foundDBAll { - log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) return nil } - log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll) - return log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) + this.migrationContext.Log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll) + return this.migrationContext.Log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName)) } // restartReplication is required so that we are _certain_ the binlog format and @@ -270,7 +269,7 @@ func (this *Inspector) validateGrants() error { // It is entirely possible, for example, that the replication is using 'STATEMENT' // binlog format even as the variable says 'ROW' func (this *Inspector) restartReplication() error { - log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) masterKey, _ := mysql.GetMasterKeyFromSlaveStatus(this.connectionConfig) if masterKey == nil { @@ -289,7 +288,7 @@ func (this *Inspector) restartReplication() error { } time.Sleep(startSlavePostWaitMilliseconds) - log.Debugf("Replication restarted") + this.migrationContext.Log.Debugf("Replication restarted") return nil } @@ -309,7 +308,7 @@ func (this *Inspector) applyBinlogFormat() error { if err := this.restartReplication(); err != nil { return err } - log.Debugf("'ROW' binlog format applied") + this.migrationContext.Log.Debugf("'ROW' binlog format applied") return nil } // We already have RBR, no explicit switch @@ -347,7 +346,7 @@ func (this *Inspector) validateBinlogs() error { if countReplicas > 0 { return fmt.Errorf("%s:%d has %s binlog_format, but I'm too scared to change it to ROW because it has replicas. Bailing out", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) } - log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) + this.migrationContext.Log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat) } query = `select @@global.binlog_row_image` if err := this.db.QueryRow(query).Scan(&this.migrationContext.OriginalBinlogRowImage); err != nil { @@ -359,7 +358,7 @@ func (this *Inspector) validateBinlogs() error { return fmt.Errorf("%s:%d has '%s' binlog_row_image, and only 'FULL' is supported. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogRowImage) } - log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -372,12 +371,12 @@ func (this *Inspector) validateLogSlaveUpdates() error { } if logSlaveUpdates { - log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } if this.migrationContext.IsTungsten { - log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -386,7 +385,7 @@ func (this *Inspector) validateLogSlaveUpdates() error { } if this.migrationContext.InspectorIsAlsoApplier() { - log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) + this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port) return nil } @@ -413,17 +412,17 @@ func (this *Inspector) validateTable() error { return err } if !tableFound { - return log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine) - log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate) + this.migrationContext.Log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine) + this.migrationContext.Log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate) return nil } // validateTableForeignKeys makes sure no foreign keys exist on the migrated table func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error { if this.migrationContext.SkipForeignKeyChecks { - log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys") + this.migrationContext.Log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys") return nil } query := ` @@ -457,16 +456,16 @@ func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) erro return err } if numParentForeignKeys > 0 { - return log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } if numChildForeignKeys > 0 { if allowChildForeignKeys { - log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") + this.migrationContext.Log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") return nil } - return log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Debugf("Validated no foreign keys exist on table") + this.migrationContext.Log.Debugf("Validated no foreign keys exist on table") return nil } @@ -492,9 +491,9 @@ func (this *Inspector) validateTableTriggers() error { return err } if numTriggers > 0 { - return log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Debugf("Validated no triggers exist on table") + this.migrationContext.Log.Debugf("Validated no triggers exist on table") return nil } @@ -514,9 +513,9 @@ func (this *Inspector) estimateTableRowsViaExplain() error { return err } if !outputFound { - return log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + return this.migrationContext.Log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } - log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate) + this.migrationContext.Log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate) return nil } @@ -525,7 +524,7 @@ func (this *Inspector) CountTableRows() error { atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 1) defer atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 0) - log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") + this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 @@ -535,7 +534,7 @@ func (this *Inspector) CountTableRows() error { atomic.StoreInt64(&this.migrationContext.RowsEstimate, rowsEstimate) this.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate - log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) + this.migrationContext.Log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) return nil } @@ -663,7 +662,7 @@ func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](* if err != nil { return uniqueKeys, err } - log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys) + this.migrationContext.Log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys) return uniqueKeys, nil } @@ -753,7 +752,7 @@ func (this *Inspector) readChangelogState(hint string) (string, error) { } func (this *Inspector) getMasterConnectionConfig() (applierConfig *mysql.ConnectionConfig, err error) { - log.Infof("Recursively searching for replication master") + this.migrationContext.Log.Infof("Recursively searching for replication master") visitedKeys := mysql.NewInstanceKeyMap() return mysql.GetMasterConnectionConfigSafe(this.connectionConfig, visitedKeys, this.migrationContext.AllowedMasterMaster) } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 86de5ac..4002de8 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -18,8 +18,6 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - - "github.com/outbrain/golib/log" ) type ChangelogState string @@ -216,7 +214,7 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er } changelogStateString := dmlEvent.NewColumnValues.StringColumn(3) changelogState := ReadChangelogState(changelogStateString) - log.Infof("Intercepted changelog state %s", changelogState) + this.migrationContext.Log.Infof("Intercepted changelog state %s", changelogState) switch changelogState { case GhostTableMigrated: { @@ -242,14 +240,14 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er return fmt.Errorf("Unknown changelog state: %+v", changelogState) } } - log.Infof("Handled changelog state %s", changelogState) + this.migrationContext.Log.Infof("Handled changelog state %s", changelogState) return nil } // listenOnPanicAbort aborts on abort request func (this *Migrator) listenOnPanicAbort() { err := <-this.migrationContext.PanicAbort - log.Fatale(err) + this.migrationContext.Log.Fatale(err) } // validateStatement validates the `alter` statement meets criteria. @@ -265,7 +263,7 @@ func (this *Migrator) validateStatement() (err error) { if !this.migrationContext.ApproveRenamedColumns { return fmt.Errorf("gh-ost believes the ALTER statement renames columns, as follows: %v; as precaution, you are asked to confirm gh-ost is correct, and provide with `--approve-renamed-columns`, and we're all happy. Or you can skip renamed columns via `--skip-renamed-columns`, in which case column data may be lost", this.parser.GetNonTrivialRenames()) } - log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames()) + this.migrationContext.Log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames()) } this.migrationContext.DroppedColumnsMap = this.parser.DroppedColumnsMap() return nil @@ -277,7 +275,7 @@ func (this *Migrator) countTableRows() (err error) { return nil } if this.migrationContext.Noop { - log.Debugf("Noop operation; not really counting table rows") + this.migrationContext.Log.Debugf("Noop operation; not really counting table rows") return nil } @@ -292,7 +290,7 @@ func (this *Migrator) countTableRows() (err error) { } if this.migrationContext.ConcurrentCountTableRows { - log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on") + this.migrationContext.Log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on") go countRowsFunc() // and we ignore errors, because this turns to be a background job return nil @@ -304,9 +302,9 @@ func (this *Migrator) createFlagFiles() (err error) { if this.migrationContext.PostponeCutOverFlagFile != "" { if !base.FileExists(this.migrationContext.PostponeCutOverFlagFile) { if err := base.TouchFile(this.migrationContext.PostponeCutOverFlagFile); err != nil { - return log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error()) + return this.migrationContext.Log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error()) } - log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile) + this.migrationContext.Log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile) } } return nil @@ -314,7 +312,7 @@ func (this *Migrator) createFlagFiles() (err error) { // Migrate executes the complete migration logic. This is *the* major gh-ost function. func (this *Migrator) Migrate() (err error) { - log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) this.migrationContext.StartTime = time.Now() if this.migrationContext.Hostname, err = os.Hostname(); err != nil { return err @@ -353,9 +351,9 @@ func (this *Migrator) Migrate() (err error) { } initialLag, _ := this.inspector.getReplicationLag() - log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) + this.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag) <-this.ghostTableMigrated - log.Debugf("ghost table migrated") + this.migrationContext.Log.Debugf("ghost table migrated") // Yay! We now know the Ghost and Changelog tables are good to examine! // When running on replica, this means the replica has those tables. When running // on master this is always true, of course, and yet it also implies this knowledge @@ -393,9 +391,9 @@ func (this *Migrator) Migrate() (err error) { this.migrationContext.MarkRowCopyStartTime() go this.initiateStatus() - log.Debugf("Operating until row copy is complete") + this.migrationContext.Log.Debugf("Operating until row copy is complete") this.consumeRowCopyComplete() - log.Infof("Row copy complete") + this.migrationContext.Log.Infof("Row copy complete") if err := this.hooksExecutor.onRowCopyComplete(); err != nil { return err } @@ -421,7 +419,7 @@ func (this *Migrator) Migrate() (err error) { if err := this.hooksExecutor.onSuccess(); err != nil { return err } - log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -447,14 +445,14 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) { // and swap the tables. // The difference is that we will later swap the tables back. if err := this.hooksExecutor.onStartReplication(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if this.migrationContext.TestOnReplicaSkipReplicaStop { - log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.") + this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.") } else { - log.Debugf("testing on replica. Starting replication IO thread after cut-over failure") + this.migrationContext.Log.Debugf("testing on replica. Starting replication IO thread after cut-over failure") if err := this.retryOperation(this.applier.StartReplication); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } } @@ -465,16 +463,16 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) { // type (on replica? atomic? safe?) func (this *Migrator) cutOver() (err error) { if this.migrationContext.Noop { - log.Debugf("Noop operation; not really swapping tables") + this.migrationContext.Log.Debugf("Noop operation; not really swapping tables") return nil } this.migrationContext.MarkPointOfInterest() this.throttler.throttle(func() { - log.Debugf("throttling before swapping tables") + this.migrationContext.Log.Debugf("throttling before swapping tables") }) this.migrationContext.MarkPointOfInterest() - log.Debugf("checking for cut-over postpone") + this.migrationContext.Log.Debugf("checking for cut-over postpone") this.sleepWhileTrue( func() (bool, error) { if this.migrationContext.PostponeCutOverFlagFile == "" { @@ -499,7 +497,7 @@ func (this *Migrator) cutOver() (err error) { ) atomic.StoreInt64(&this.migrationContext.IsPostponingCutOver, 0) this.migrationContext.MarkPointOfInterest() - log.Debugf("checking for cut-over postpone: complete") + this.migrationContext.Log.Debugf("checking for cut-over postpone: complete") if this.migrationContext.TestOnReplica { // With `--test-on-replica` we stop replication thread, and then proceed to use @@ -510,9 +508,9 @@ func (this *Migrator) cutOver() (err error) { return err } if this.migrationContext.TestOnReplicaSkipReplicaStop { - log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") + this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") } else { - log.Debugf("testing on replica. Stopping replication IO thread") + this.migrationContext.Log.Debugf("testing on replica. Stopping replication IO thread") if err := this.retryOperation(this.applier.StopReplication); err != nil { return err } @@ -530,7 +528,7 @@ func (this *Migrator) cutOver() (err error) { this.handleCutOverResult(err) return err } - return log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType) + return this.migrationContext.Log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType) } // Inject the "AllEventsUpToLockProcessed" state hint, wait for it to appear in the binary logs, @@ -542,32 +540,32 @@ func (this *Migrator) waitForEventsUpToLock() (err error) { waitForEventsUpToLockStartTime := time.Now() allEventsUpToLockProcessedChallenge := fmt.Sprintf("%s:%d", string(AllEventsUpToLockProcessed), waitForEventsUpToLockStartTime.UnixNano()) - log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge) + this.migrationContext.Log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge) if _, err := this.applier.WriteChangelogState(allEventsUpToLockProcessedChallenge); err != nil { return err } - log.Infof("Waiting for events up to lock") + this.migrationContext.Log.Infof("Waiting for events up to lock") atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 1) for found := false; !found; { select { case <-timeout.C: { - return log.Errorf("Timeout while waiting for events up to lock") + return this.migrationContext.Log.Errorf("Timeout while waiting for events up to lock") } case state := <-this.allEventsUpToLockProcessed: { if state == allEventsUpToLockProcessedChallenge { - log.Infof("Waiting for events up to lock: got %s", state) + this.migrationContext.Log.Infof("Waiting for events up to lock: got %s", state) found = true } else { - log.Infof("Waiting for events up to lock: skipping %s", state) + this.migrationContext.Log.Infof("Waiting for events up to lock: skipping %s", state) } } } } waitForEventsUpToLockDuration := time.Since(waitForEventsUpToLockStartTime) - log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration) + this.migrationContext.Log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration) this.printStatus(ForcePrintStatusAndHintRule) return nil @@ -598,7 +596,7 @@ func (this *Migrator) cutOverTwoStep() (err error) { lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime) renameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.RenameTablesStartTime) - log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -620,18 +618,18 @@ func (this *Migrator) atomicCutOver() (err error) { tableUnlocked := make(chan error, 2) go func() { if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } }() if err := <-tableLocked; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } lockOriginalSessionId := <-lockOriginalSessionIdChan - log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId) + this.migrationContext.Log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId) // At this point we know the original table is locked. // We know any newly incoming DML on original table is blocked. if err := this.waitForEventsUpToLock(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // Step 2 @@ -649,7 +647,7 @@ func (this *Migrator) atomicCutOver() (err error) { } }() renameSessionId := <-renameSessionIdChan - log.Infof("Session renaming tables is %+v", renameSessionId) + this.migrationContext.Log.Infof("Session renaming tables is %+v", renameSessionId) waitForRename := func() error { if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 1 { @@ -666,13 +664,13 @@ func (this *Migrator) atomicCutOver() (err error) { return err } if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 0 { - log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)") + this.migrationContext.Log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)") } if err := this.applier.ExpectUsedLock(lockOriginalSessionId); err != nil { // Abort operation. Just make sure to drop the magic table. - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } - log.Infof("Connection holding lock on original table still exists") + this.migrationContext.Log.Infof("Connection holding lock on original table still exists") // Now that we've found the RENAME blocking, AND the locking connection still alive, // we know it is safe to proceed to release the lock @@ -681,16 +679,16 @@ func (this *Migrator) atomicCutOver() (err error) { // BAM! magic table dropped, original table lock is released // -> RENAME released -> queries on original are unblocked. if err := <-tableUnlocked; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if err := <-tablesRenamed; err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } this.migrationContext.RenameTablesEndTime = time.Now() // ooh nice! We're actually truly and thankfully done lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime) - log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) + this.migrationContext.Log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -736,7 +734,7 @@ func (this *Migrator) initiateInspector() (err error) { if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil { return err } - log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + this.migrationContext.Log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } else { // Forced master host. key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname) @@ -750,14 +748,14 @@ func (this *Migrator) initiateInspector() (err error) { if this.migrationContext.CliMasterPassword != "" { this.migrationContext.ApplierConnectionConfig.Password = this.migrationContext.CliMasterPassword } - log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + this.migrationContext.Log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } // validate configs if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica { if this.migrationContext.InspectorIsAlsoApplier() { return fmt.Errorf("Instructed to --test-on-replica or --migrate-on-replica, but the server we connect to doesn't seem to be a replica") } - log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", + this.migrationContext.Log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", *this.migrationContext.ApplierConnectionConfig.ImpliedKey, *this.migrationContext.InspectorConnectionConfig.ImpliedKey, ) this.migrationContext.ApplierConnectionConfig = this.migrationContext.InspectorConnectionConfig.Duplicate() @@ -995,12 +993,12 @@ func (this *Migrator) initiateStreaming() error { ) go func() { - log.Debugf("Beginning streaming") + this.migrationContext.Log.Debugf("Beginning streaming") err := this.eventsStreamer.StreamEvents(this.canStopStreaming) if err != nil { this.migrationContext.PanicAbort <- err } - log.Debugf("Done streaming") + this.migrationContext.Log.Debugf("Done streaming") }() go func() { @@ -1035,11 +1033,11 @@ func (this *Migrator) initiateThrottler() error { this.throttler = NewThrottler(this.migrationContext, this.applier, this.inspector) go this.throttler.initiateThrottlerCollection(this.firstThrottlingCollected) - log.Infof("Waiting for first throttle metrics to be collected") + this.migrationContext.Log.Infof("Waiting for first throttle metrics to be collected") <-this.firstThrottlingCollected // replication lag <-this.firstThrottlingCollected // HTTP status <-this.firstThrottlingCollected // other, general metrics - log.Infof("First throttle metrics collected") + this.migrationContext.Log.Infof("First throttle metrics collected") go this.throttler.initiateThrottlerChecks() return nil @@ -1054,16 +1052,16 @@ func (this *Migrator) initiateApplier() error { return err } if err := this.applier.CreateChangelogTable(); err != nil { - log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") + this.migrationContext.Log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") return err } if err := this.applier.CreateGhostTable(); err != nil { - log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") + this.migrationContext.Log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") return err } if err := this.applier.AlterGhost(); err != nil { - log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out") + this.migrationContext.Log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out") return err } @@ -1077,14 +1075,14 @@ func (this *Migrator) initiateApplier() error { func (this *Migrator) iterateChunks() error { terminateRowIteration := func(err error) error { this.rowCopyComplete <- err - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if this.migrationContext.Noop { - log.Debugf("Noop operation; not really copying data") + this.migrationContext.Log.Debugf("Noop operation; not really copying data") return terminateRowIteration(nil) } if this.migrationContext.MigrationRangeMinValues == nil { - log.Debugf("No rows found in table. Rowcopy will be implicitly empty") + this.migrationContext.Log.Debugf("No rows found in table. Rowcopy will be implicitly empty") return terminateRowIteration(nil) } @@ -1152,7 +1150,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error { if eventStruct.writeFunc != nil { if err := this.retryOperation(*eventStruct.writeFunc); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } return nil @@ -1186,13 +1184,13 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { return this.applier.ApplyDMLEventQueries(dmlEvents) } if err := this.retryOperation(applyEventFunc); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if nonDmlStructToApply != nil { // We pulled DML events from the queue, and then we hit a non-DML event. Wait! // We need to handle it! if err := handleNonDMLEventStruct(nonDmlStructToApply); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } } } @@ -1204,7 +1202,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { // Both event backlog and rowcopy events are polled; the backlog events have precedence. func (this *Migrator) executeWriteFuncs() error { if this.migrationContext.Noop { - log.Debugf("Noop operation; not really executing write funcs") + this.migrationContext.Log.Debugf("Noop operation; not really executing write funcs") return nil } for { @@ -1231,7 +1229,7 @@ func (this *Migrator) executeWriteFuncs() error { copyRowsStartTime := time.Now() // Retries are handled within the copyRowsFunc if err := copyRowsFunc(); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } if niceRatio := this.migrationContext.GetNiceRatio(); niceRatio > 0 { copyRowsDuration := time.Since(copyRowsStartTime) @@ -1244,7 +1242,7 @@ func (this *Migrator) executeWriteFuncs() error { { // Hmmmmm... nothing in the queue; no events, but also no row copy. // This is possible upon load. Let's just sleep it over. - log.Debugf("Getting nothing in the write queue. Sleeping...") + this.migrationContext.Log.Debugf("Getting nothing in the write queue. Sleeping...") time.Sleep(time.Second) } } @@ -1260,14 +1258,14 @@ func (this *Migrator) finalCleanup() error { if this.migrationContext.Noop { if createTableStatement, err := this.inspector.showCreateTable(this.migrationContext.GetGhostTableName()); err == nil { - log.Infof("New table structure follows") + this.migrationContext.Log.Infof("New table structure follows") fmt.Println(createTableStatement) } else { - log.Errore(err) + this.migrationContext.Log.Errore(err) } } if err := this.eventsStreamer.Close(); err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } if err := this.retryOperation(this.applier.DropChangelogTable); err != nil { @@ -1279,8 +1277,8 @@ func (this *Migrator) finalCleanup() error { } } else { if !this.migrationContext.Noop { - log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:") - log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName())) + this.migrationContext.Log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:") + this.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName())) } } if this.migrationContext.Noop { @@ -1296,22 +1294,22 @@ func (this *Migrator) teardown() { atomic.StoreInt64(&this.finishedMigrating, 1) if this.inspector != nil { - log.Infof("Tearing down inspector") + this.migrationContext.Log.Infof("Tearing down inspector") this.inspector.Teardown() } if this.applier != nil { - log.Infof("Tearing down applier") + this.migrationContext.Log.Infof("Tearing down applier") this.applier.Teardown() } if this.eventsStreamer != nil { - log.Infof("Tearing down streamer") + this.migrationContext.Log.Infof("Tearing down streamer") this.eventsStreamer.Teardown() } if this.throttler != nil { - log.Infof("Tearing down throttler") + this.migrationContext.Log.Infof("Tearing down throttler") this.throttler.Teardown() } } diff --git a/go/logic/server.go b/go/logic/server.go index 7af80b5..1606884 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -16,7 +16,6 @@ import ( "sync/atomic" "github.com/github/gh-ost/go/base" - "github.com/outbrain/golib/log" ) type printStatusFunc func(PrintStatusRule, io.Writer) @@ -49,12 +48,12 @@ func (this *Server) BindSocketFile() (err error) { if err != nil { return err } - log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile) + this.migrationContext.Log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile) return nil } func (this *Server) RemoveSocketFile() (err error) { - log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile) + this.migrationContext.Log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile) return os.Remove(this.migrationContext.ServeSocketFile) } @@ -66,7 +65,7 @@ func (this *Server) BindTCPPort() (err error) { if err != nil { return err } - log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort) + this.migrationContext.Log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort) return nil } @@ -76,7 +75,7 @@ func (this *Server) Serve() (err error) { for { conn, err := this.unixListener.Accept() if err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } go this.handleConnection(conn) } @@ -88,7 +87,7 @@ func (this *Server) Serve() (err error) { for { conn, err := this.tcpListener.Accept() if err != nil { - log.Errore(err) + this.migrationContext.Log.Errore(err) } go this.handleConnection(conn) } @@ -118,7 +117,7 @@ func (this *Server) onServerCommand(command string, writer *bufio.Writer) (err e } else { fmt.Fprintf(writer, "%s\n", err.Error()) } - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } // applyServerCommand parses and executes commands by user diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 25d9f53..5f11fd0 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -16,7 +16,6 @@ import ( "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" - "github.com/outbrain/golib/log" "github.com/outbrain/golib/sqlutils" ) @@ -160,7 +159,7 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error { if !foundMasterStatus { return fmt.Errorf("Got no results from SHOW MASTER STATUS. Bailing out") } - log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates) + this.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates) return nil } @@ -186,7 +185,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { return nil } - log.Infof("StreamEvents encountered unexpected error: %+v", err) + this.migrationContext.Log.Infof("StreamEvents encountered unexpected error: %+v", err) this.migrationContext.MarkPointOfInterest() time.Sleep(ReconnectStreamerSleepSeconds * time.Second) @@ -202,7 +201,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { // Reposition at same binlog file. lastAppliedRowsEventHint = this.binlogReader.LastAppliedRowsEventHint - log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint) + this.migrationContext.Log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint) if err := this.initBinlogReader(this.GetReconnectBinlogCoordinates()); err != nil { return err } @@ -213,7 +212,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error { func (this *EventsStreamer) Close() (err error) { err = this.binlogReader.Close() - log.Infof("Closed streamer connection. err=%+v", err) + this.migrationContext.Log.Infof("Closed streamer connection. err=%+v", err) return err } diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 6f9f4bb..1fe413c 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -15,7 +15,6 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" ) var ( @@ -120,7 +119,7 @@ func parseChangelogHeartbeat(heartbeatValue string) (lag time.Duration, err erro // parseChangelogHeartbeat parses a string timestamp and deduces replication lag func (this *Throttler) parseChangelogHeartbeat(heartbeatValue string) (err error) { if lag, err := parseChangelogHeartbeat(heartbeatValue); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) return nil @@ -142,13 +141,13 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo // This means we will always get a good heartbeat value. // When running on replica, we should instead check the `SHOW SLAVE STATUS` output. if lag, err := mysql.GetReplicationLagFromSlaveStatus(this.inspector.informationSchemaDb); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag)) } } else { if heartbeatValue, err := this.inspector.readChangelogState("heartbeat"); err != nil { - return log.Errore(err) + return this.migrationContext.Log.Errore(err) } else { this.parseChangelogHeartbeat(heartbeatValue) } @@ -330,7 +329,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error { hibernateDuration := time.Duration(this.migrationContext.CriticalLoadHibernateSeconds) * time.Second hibernateUntilTime := time.Now().Add(hibernateDuration) atomic.StoreInt64(&this.migrationContext.HibernateUntil, hibernateUntilTime.UnixNano()) - log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime) + this.migrationContext.Log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime) go func() { time.Sleep(hibernateDuration) this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(true, "leaving hibernation", base.LeavingHibernationThrottleReasonHint)) @@ -343,7 +342,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error { this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold) } if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 { - log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds) + this.migrationContext.Log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds) go func() { timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds)) <-timer.C @@ -461,6 +460,6 @@ func (this *Throttler) throttle(onThrottled func()) { } func (this *Throttler) Teardown() { - log.Debugf("Tearing down...") + this.migrationContext.Log.Debugf("Tearing down...") atomic.StoreInt64(&this.finishedMigrating, 1) } From 3ca5f89fa52aa76d4ef0725ddad1eb85e13dc5f7 Mon Sep 17 00:00:00 2001 From: Abeyu M Date: Tue, 8 Oct 2019 13:49:15 -0400 Subject: [PATCH 133/176] add migrationcontext to gomysql_reader --- go/binlog/gomysql_reader.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 41d1ead..bc80cb5 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -13,13 +13,13 @@ import ( "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/outbrain/golib/log" gomysql "github.com/siddontang/go-mysql/mysql" "github.com/siddontang/go-mysql/replication" "golang.org/x/net/context" ) type GoMySQLReader struct { + migrationContext *base.MigrationContext connectionConfig *mysql.ConnectionConfig binlogSyncer *replication.BinlogSyncer binlogStreamer *replication.BinlogStreamer @@ -30,6 +30,7 @@ type GoMySQLReader struct { func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *GoMySQLReader, err error) { binlogReader = &GoMySQLReader{ + migrationContext: migrationContext, connectionConfig: migrationContext.InspectorConnectionConfig, currentCoordinates: mysql.BinlogCoordinates{}, currentCoordinatesMutex: &sync.Mutex{}, @@ -57,11 +58,11 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go // ConnectBinlogStreamer func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordinates) (err error) { if coordinates.IsEmpty() { - return log.Errorf("Empty coordinates at ConnectBinlogStreamer()") + return this.migrationContext.Log.Errorf("Empty coordinates at ConnectBinlogStreamer()") } this.currentCoordinates = coordinates - log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) + this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates) // Start sync with specified binlog file and position this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{this.currentCoordinates.LogFile, uint32(this.currentCoordinates.LogPos)}) @@ -78,7 +79,7 @@ func (this *GoMySQLReader) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinate // StreamEvents func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error { if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) { - log.Debugf("Skipping handled query at %+v", this.currentCoordinates) + this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates) return nil } @@ -147,14 +148,14 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha defer this.currentCoordinatesMutex.Unlock() this.currentCoordinates.LogFile = string(rotateEvent.NextLogName) }() - log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName) + this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName) } else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok { if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil { return err } } } - log.Debugf("done streaming events") + this.migrationContext.Log.Debugf("done streaming events") return nil } From d0ce7c014ee7fd4f1692574527506c0f8ce9b258 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 23 Oct 2019 10:18:56 +0300 Subject: [PATCH 134/176] truncate lag digits --- go/logic/migrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index b0bec68..b1a238f 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -960,7 +960,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %+vs, State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied), len(this.applyEventsQueue), cap(this.applyEventsQueue), From 991cdc51f0f882a917d3b559945595f485a1d697 Mon Sep 17 00:00:00 2001 From: Mohit Maroliya <35539313+mohitm15@users.noreply.github.com> Date: Thu, 31 Oct 2019 19:28:54 +0530 Subject: [PATCH 135/176] Update command-line-flags.md --- doc/command-line-flags.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 61ffdbc..ee0051c 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -18,7 +18,7 @@ If, for some reason, you do not wish `gh-ost` to connect to a replica, you may c ### approve-renamed-columns -When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try an associate the old column name with new column name. Otherwise the new structure may also look like some column was dropped and another was added. +When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise the new structure may also look like some column was dropped and another was added. `gh-ost` will print out what it thinks the _rename_ implied, but will not issue the migration unless you provide with `--approve-renamed-columns`. @@ -161,7 +161,7 @@ List of metrics and threshold values; topping the threshold of any will cause th ### migrate-on-replica -Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but other issue no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. +Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but other issue will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. ### postpone-cut-over-flag-file From 859800079bfb73a3e86ee778d51bfcdc1095b739 Mon Sep 17 00:00:00 2001 From: Mohit Maroliya <35539313+mohitm15@users.noreply.github.com> Date: Thu, 31 Oct 2019 20:13:54 +0530 Subject: [PATCH 136/176] Update doc/command-line-flags.md Thank you for the suggestion @shlomi-noach Co-Authored-By: Shlomi Noach --- doc/command-line-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index ee0051c..629e9f9 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -161,7 +161,7 @@ List of metrics and threshold values; topping the threshold of any will cause th ### migrate-on-replica -Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but other issue will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. +Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but otherwise will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. ### postpone-cut-over-flag-file From 19215b268834c8d909f866f7561a89ebad0900b6 Mon Sep 17 00:00:00 2001 From: Tobias Johansson Date: Tue, 10 Dec 2019 13:57:18 +0100 Subject: [PATCH 137/176] Update rds docs --- doc/rds.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/rds.md b/doc/rds.md index 37d1d80..da59abb 100644 --- a/doc/rds.md +++ b/doc/rds.md @@ -26,6 +26,14 @@ If you use `pt-table-checksum` as a part of your data integrity checks, you migh This tool requires binlog_format=STATEMENT, but the current binlog_format is set to ROW and an error occurred while attempting to change it. If running MySQL 5.1.29 or newer, setting binlog_format requires the SUPER privilege. You will need to manually set binlog_format to 'STATEMENT' before running this tool. ``` +#### Binlog filtering + +In Aurora, the [binlog filtering feature][aws_replication_docs_bin_log_filtering] is enabled by default. This becomes an issue when gh-ost tries to do the cut-over, because gh-ost waits for an entry in the binlog to proceed but this entry will never end up in the binlog because it gets filtered out by the binlog filtering feature. +You need to turn this feature off during the migration process. +Set the `aurora_enable_repl_bin_log_filtering` parameter to 0 in the Parameter Group for your cluster. +When the migration is done, set it back to 1 (default). + + #### Preflight checklist Before trying to run any `gh-ost` migrations you will want to confirm the following: @@ -35,6 +43,7 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow - [ ] Executing `SHOW SLAVE STATUS\G` on your replica cluster displays the correct master host, binlog position, etc. - [ ] Database backup retention is greater than 1 day to enable binlogs - [ ] You have setup [`hooks`][ghost_hooks] to issue RDS procedures for stopping and starting replication. (see [github/gh-ost#163][ghost_rds_issue_tracking] for examples) +- [ ] The parameter `aurora_enable_repl_bin_log_filtering` is set to 0 [new_issue]: https://github.com/github/gh-ost/issues/new [assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr @@ -43,3 +52,4 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow [percona_toolkit_patch]: https://github.com/jacobbednarz/percona-toolkit/commit/0271ba6a094da446a5e5bb8d99b5c26f1777f2b9 [ghost_hooks]: https://github.com/github/gh-ost/blob/master/doc/hooks.md [ghost_rds_issue_tracking]: https://github.com/github/gh-ost/issues/163 +[aws_replication_docs_bin_log_filtering]: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Replication.html#AuroraMySQL.Replication.Performance \ No newline at end of file From 2eb141010bbc63da9aed593304026affa8b6606f Mon Sep 17 00:00:00 2001 From: Yaser Azfar Date: Fri, 27 Dec 2019 16:54:43 +1300 Subject: [PATCH 138/176] adds error checking for an err variable that was left unchecked --- go/sql/builder.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/sql/builder.go b/go/sql/builder.go index 2c5a7ae..4b019bc 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -492,6 +492,9 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol } setClause, err := BuildSetPreparedClause(mappedSharedColumns) + if err != nil { + return "", sharedArgs, uniqueKeyArgs, err + } equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names()) result = fmt.Sprintf(` From 9af130d93127fb2b3f8109ff764c27bef9905bd1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:46:35 +0200 Subject: [PATCH 139/176] Replication tests on 8.0 --- script/cibuild-gh-ost-replica-tests | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index ab8c689..447e5e0 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -61,6 +61,9 @@ test_mysql_version() { echo "Building..." . script/build # Test all versions: +find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do + echo "found MySQL version: $mysql_version" +done find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do test_mysql_version "$mysql_version" done From 14674274dc9858ef8fa027db4fd366aacd613c0d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:48:55 +0200 Subject: [PATCH 140/176] Experiment with https://github.com/github/gh-ost-ci-env/pull/8 --- script/cibuild-gh-ost-replica-tests | 1 + 1 file changed, 1 insertion(+) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 447e5e0..942e95a 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -16,6 +16,7 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git + git checkout real-tar-gz fi test_mysql_version() { From 919647cbbc32a7ec5514d64396cbb9349694d35a Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:50:34 +0200 Subject: [PATCH 141/176] removing travis --- .travis.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c1c1891..0000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -# http://docs.travis-ci.com/user/languages/go/ -language: go - -go: - - "1.12.x" - -os: - - linux - -services: - - mysql - -env: -- MYSQL_USER=root -- CURRENT_CI_ENV=travis - -addons: - apt: - packages: - - git - - numactl - - libaio1 - -before_install: - - mysql -e 'CREATE DATABASE IF NOT EXISTS test;' - -install: true - -script: - - script/cibuild - -notifications: - email: false From 3528d675502f2c1cefd4cee1f6caa77042faf21d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:51:40 +0200 Subject: [PATCH 142/176] fetch the correct branch. Oh and removing travis --- script/cibuild-gh-ost-replica-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 942e95a..c0b53d9 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -6,7 +6,7 @@ whoami # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git real-tar-gz | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) @@ -16,6 +16,7 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git + git fetch origin real-tar-gz git checkout real-tar-gz fi From 9201b9bc04b41dbfadd9da7406dd3549f3fe4ad7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 09:54:07 +0200 Subject: [PATCH 143/176] fetch and checkout from git path --- script/cibuild-gh-ost-replica-tests | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c0b53d9..813fdd8 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -16,8 +16,11 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git - git fetch origin real-tar-gz - git checkout real-tar-gz + ( + cd gh-ost-ci-env + git fetch origin real-tar-gz + git checkout real-tar-gz + ) fi test_mysql_version() { From b0f487ea613445c6d5a444b9176b3e93e333d7ea Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:07:14 +0200 Subject: [PATCH 144/176] create user --- script/cibuild-gh-ost-replica-tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 813fdd8..c0c925c 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -55,7 +55,8 @@ test_mysql_version() { export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" - gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%' identified by 'gh-ost'" + gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" + gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" echo "### Running gh-ost tests for $mysql_version" ./localtests/test.sh -b bin/gh-ost From a6d0d5ca26fa9291721afc73f61792933e9565d7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:12:29 +0200 Subject: [PATCH 145/176] not null for unique key test --- localtests/swap-uk-uk/create.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localtests/swap-uk-uk/create.sql b/localtests/swap-uk-uk/create.sql index 5fcbf32..30c1542 100644 --- a/localtests/swap-uk-uk/create.sql +++ b/localtests/swap-uk-uk/create.sql @@ -1,8 +1,8 @@ drop table if exists gh_ost_test; create table gh_ost_test ( - id bigint, + id bigint not null, i int not null, - ts timestamp(6), + ts timestamp(6) not null, unique key id_uidx(id), unique key its_uidx(i, ts) ) ; From 4f827a271e5094a3b59c195dd9e4af54a1fb857d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:42:36 +0200 Subject: [PATCH 146/176] 8.0 is on master now --- script/cibuild-gh-ost-replica-tests | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c0c925c..8b518e1 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -6,7 +6,7 @@ whoami # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git real-tar-gz | cut -f1) +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) @@ -16,11 +16,6 @@ echo "local commit is: $local_commit" if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git - ( - cd gh-ost-ci-env - git fetch origin real-tar-gz - git checkout real-tar-gz - ) fi test_mysql_version() { From c30e5862f7b46170c1a8d444b7439e02bd16a752 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:43:08 +0200 Subject: [PATCH 147/176] 8.0 is on master now --- script/cibuild-gh-ost-replica-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 8b518e1..3de9e05 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -6,7 +6,7 @@ whoami # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git | cut -f1) +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) From 05b4ac08b22991c9f6e31a6f04acfe79a3034321 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 11:43:19 +0200 Subject: [PATCH 148/176] Release v1.0.49 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 56d0dad..f133985 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.48 +1.0.49 From 8bdf2d2dbd698a71c40e20d3885f92aca85a6805 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 16:11:19 +0200 Subject: [PATCH 149/176] Adding script/dock: simplifying testing/packaging --- script/dock | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 script/dock diff --git a/script/dock b/script/dock new file mode 100755 index 0000000..d17ae5d --- /dev/null +++ b/script/dock @@ -0,0 +1,30 @@ +#!/bin/bash + +# Usage: +# dock [arg] +# dock test: build gh-ost & run unit and integration tests +# docker alpine: build and run gh-ost on alpine linux +# docker pkg [target-path]: build gh-ost release packages and copy to target path (default path: /tmp/gh-ost-release) + +command="$1" + +case "$command" in + "test") + docker_target="gh-ost-test" + docker build . -f Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest" + ;; + "alpine") + docker_target="gh-ost-alpine" + docker build . -f Dockerfile -t "${docker_target}" && docker run --rm -it -p 3000:3000 "${docker_target}:latest" + ;; + "pkg") + packages_path="${2:-/tmp/gh-ost-release}" + docker_target="gh-ost-packaging" + docker build . -f Dockerfile.packaging -t "${docker_target}" && docker run --rm -it -v "${packages_path}:/tmp/pkg" "${docker_target}:latest" bash -c 'find /tmp/gh-ost-release/ -maxdepth 1 -type f | xargs cp -t /tmp/pkg' + echo "packages generated on ${packages_path}:" + ls -l "${packages_path}" + ;; + *) + >&2 echo "Usage: dock dock [arg]" + exit 1 +esac From 40953fe82f9db405cbd45f04ee4f473b2f0511ee Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 16:11:34 +0200 Subject: [PATCH 150/176] Adding script/dock: simplifying testing/packaging --- build.sh | 4 ++-- script/dock | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index b7f3a54..46db9c2 100755 --- a/build.sh +++ b/build.sh @@ -61,11 +61,11 @@ main() { mkdir -p ${buildpath} rm -rf ${buildpath:?}/* - build macOS osx darwin amd64 build GNU/Linux linux linux amd64 + # build macOS osx darwin amd64 echo "Binaries found in:" - ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz + find $buildpath/gh-ost* -type f -maxdepth 1 } main "$@" diff --git a/script/dock b/script/dock index d17ae5d..486061d 100755 --- a/script/dock +++ b/script/dock @@ -1,9 +1,8 @@ #!/bin/bash # Usage: -# dock [arg] +# dock [arg] # dock test: build gh-ost & run unit and integration tests -# docker alpine: build and run gh-ost on alpine linux # docker pkg [target-path]: build gh-ost release packages and copy to target path (default path: /tmp/gh-ost-release) command="$1" @@ -13,10 +12,6 @@ case "$command" in docker_target="gh-ost-test" docker build . -f Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest" ;; - "alpine") - docker_target="gh-ost-alpine" - docker build . -f Dockerfile -t "${docker_target}" && docker run --rm -it -p 3000:3000 "${docker_target}:latest" - ;; "pkg") packages_path="${2:-/tmp/gh-ost-release}" docker_target="gh-ost-packaging" From f6ea377a4b372e2cf11645dbffd326d5a9f57a38 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 9 Feb 2020 16:11:50 +0200 Subject: [PATCH 151/176] Dockerfile.packaging --- Dockerfile.packaging | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Dockerfile.packaging diff --git a/Dockerfile.packaging b/Dockerfile.packaging new file mode 100644 index 0000000..6aa2ace --- /dev/null +++ b/Dockerfile.packaging @@ -0,0 +1,36 @@ +# Requires Docker 17.09 or later (multi stage builds) +# +# Orchestrator will look for a configuration file at /etc/orchestrator.conf.json. +# It will listen on port 3000. +# If not present a minimal configuration will be generated using the following environment variables: +# +# Default variables which can be used are: +# +# ORC_TOPOLOGY_USER (default: orchestrator): username used by orchestrator to login to MySQL when polling/discovering +# ORC_TOPOLOGY_PASSWORD (default: orchestrator): password needed to login to MySQL when polling/discovering +# ORC_DB_HOST (default: orchestrator): orchestrator backend MySQL host +# ORC_DB_PORT (default: 3306): port used by orchestrator backend MySQL server +# ORC_DB_NAME (default: orchestrator): database named used by orchestrator backend MySQL server +# ORC_USER (default: orc_server_user): username used to login to orchestrator backend MySQL server +# ORC_PASSWORD (default: orc_server_password): password used to login to orchestrator backend MySQL server + +FROM golang:1.12.6 + +RUN apt-get update +RUN apt-get install -y ruby ruby-dev rubygems build-essential +RUN gem install --no-ri --no-rdoc fpm +ENV GOPATH=/tmp/go + +RUN apt-get install -y curl +RUN apt-get install -y rsync +RUN apt-get install -y gcc +RUN apt-get install -y g++ +RUN apt-get install -y bash +RUN apt-get install -y git +RUN apt-get install -y tar +RUN apt-get install -y rpm + +RUN mkdir -p $GOPATH/src/github.com/github/gh-ost +WORKDIR $GOPATH/src/github.com/github/gh-ost +COPY . . +RUN bash build.sh From 85a285a9e4ecfd63f817c04f998b572a2fd18e5d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 10 Feb 2020 08:59:44 +0200 Subject: [PATCH 152/176] cleanup --- Dockerfile.packaging | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 6aa2ace..214c70c 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,18 +1,4 @@ -# Requires Docker 17.09 or later (multi stage builds) # -# Orchestrator will look for a configuration file at /etc/orchestrator.conf.json. -# It will listen on port 3000. -# If not present a minimal configuration will be generated using the following environment variables: -# -# Default variables which can be used are: -# -# ORC_TOPOLOGY_USER (default: orchestrator): username used by orchestrator to login to MySQL when polling/discovering -# ORC_TOPOLOGY_PASSWORD (default: orchestrator): password needed to login to MySQL when polling/discovering -# ORC_DB_HOST (default: orchestrator): orchestrator backend MySQL host -# ORC_DB_PORT (default: 3306): port used by orchestrator backend MySQL server -# ORC_DB_NAME (default: orchestrator): database named used by orchestrator backend MySQL server -# ORC_USER (default: orc_server_user): username used to login to orchestrator backend MySQL server -# ORC_PASSWORD (default: orc_server_password): password used to login to orchestrator backend MySQL server FROM golang:1.12.6 From 08f171790b01810f83e5972a0d0c1f4cdbc74276 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 13 Feb 2020 13:45:39 +0200 Subject: [PATCH 153/176] Update go-sql-driver to latest --- .../mysql/{ => .github}/CONTRIBUTING.md | 0 .../go-sql-driver/mysql/.travis.yml | 3 +- vendor/github.com/go-sql-driver/mysql/AUTHORS | 5 + .../go-sql-driver/mysql/CHANGELOG.md | 39 ++++ .../github.com/go-sql-driver/mysql/README.md | 26 ++- .../go-sql-driver/mysql/appengine.go | 25 --- .../go-sql-driver/mysql/conncheck.go | 43 ++--- .../go-sql-driver/mysql/conncheck_dummy.go | 4 +- .../go-sql-driver/mysql/conncheck_test.go | 2 +- .../go-sql-driver/mysql/connection.go | 13 +- .../go-sql-driver/mysql/connection_test.go | 28 +++ .../go-sql-driver/mysql/connector.go | 13 +- .../go-sql-driver/mysql/connector_test.go | 30 +++ .../github.com/go-sql-driver/mysql/driver.go | 22 +++ .../go-sql-driver/mysql/driver_go110.go | 37 ---- .../go-sql-driver/mysql/driver_go110_test.go | 137 -------------- .../go-sql-driver/mysql/driver_test.go | 171 +++++++++++++++++- vendor/github.com/go-sql-driver/mysql/dsn.go | 166 +++++------------ .../go-sql-driver/mysql/dsn_test.go | 36 ++-- vendor/github.com/go-sql-driver/mysql/go.mod | 3 + .../go-sql-driver/mysql/nulltime.go | 50 +++++ .../go-sql-driver/mysql/nulltime_go113.go | 31 ++++ .../go-sql-driver/mysql/nulltime_legacy.go | 34 ++++ .../go-sql-driver/mysql/nulltime_test.go | 62 +++++++ .../github.com/go-sql-driver/mysql/packets.go | 2 +- .../github.com/go-sql-driver/mysql/utils.go | 54 ------ .../go-sql-driver/mysql/utils_test.go | 41 ----- 27 files changed, 601 insertions(+), 476 deletions(-) rename vendor/github.com/go-sql-driver/mysql/{ => .github}/CONTRIBUTING.md (100%) delete mode 100644 vendor/github.com/go-sql-driver/mysql/appengine.go create mode 100644 vendor/github.com/go-sql-driver/mysql/connector_test.go delete mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110.go delete mode 100644 vendor/github.com/go-sql-driver/mysql/driver_go110_test.go create mode 100644 vendor/github.com/go-sql-driver/mysql/go.mod create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_go113.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_test.go diff --git a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/go-sql-driver/mysql/.github/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md rename to vendor/github.com/go-sql-driver/mysql/.github/CONTRIBUTING.md diff --git a/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/go-sql-driver/mysql/.travis.yml index eae311b..56fcf25 100644 --- a/vendor/github.com/go-sql-driver/mysql/.travis.yml +++ b/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -1,10 +1,10 @@ sudo: false language: go go: - - 1.9.x - 1.10.x - 1.11.x - 1.12.x + - 1.13.x - master before_install: @@ -105,6 +105,7 @@ matrix: homebrew: packages: - mysql + update: true go: 1.12.x before_install: - go get golang.org/x/tools/cmd/cover diff --git a/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/go-sql-driver/mysql/AUTHORS index bfe74c4..0896ba1 100644 --- a/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -13,6 +13,7 @@ Aaron Hopkins Achille Roussel +Alex Snast Alexey Palazhchenko Andrew Reid Arne Hormann @@ -44,6 +45,7 @@ James Harr Jeff Hodges Jeffrey Charles Jerome Meyer +Jiajia Zhong Jian Zhen Joshua Prunier Julien Lefevre @@ -62,6 +64,7 @@ Lucas Liu Luke Scott Maciej Zimnoch Michael Woolnough +Nathanial Murphy Nicola Peduzzi Olivier Mengué oscarzhao @@ -81,6 +84,7 @@ Steven Hartland Thomas Wodarek Tim Ruffles Tom Jenkinson +Vladimir Kovpak Xiangyu Hu Xiaobing Jiang Xiuming Chen @@ -90,6 +94,7 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +DigitalOcean Inc. Facebook Inc. GitHub Inc. Google Inc. diff --git a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md index 2d87d74..9cb97b3 100644 --- a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md +++ b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -1,3 +1,42 @@ +## Version 1.5 (2020-01-07) + +Changes: + + - Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017) + - Improve buffer handling (#890) + - Document potentially insecure TLS configs (#901) + - Use a double-buffering scheme to prevent data races (#943) + - Pass uint64 values without converting them to string (#838, #955) + - Update collations and make utf8mb4 default (#877, #1054) + - Make NullTime compatible with sql.NullTime in Go 1.13+ (#995) + - Removed CloudSQL support (#993, #1007) + - Add Go Module support (#1003) + +New Features: + + - Implement support of optional TLS (#900) + - Check connection liveness (#934, #964, #997, #1048, #1051, #1052) + - Implement Connector Interface (#941, #958, #1020, #1035) + +Bugfixes: + + - Mark connections as bad on error during ping (#875) + - Mark connections as bad on error during dial (#867) + - Fix connection leak caused by rapid context cancellation (#1024) + - Mark connections as bad on error during Conn.Prepare (#1030) + + +## Version 1.4.1 (2018-11-14) + +Bugfixes: + + - Fix TIME format for binary columns (#818) + - Fix handling of empty auth plugin names (#835) + - Fix caching_sha2_password with empty password (#826) + - Fix canceled context broke mysqlConn (#862) + - Fix OldAuthSwitchRequest support (#870) + - Fix Auth Response packet for cleartext password (#887) + ## Version 1.4 (2018-06-03) Changes: diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md index c6adf1d..d2627a4 100644 --- a/vendor/github.com/go-sql-driver/mysql/README.md +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.9 or higher. We aim to support the 3 latest versions of Go. + * Go 1.10 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- @@ -166,6 +166,17 @@ Sets the charset used for client-server interaction (`"SET NAMES "`). If Usage of the `charset` parameter is discouraged because it issues additional queries to the server. Unless you need the fallback behavior, please use `collation` instead. +##### `checkConnLiveness` + +``` +Type: bool +Valid Values: true, false +Default: true +``` + +On supported platforms connections retrieved from the connection pool are checked for liveness before using them. If the check fails, the respective connection is marked as bad and the query retried with another connection. +`checkConnLiveness=false` disables this liveness check of connections. + ##### `collation` ``` @@ -396,14 +407,9 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine (First Generation MySQL Server): +Google Cloud SQL on App Engine: ``` -user@cloudsql(project-id:instance-name)/dbname -``` - -Google Cloud SQL on App Engine (Second Generation MySQL Server): -``` -user@cloudsql(project-id:regionname:instance-name)/dbname +user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname ``` TCP using default port (3306) on localhost: @@ -457,13 +463,13 @@ Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-d ### Unicode support -Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. +Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. -See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. +See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. diff --git a/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/go-sql-driver/mysql/appengine.go deleted file mode 100644 index 914e662..0000000 --- a/vendor/github.com/go-sql-driver/mysql/appengine.go +++ /dev/null @@ -1,25 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build appengine - -package mysql - -import ( - "context" - "net" - - "google.golang.org/appengine/cloudsql" -) - -func init() { - RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) { - // XXX: the cloudsql driver still does not export a Context-aware dialer. - return cloudsql.Dial(instance) - }) -} diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/go-sql-driver/mysql/conncheck.go index cc47aa5..024eb28 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build !windows,!appengine +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql @@ -19,35 +19,36 @@ import ( var errUnexpectedRead = errors.New("unexpected read from socket") -func connCheck(c net.Conn) error { - var ( - n int - err error - buff [1]byte - ) +func connCheck(conn net.Conn) error { + var sysErr error - sconn, ok := c.(syscall.Conn) + sysConn, ok := conn.(syscall.Conn) if !ok { return nil } - rc, err := sconn.SyscallConn() + rawConn, err := sysConn.SyscallConn() if err != nil { return err } - rerr := rc.Read(func(fd uintptr) bool { - n, err = syscall.Read(int(fd), buff[:]) + + err = rawConn.Read(func(fd uintptr) bool { + var buf [1]byte + n, err := syscall.Read(int(fd), buf[:]) + switch { + case n == 0 && err == nil: + sysErr = io.EOF + case n > 0: + sysErr = errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + sysErr = nil + default: + sysErr = err + } return true }) - switch { - case rerr != nil: - return rerr - case n == 0 && err == nil: - return io.EOF - case n > 0: - return errUnexpectedRead - case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: - return nil - default: + if err != nil { return err } + + return sysErr } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go index fd01f64..ea7fb60 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -6,12 +6,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build windows appengine +// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos package mysql import "net" -func connCheck(c net.Conn) error { +func connCheck(conn net.Conn) error { return nil } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go index b7234b0..5399551 100644 --- a/vendor/github.com/go-sql-driver/mysql/conncheck_test.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_test.go @@ -6,7 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build go1.10,!windows +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index 565a548..b07cd76 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -12,6 +12,7 @@ import ( "context" "database/sql" "database/sql/driver" + "encoding/json" "io" "net" "strconv" @@ -154,7 +155,9 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, mc.markBadConn(err) + // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. + errLog.Print(err) + return nil, driver.ErrBadConn } stmt := &mysqlStmt{ @@ -269,6 +272,14 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin } buf = append(buf, '\'') } + case json.RawMessage: + buf = append(buf, '\'') + if mc.status&statusNoBackslashEscapes == 0 { + buf = escapeBytesBackslash(buf, v) + } else { + buf = escapeBytesQuotes(buf, v) + } + buf = append(buf, '\'') case []byte: if v == nil { buf = append(buf, "NULL"...) diff --git a/vendor/github.com/go-sql-driver/mysql/connection_test.go b/vendor/github.com/go-sql-driver/mysql/connection_test.go index 19c17ff..a6d6773 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection_test.go +++ b/vendor/github.com/go-sql-driver/mysql/connection_test.go @@ -11,6 +11,7 @@ package mysql import ( "context" "database/sql/driver" + "encoding/json" "errors" "net" "testing" @@ -36,6 +37,33 @@ func TestInterpolateParams(t *testing.T) { } } +func TestInterpolateParamsJSONRawMessage(t *testing.T) { + mc := &mysqlConn{ + buf: newBuffer(nil), + maxAllowedPacket: maxPacketSize, + cfg: &Config{ + InterpolateParams: true, + }, + } + + buf, err := json.Marshal(struct { + Value int `json:"value"` + }{Value: 42}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + q, err := mc.interpolateParams("SELECT ?", []driver.Value{json.RawMessage(buf)}) + if err != nil { + t.Errorf("Expected err=nil, got %#v", err) + return + } + expected := `SELECT '{\"value\":42}'` + if q != expected { + t.Errorf("Expected: %q\nGot: %q", expected, q) + } +} + func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { mc := &mysqlConn{ buf: newBuffer(nil), diff --git a/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/go-sql-driver/mysql/connector.go index 5aaaba4..d567b4e 100644 --- a/vendor/github.com/go-sql-driver/mysql/connector.go +++ b/vendor/github.com/go-sql-driver/mysql/connector.go @@ -37,17 +37,19 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { dial, ok := dials[mc.cfg.Net] dialsLock.RUnlock() if ok { - mc.netConn, err = dial(ctx, mc.cfg.Addr) + dctx := ctx + if mc.cfg.Timeout > 0 { + var cancel context.CancelFunc + dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) + defer cancel() + } + mc.netConn, err = dial(dctx, mc.cfg.Addr) } else { nd := net.Dialer{Timeout: mc.cfg.Timeout} mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) } if err != nil { - if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - errLog.Print("net.Error from Dial()': ", nerr.Error()) - return nil, driver.ErrBadConn - } return nil, err } @@ -64,6 +66,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { // Call startWatcher for context support (From Go 1.8) mc.startWatcher() if err := mc.watchCancel(ctx); err != nil { + mc.cleanup() return nil, err } defer mc.finish() diff --git a/vendor/github.com/go-sql-driver/mysql/connector_test.go b/vendor/github.com/go-sql-driver/mysql/connector_test.go new file mode 100644 index 0000000..976903c --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connector_test.go @@ -0,0 +1,30 @@ +package mysql + +import ( + "context" + "net" + "testing" + "time" +) + +func TestConnectorReturnsTimeout(t *testing.T) { + connector := &connector{&Config{ + Net: "tcp", + Addr: "1.1.1.1:1234", + Timeout: 10 * time.Millisecond, + }} + + _, err := connector.Connect(context.Background()) + if err == nil { + t.Fatal("error expected") + } + + if nerr, ok := err.(*net.OpError); ok { + expected := "dial tcp 1.1.1.1:1234: i/o timeout" + if nerr.Error() != expected { + t.Fatalf("expected %q, got %q", expected, nerr.Error()) + } + } else { + t.Fatalf("expected %T, got %T", nerr, err) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index 1f9decf..c1bdf11 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -83,3 +83,25 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { func init() { sql.Register("mysql", &MySQLDriver{}) } + +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { + return nil, err + } + return &connector{cfg: cfg}, nil +} + +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) + if err != nil { + return nil, err + } + return &connector{ + cfg: cfg, + }, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110.go b/vendor/github.com/go-sql-driver/mysql/driver_go110.go deleted file mode 100644 index eb5a8fe..0000000 --- a/vendor/github.com/go-sql-driver/mysql/driver_go110.go +++ /dev/null @@ -1,37 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "database/sql/driver" -) - -// NewConnector returns new driver.Connector. -func NewConnector(cfg *Config) (driver.Connector, error) { - cfg = cfg.Clone() - // normalize the contents of cfg so calls to NewConnector have the same - // behavior as MySQLDriver.OpenConnector - if err := cfg.normalize(); err != nil { - return nil, err - } - return &connector{cfg: cfg}, nil -} - -// OpenConnector implements driver.DriverContext. -func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { - cfg, err := ParseDSN(dsn) - if err != nil { - return nil, err - } - return &connector{ - cfg: cfg, - }, nil -} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go b/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go deleted file mode 100644 index 19a0e59..0000000 --- a/vendor/github.com/go-sql-driver/mysql/driver_go110_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.10 - -package mysql - -import ( - "context" - "database/sql" - "database/sql/driver" - "fmt" - "net" - "testing" - "time" -) - -var _ driver.DriverContext = &MySQLDriver{} - -type dialCtxKey struct{} - -func TestConnectorObeysDialTimeouts(t *testing.T) { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - if !ctx.Value(dialCtxKey{}).(bool) { - return nil, fmt.Errorf("test error: query context is not propagated to our dialer") - } - return d.DialContext(ctx, prot, addr) - }) - - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) - if err != nil { - t.Fatalf("error connecting: %s", err.Error()) - } - defer db.Close() - - ctx := context.WithValue(context.Background(), dialCtxKey{}, true) - - _, err = db.ExecContext(ctx, "DO 1") - if err != nil { - t.Fatal(err) - } -} - -func configForTests(t *testing.T) *Config { - if !available { - t.Skipf("MySQL server not running on %s", netAddr) - } - - mycnf := NewConfig() - mycnf.User = user - mycnf.Passwd = pass - mycnf.Addr = addr - mycnf.Net = prot - mycnf.DBName = dbname - return mycnf -} - -func TestNewConnector(t *testing.T) { - mycnf := configForTests(t) - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - db := sql.OpenDB(conn) - defer db.Close() - - if err := db.Ping(); err != nil { - t.Fatal(err) - } -} - -type slowConnection struct { - net.Conn - slowdown time.Duration -} - -func (sc *slowConnection) Read(b []byte) (int, error) { - time.Sleep(sc.slowdown) - return sc.Conn.Read(b) -} - -type connectorHijack struct { - driver.Connector - connErr error -} - -func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { - var conn driver.Conn - conn, cw.connErr = cw.Connector.Connect(ctx) - return conn, cw.connErr -} - -func TestConnectorTimeoutsDuringOpen(t *testing.T) { - RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { - var d net.Dialer - conn, err := d.DialContext(ctx, prot, addr) - if err != nil { - return nil, err - } - return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil - }) - - mycnf := configForTests(t) - mycnf.Net = "slowconn" - - conn, err := NewConnector(mycnf) - if err != nil { - t.Fatal(err) - } - - hijack := &connectorHijack{Connector: conn} - - db := sql.OpenDB(hijack) - defer db.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - _, err = db.ExecContext(ctx, "DO 1") - if err != context.DeadlineExceeded { - t.Fatalf("ExecContext should have timed out") - } - if hijack.connErr != context.DeadlineExceeded { - t.Fatalf("(*Connector).Connect should have timed out") - } -} diff --git a/vendor/github.com/go-sql-driver/mysql/driver_test.go b/vendor/github.com/go-sql-driver/mysql/driver_test.go index 3dee1ba..ace083d 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver_test.go +++ b/vendor/github.com/go-sql-driver/mysql/driver_test.go @@ -1874,7 +1874,7 @@ func TestDialNonRetryableNetErr(t *testing.T) { func TestDialTemporaryNetErr(t *testing.T) { testErr := netErrorMock{temporary: true} - testDialError(t, testErr, driver.ErrBadConn) + testDialError(t, testErr, testErr) } // Tests custom dial functions @@ -2994,3 +2994,172 @@ func TestRawBytesAreNotModified(t *testing.T) { } }) } + +var _ driver.DriverContext = &MySQLDriver{} + +type dialCtxKey struct{} + +func TestConnectorObeysDialTimeouts(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + if !ctx.Value(dialCtxKey{}).(bool) { + return nil, fmt.Errorf("test error: query context is not propagated to our dialer") + } + return d.DialContext(ctx, prot, addr) + }) + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + + ctx := context.WithValue(context.Background(), dialCtxKey{}, true) + + _, err = db.ExecContext(ctx, "DO 1") + if err != nil { + t.Fatal(err) + } +} + +func configForTests(t *testing.T) *Config { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + mycnf := NewConfig() + mycnf.User = user + mycnf.Passwd = pass + mycnf.Addr = addr + mycnf.Net = prot + mycnf.DBName = dbname + return mycnf +} + +func TestNewConnector(t *testing.T) { + mycnf := configForTests(t) + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +type slowConnection struct { + net.Conn + slowdown time.Duration +} + +func (sc *slowConnection) Read(b []byte) (int, error) { + time.Sleep(sc.slowdown) + return sc.Conn.Read(b) +} + +type connectorHijack struct { + driver.Connector + connErr error +} + +func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) { + var conn driver.Conn + conn, cw.connErr = cw.Connector.Connect(ctx) + return conn, cw.connErr +} + +func TestConnectorTimeoutsDuringOpen(t *testing.T) { + RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, prot, addr) + if err != nil { + return nil, err + } + return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil + }) + + mycnf := configForTests(t) + mycnf.Net = "slowconn" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + hijack := &connectorHijack{Connector: conn} + + db := sql.OpenDB(hijack) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err = db.ExecContext(ctx, "DO 1") + if err != context.DeadlineExceeded { + t.Fatalf("ExecContext should have timed out") + } + if hijack.connErr != context.DeadlineExceeded { + t.Fatalf("(*Connector).Connect should have timed out") + } +} + +// A connection which can only be closed. +type dummyConnection struct { + net.Conn + closed bool +} + +func (d *dummyConnection) Close() error { + d.closed = true + return nil +} + +func TestConnectorTimeoutsWatchCancel(t *testing.T) { + var ( + cancel func() // Used to cancel the context just after connecting. + created *dummyConnection // The created connection. + ) + + RegisterDialContext("TestConnectorTimeoutsWatchCancel", func(ctx context.Context, addr string) (net.Conn, error) { + // Canceling at this time triggers the watchCancel error branch in Connect(). + cancel() + created = &dummyConnection{} + return created, nil + }) + + mycnf := NewConfig() + mycnf.User = "root" + mycnf.Addr = "foo" + mycnf.Net = "TestConnectorTimeoutsWatchCancel" + + conn, err := NewConnector(mycnf) + if err != nil { + t.Fatal(err) + } + + db := sql.OpenDB(conn) + defer db.Close() + + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + + if _, err := db.Conn(ctx); err != context.Canceled { + t.Errorf("got %v, want context.Canceled", err) + } + + if created == nil { + t.Fatal("no connection created") + } + if !created.closed { + t.Errorf("connection not closed") + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go index 1d9b4ab..75c8c24 100644 --- a/vendor/github.com/go-sql-driver/mysql/dsn.go +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -55,6 +55,7 @@ type Config struct { AllowCleartextPasswords bool // Allows the cleartext client side plugin AllowNativePasswords bool // Allows the native password authentication method AllowOldPasswords bool // Allows the old insecure password method + CheckConnLiveness bool // Check connections for liveness before using them ClientFoundRows bool // Return number of matching rows instead of rows changed ColumnsWithAlias bool // Prepend table alias to column names InterpolateParams bool // Interpolate placeholders into query string @@ -70,6 +71,7 @@ func NewConfig() *Config { Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, + CheckConnLiveness: true, } } @@ -148,6 +150,19 @@ func (cfg *Config) normalize() error { return nil } +func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { + buf.Grow(1 + len(name) + 1 + len(value)) + if !*hasParam { + *hasParam = true + buf.WriteByte('?') + } else { + buf.WriteByte('&') + } + buf.WriteString(name) + buf.WriteByte('=') + buf.WriteString(value) +} + // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. func (cfg *Config) FormatDSN() string { @@ -186,165 +201,75 @@ func (cfg *Config) FormatDSN() string { } if cfg.AllowCleartextPasswords { - if hasParam { - buf.WriteString("&allowCleartextPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowCleartextPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true") } if !cfg.AllowNativePasswords { - if hasParam { - buf.WriteString("&allowNativePasswords=false") - } else { - hasParam = true - buf.WriteString("?allowNativePasswords=false") - } + writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false") } if cfg.AllowOldPasswords { - if hasParam { - buf.WriteString("&allowOldPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowOldPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowOldPasswords", "true") + } + + if !cfg.CheckConnLiveness { + writeDSNParam(&buf, &hasParam, "checkConnLiveness", "false") } if cfg.ClientFoundRows { - if hasParam { - buf.WriteString("&clientFoundRows=true") - } else { - hasParam = true - buf.WriteString("?clientFoundRows=true") - } + writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } if col := cfg.Collation; col != defaultCollation && len(col) > 0 { - if hasParam { - buf.WriteString("&collation=") - } else { - hasParam = true - buf.WriteString("?collation=") - } - buf.WriteString(col) + writeDSNParam(&buf, &hasParam, "collation", col) } if cfg.ColumnsWithAlias { - if hasParam { - buf.WriteString("&columnsWithAlias=true") - } else { - hasParam = true - buf.WriteString("?columnsWithAlias=true") - } + writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true") } if cfg.InterpolateParams { - if hasParam { - buf.WriteString("&interpolateParams=true") - } else { - hasParam = true - buf.WriteString("?interpolateParams=true") - } + writeDSNParam(&buf, &hasParam, "interpolateParams", "true") } if cfg.Loc != time.UTC && cfg.Loc != nil { - if hasParam { - buf.WriteString("&loc=") - } else { - hasParam = true - buf.WriteString("?loc=") - } - buf.WriteString(url.QueryEscape(cfg.Loc.String())) + writeDSNParam(&buf, &hasParam, "loc", url.QueryEscape(cfg.Loc.String())) } if cfg.MultiStatements { - if hasParam { - buf.WriteString("&multiStatements=true") - } else { - hasParam = true - buf.WriteString("?multiStatements=true") - } + writeDSNParam(&buf, &hasParam, "multiStatements", "true") } if cfg.ParseTime { - if hasParam { - buf.WriteString("&parseTime=true") - } else { - hasParam = true - buf.WriteString("?parseTime=true") - } + writeDSNParam(&buf, &hasParam, "parseTime", "true") } if cfg.ReadTimeout > 0 { - if hasParam { - buf.WriteString("&readTimeout=") - } else { - hasParam = true - buf.WriteString("?readTimeout=") - } - buf.WriteString(cfg.ReadTimeout.String()) + writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } if cfg.RejectReadOnly { - if hasParam { - buf.WriteString("&rejectReadOnly=true") - } else { - hasParam = true - buf.WriteString("?rejectReadOnly=true") - } + writeDSNParam(&buf, &hasParam, "rejectReadOnly", "true") } if len(cfg.ServerPubKey) > 0 { - if hasParam { - buf.WriteString("&serverPubKey=") - } else { - hasParam = true - buf.WriteString("?serverPubKey=") - } - buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + writeDSNParam(&buf, &hasParam, "serverPubKey", url.QueryEscape(cfg.ServerPubKey)) } if cfg.Timeout > 0 { - if hasParam { - buf.WriteString("&timeout=") - } else { - hasParam = true - buf.WriteString("?timeout=") - } - buf.WriteString(cfg.Timeout.String()) + writeDSNParam(&buf, &hasParam, "timeout", cfg.Timeout.String()) } if len(cfg.TLSConfig) > 0 { - if hasParam { - buf.WriteString("&tls=") - } else { - hasParam = true - buf.WriteString("?tls=") - } - buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + writeDSNParam(&buf, &hasParam, "tls", url.QueryEscape(cfg.TLSConfig)) } if cfg.WriteTimeout > 0 { - if hasParam { - buf.WriteString("&writeTimeout=") - } else { - hasParam = true - buf.WriteString("?writeTimeout=") - } - buf.WriteString(cfg.WriteTimeout.String()) + writeDSNParam(&buf, &hasParam, "writeTimeout", cfg.WriteTimeout.String()) } if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { - if hasParam { - buf.WriteString("&maxAllowedPacket=") - } else { - hasParam = true - buf.WriteString("?maxAllowedPacket=") - } - buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) - + writeDSNParam(&buf, &hasParam, "maxAllowedPacket", strconv.Itoa(cfg.MaxAllowedPacket)) } // other params @@ -355,16 +280,7 @@ func (cfg *Config) FormatDSN() string { } sort.Strings(params) for _, param := range params { - if hasParam { - buf.WriteByte('&') - } else { - hasParam = true - buf.WriteByte('?') - } - - buf.WriteString(param) - buf.WriteByte('=') - buf.WriteString(url.QueryEscape(cfg.Params[param])) + writeDSNParam(&buf, &hasParam, param, url.QueryEscape(cfg.Params[param])) } } @@ -491,6 +407,14 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // Check connections for Liveness before using them + case "checkConnLiveness": + var isBool bool + cfg.CheckConnLiveness, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + // Switch "rowsAffected" mode case "clientFoundRows": var isBool bool diff --git a/vendor/github.com/go-sql-driver/mysql/dsn_test.go b/vendor/github.com/go-sql-driver/mysql/dsn_test.go index 50dc293..89815b3 100644 --- a/vendor/github.com/go-sql-driver/mysql/dsn_test.go +++ b/vendor/github.com/go-sql-driver/mysql/dsn_test.go @@ -22,55 +22,55 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { - "user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false}, + "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, }, } diff --git a/vendor/github.com/go-sql-driver/mysql/go.mod b/vendor/github.com/go-sql-driver/mysql/go.mod new file mode 100644 index 0000000..fffbf6a --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/go.mod @@ -0,0 +1,3 @@ +module github.com/go-sql-driver/mysql + +go 1.10 diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime.go b/vendor/github.com/go-sql-driver/mysql/nulltime.go new file mode 100644 index 0000000..afa8a89 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime.go @@ -0,0 +1,50 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "fmt" + "time" +) + +// Scan implements the Scanner interface. +// The value type must be time.Time or string / []byte (formatted time-string), +// otherwise Scan fails. +func (nt *NullTime) Scan(value interface{}) (err error) { + if value == nil { + nt.Time, nt.Valid = time.Time{}, false + return + } + + switch v := value.(type) { + case time.Time: + nt.Time, nt.Valid = v, true + return + case []byte: + nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Valid = (err == nil) + return + case string: + nt.Time, err = parseDateTime(v, time.UTC) + nt.Valid = (err == nil) + return + } + + nt.Valid = false + return fmt.Errorf("Can't convert %T to time.Time", value) +} + +// Value implements the driver Valuer interface. +func (nt NullTime) Value() (driver.Value, error) { + if !nt.Valid { + return nil, nil + } + return nt.Time, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go new file mode 100644 index 0000000..c392594 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go @@ -0,0 +1,31 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.13 + +package mysql + +import ( + "database/sql" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime sql.NullTime diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go new file mode 100644 index 0000000..86d159d --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go @@ -0,0 +1,34 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !go1.13 + +package mysql + +import ( + "time" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_test.go b/vendor/github.com/go-sql-driver/mysql/nulltime_test.go new file mode 100644 index 0000000..a14ec06 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_test.go @@ -0,0 +1,62 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql" + "database/sql/driver" + "testing" + "time" +) + +var ( + // Check implementation of interfaces + _ driver.Valuer = NullTime{} + _ sql.Scanner = (*NullTime)(nil) +) + +func TestScanNullTime(t *testing.T) { + var scanTests = []struct { + in interface{} + error bool + valid bool + time time.Time + }{ + {tDate, false, true, tDate}, + {sDate, false, true, tDate}, + {[]byte(sDate), false, true, tDate}, + {tDateTime, false, true, tDateTime}, + {sDateTime, false, true, tDateTime}, + {[]byte(sDateTime), false, true, tDateTime}, + {tDate0, false, true, tDate0}, + {sDate0, false, true, tDate0}, + {[]byte(sDate0), false, true, tDate0}, + {sDateTime0, false, true, tDate0}, + {[]byte(sDateTime0), false, true, tDate0}, + {"", true, false, tDate0}, + {"1234", true, false, tDate0}, + {0, true, false, tDate0}, + } + + var nt = NullTime{} + var err error + + for _, tst := range scanTests { + err = nt.Scan(tst.in) + if (err != nil) != tst.error { + t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) + } + if nt.Valid != tst.valid { + t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) + } + if nt.Time != tst.time { + t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) + } + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index 30b3352..82ad7a2 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -115,7 +115,7 @@ func (mc *mysqlConn) writePacket(data []byte) error { if mc.cfg.ReadTimeout != 0 { err = conn.SetReadDeadline(time.Time{}) } - if err == nil { + if err == nil && mc.cfg.CheckConnLiveness { err = connCheck(conn) } if err != nil { diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index cfa10e9..9552e80 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -106,60 +106,6 @@ func readBool(input string) (value bool, valid bool) { * Time related utils * ******************************************************************************/ -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { base := "0000-00-00 00:00:00.0000000" switch len(str) { diff --git a/vendor/github.com/go-sql-driver/mysql/utils_test.go b/vendor/github.com/go-sql-driver/mysql/utils_test.go index 8951a7a..10a60c2 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils_test.go +++ b/vendor/github.com/go-sql-driver/mysql/utils_test.go @@ -14,49 +14,8 @@ import ( "database/sql/driver" "encoding/binary" "testing" - "time" ) -func TestScanNullTime(t *testing.T) { - var scanTests = []struct { - in interface{} - error bool - valid bool - time time.Time - }{ - {tDate, false, true, tDate}, - {sDate, false, true, tDate}, - {[]byte(sDate), false, true, tDate}, - {tDateTime, false, true, tDateTime}, - {sDateTime, false, true, tDateTime}, - {[]byte(sDateTime), false, true, tDateTime}, - {tDate0, false, true, tDate0}, - {sDate0, false, true, tDate0}, - {[]byte(sDate0), false, true, tDate0}, - {sDateTime0, false, true, tDate0}, - {[]byte(sDateTime0), false, true, tDate0}, - {"", true, false, tDate0}, - {"1234", true, false, tDate0}, - {0, true, false, tDate0}, - } - - var nt = NullTime{} - var err error - - for _, tst := range scanTests { - err = nt.Scan(tst.in) - if (err != nil) != tst.error { - t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil)) - } - if nt.Valid != tst.valid { - t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid) - } - if nt.Time != tst.time { - t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time) - } - } -} - func TestLengthEncodedInteger(t *testing.T) { var integerTests = []struct { num uint64 From 9f5edc923e5eaef2150b9680307dee02818e6e46 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Feb 2020 12:48:23 +0200 Subject: [PATCH 154/176] Support --mysql-timeout flag --- go/cmd/gh-ost/main.go | 1 + go/mysql/connection.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..74c62d7 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -48,6 +48,7 @@ func main() { flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") + flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user") flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password") flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host") diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 654998c..6855ee0 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -27,6 +27,7 @@ type ConnectionConfig struct { Password string ImpliedKey *InstanceKey tlsConfig *tls.Config + Timeout float64 } func NewConnectionConfig() *ConnectionConfig { @@ -44,6 +45,7 @@ func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionC User: this.User, Password: this.Password, tlsConfig: this.tlsConfig, + Timeout: this.Timeout, } config.ImpliedKey = &config.Key return config @@ -116,5 +118,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { if this.tlsConfig != nil { tlsOption = TLS_CONFIG_KEY } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?timeout=%fs&readTimeout=%fs&writeTimeout=%fs&interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, this.Timeout, this.Timeout, this.Timeout, interpolateParams, tlsOption) } From 97f2d7161637b56351d9ed2c3dad136558b869c0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Feb 2020 12:52:26 +0200 Subject: [PATCH 155/176] fix unit tests --- go/mysql/connection_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 6ce85a7..2befbc9 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") } func TestGetDBUriWithTLSSetup(t *testing.T) { @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") } From d5a7e43266c36d86b7ef478bc975983ca24317a9 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Tue, 24 Mar 2020 14:54:54 -0500 Subject: [PATCH 156/176] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 63f0df9..605546d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /bin/ /libexec/ /.vendor/ +.idea/ From 0b2702bf539aebea09948fe5e9486a30135e4241 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 14:12:00 -0500 Subject: [PATCH 157/176] Throttle on no metrics/error --- go/logic/throttler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 6f9f4bb..0923c7b 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -104,7 +104,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint } } // Got here? No metrics indicates we need throttling. - return false, "", base.NoThrottleReasonHint + return true, "", base.NoThrottleReasonHint } // parseChangelogHeartbeat parses a string timestamp and deduces replication lag From 64083e4705042a1d6a816a5671a61735139fe347 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:40:09 -0500 Subject: [PATCH 158/176] Throttle on no metrics/error --- go/logic/throttler.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 0923c7b..25ad6cf 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -84,6 +84,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint if statusCode != 0 && statusCode != http.StatusOK { return true, this.throttleHttpMessage(int(statusCode)), base.NoThrottleReasonHint } + // Replication lag throttle maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold) lag := atomic.LoadInt64(&this.migrationContext.CurrentLag) @@ -104,7 +105,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint } } // Got here? No metrics indicates we need throttling. - return true, "", base.NoThrottleReasonHint + return false, "", base.NoThrottleReasonHint } // parseChangelogHeartbeat parses a string timestamp and deduces replication lag @@ -288,7 +289,11 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- return false, nil } - collectFunc() + _, err := collectFunc() + if err != nil { + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } + firstThrottlingCollected <- true ticker := time.Tick(100 * time.Millisecond) From ca0b822a3de06345a36326fa10a15bead239adaf Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:41:23 -0500 Subject: [PATCH 159/176] Add comment --- go/logic/throttler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 25ad6cf..6113374 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -291,6 +291,7 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- _, err := collectFunc() if err != nil { + // If an error occurs during the HTTP throttle check, let's throttle to be safe atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } From d31f3aac0c09ecb258a29f1cf0f0bb3d2b4a639c Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:47:56 -0500 Subject: [PATCH 160/176] Add CLI flag for ignoring http errors --- go/base/context.go | 8 ++++++++ go/cmd/gh-ost/main.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/go/base/context.go b/go/base/context.go index 5ebf092..983ae8f 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,6 +119,7 @@ type MigrationContext struct { ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string + ignoreHTTPErrors bool ThrottleCommandedByUser int64 HibernateUntil int64 maxLoad LoadMap @@ -574,6 +575,13 @@ func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string) { this.throttleHTTP = throttleHTTP } +func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) { + this.throttleHTTPMutex.Lock() + defer this.throttleHTTPMutex.Unlock() + + this.ignoreHTTPErrors = ignoreHTTPErrors +} + func (this *MigrationContext) GetMaxLoad() LoadMap { this.throttleMutex.Lock() defer this.throttleMutex.Unlock() diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..bfd852a 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -106,6 +106,7 @@ func main() { throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") + ignoreHTTPErrors := flag.Bool("ignore-http-error", false, "ignore HTTP connection errors during throttle check") heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value") flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered") flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations") @@ -259,6 +260,7 @@ func main() { migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis) migrationContext.SetThrottleQuery(*throttleQuery) migrationContext.SetThrottleHTTP(*throttleHTTP) + migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors) migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() if err := migrationContext.SetupTLS(); err != nil { From 46dabd338b40ef73fb782046b792d3fc1f30d83d Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:48:19 -0500 Subject: [PATCH 161/176] go fmt --- go/base/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/base/context.go b/go/base/context.go index 983ae8f..9a3c577 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,7 +119,7 @@ type MigrationContext struct { ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string - ignoreHTTPErrors bool + ignoreHTTPErrors bool ThrottleCommandedByUser int64 HibernateUntil int64 maxLoad LoadMap From 57cf5f3c90c2e2db50863505362e484f1bb26695 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 15:58:32 -0500 Subject: [PATCH 162/176] add override to ignore http errors --- go/base/context.go | 4 ++-- go/cmd/gh-ost/main.go | 2 +- go/logic/throttler.go | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 9a3c577..52d02dd 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -119,7 +119,7 @@ type MigrationContext struct { ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string - ignoreHTTPErrors bool + IgnoreHTTPErrors bool ThrottleCommandedByUser int64 HibernateUntil int64 maxLoad LoadMap @@ -579,7 +579,7 @@ func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) { this.throttleHTTPMutex.Lock() defer this.throttleHTTPMutex.Unlock() - this.ignoreHTTPErrors = ignoreHTTPErrors + this.IgnoreHTTPErrors = ignoreHTTPErrors } func (this *MigrationContext) GetMaxLoad() LoadMap { diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index bfd852a..33fc6f4 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -106,7 +106,7 @@ func main() { throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") - ignoreHTTPErrors := flag.Bool("ignore-http-error", false, "ignore HTTP connection errors during throttle check") + ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check") heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value") flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered") flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations") diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 6113374..d3d184a 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -291,8 +291,10 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- _, err := collectFunc() if err != nil { - // If an error occurs during the HTTP throttle check, let's throttle to be safe - atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + // If not told to ignore errors, we'll throttle on HTTP connection issues + if !this.migrationContext.IgnoreHTTPErrors { + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } } firstThrottlingCollected <- true From df60fa4204bc7e84a02b5a9b82a19938353149e9 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Wed, 25 Mar 2020 16:06:17 -0500 Subject: [PATCH 163/176] add error logging --- go/logic/throttler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index d3d184a..dd872e7 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -293,6 +293,7 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- if err != nil { // If not told to ignore errors, we'll throttle on HTTP connection issues if !this.migrationContext.IgnoreHTTPErrors { + log.Errorf("errors occurred during HTTP throttle check: %+v", err) atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } } From 7893f8e1c3b575523176705de127e790c2a3ca40 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Mon, 30 Mar 2020 09:53:10 -0500 Subject: [PATCH 164/176] catch error in collectFunc --- go/logic/throttler.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index dd872e7..ace1e49 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -306,7 +306,16 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- return } - if sleep, _ := collectFunc(); sleep { + sleep, err := collectFunc() + if err != nil { + // If not told to ignore errors, we'll throttle on HTTP connection issues + if !this.migrationContext.IgnoreHTTPErrors { + log.Errorf("errors occurred during HTTP throttle check: %+v", err) + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } + } + + if sleep { time.Sleep(1 * time.Second) } } From 5816ede7b3350d3810878d299dd8341f79b4eb2d Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Mon, 30 Mar 2020 09:55:49 -0500 Subject: [PATCH 165/176] add error message --- go/logic/throttler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index ace1e49..be752fe 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -19,20 +19,22 @@ import ( ) var ( - httpStatusMessages map[int]string = map[int]string{ + httpStatusMessages = map[int]string{ 200: "OK", 404: "Not found", 417: "Expectation failed", 429: "Too many requests", 500: "Internal server error", + -1: "Connection error", } // See https://github.com/github/freno/blob/master/doc/http.md - httpStatusFrenoMessages map[int]string = map[int]string{ + httpStatusFrenoMessages = map[int]string{ 200: "OK", 404: "freno: unknown metric", 417: "freno: access forbidden", 429: "freno: threshold exceeded", 500: "freno: internal error", + -1: "freno: connection error", } ) From 2178b5947be80da499fef2c31ac827eaf4920d48 Mon Sep 17 00:00:00 2001 From: Justin Fudally Date: Mon, 30 Mar 2020 10:05:23 -0500 Subject: [PATCH 166/176] remove spammy error --- go/logic/throttler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index be752fe..2cf0d97 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -25,7 +25,7 @@ var ( 417: "Expectation failed", 429: "Too many requests", 500: "Internal server error", - -1: "Connection error", + -1: "Connection error", } // See https://github.com/github/freno/blob/master/doc/http.md httpStatusFrenoMessages = map[int]string{ @@ -295,7 +295,6 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- if err != nil { // If not told to ignore errors, we'll throttle on HTTP connection issues if !this.migrationContext.IgnoreHTTPErrors { - log.Errorf("errors occurred during HTTP throttle check: %+v", err) atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } } @@ -312,7 +311,6 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- if err != nil { // If not told to ignore errors, we'll throttle on HTTP connection issues if !this.migrationContext.IgnoreHTTPErrors { - log.Errorf("errors occurred during HTTP throttle check: %+v", err) atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) } } From 1a8c372947bd441ab95bf6a4b2ea5b1922475eb4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jun 2020 08:39:16 +0300 Subject: [PATCH 167/176] Using golang 1.14 --- .github/workflows/ci.yml | 7 +++---- .github/workflows/replica-tests.yml | 7 +++---- Dockerfile.packaging | 2 +- Dockerfile.test | 2 +- build.sh | 4 ++-- script/ensure-go-installed | 4 ++-- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec8978d..ded3c8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - - name: Set up Go 1.12 + - name: Set up Go 1.14 uses: actions/setup-go@v1 with: - version: 1.12 - id: go + go-version: 1.14 - name: Build run: script/cibuild diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index d775131..31e2052 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -8,13 +8,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - - name: Set up Go 1.12 + - name: Set up Go 1.14 uses: actions/setup-go@v1 with: - version: 1.12 - id: go + go-version: 1.14 - name: migration tests run: script/cibuild-gh-ost-replica-tests diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 214c70c..74aa190 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,6 @@ # -FROM golang:1.12.6 +FROM golang:1.14.4 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index 6abc4a0..3b3bfe5 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.12.1 +FROM golang:1.14.4 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/build.sh b/build.sh index 46db9c2..98633cd 100755 --- a/build.sh +++ b/build.sh @@ -18,8 +18,8 @@ function build { GOOS=$3 GOARCH=$4 - if ! go version | egrep -q 'go(1\.1[234])' ; then - echo "go version must be 1.12 or above" + if ! go version | egrep -q 'go(1\.1[23456])' ; then + echo "go version must be 1.14 or above" exit 1 fi diff --git a/script/ensure-go-installed b/script/ensure-go-installed index c9c2ae8..1abdeb3 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,7 +1,7 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.12.6 -SUPPORTED_GO_VERSIONS='go1.1[234]' +PREFERRED_GO_VERSION=go1.14 +SUPPORTED_GO_VERSIONS='go1.1[23456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815 From fb4aca156751db491d8a83096367c379cdebcc64 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jun 2020 08:49:30 +0300 Subject: [PATCH 168/176] checksums --- script/ensure-go-installed | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/ensure-go-installed b/script/ensure-go-installed index 1abdeb3..32ade46 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,13 +1,13 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.14 +PREFERRED_GO_VERSION=go1.14.4 SUPPORTED_GO_VERSIONS='go1.1[23456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815 +GO_PKG_DARWIN_SHA=b518f21f823759ee30faddb1f623810a432499f050c9338777523d9c8551c62c GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz -GO_PKG_LINUX_SHA=dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c +GO_PKG_LINUX_SHA=aed845e4185a0b2a3c3d5e1d0a35491702c55889192bb9c30e67a3de6849c067 export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" cd $ROOTDIR From 2b71b73285338e96798d2530c8adec9aeb28910f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 28 Jun 2020 08:57:19 +0300 Subject: [PATCH 169/176] Actions/workflows: upload binary artifact --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ded3c8b..3556e1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,3 +17,9 @@ jobs: - name: Build run: script/cibuild + + - name: Upload gh-ost binary artifact + uses: actions/upload-artifact@v1 + with: + name: gh-ost + path: bin/gh-ost From 8eb300bdc8bb183e79e4f50f40288e114aa5c55a Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:52:47 +0300 Subject: [PATCH 170/176] expect 1.14 and above in build scripts; update to readme.md --- README.md | 2 +- build.sh | 2 +- script/ensure-go-installed | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cd6f592..7b2bc1d 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d [Download latest release here](https://github.com/github/gh-ost/releases/latest) -`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either: +`gh-ost` is a Go project; it is built with Go `1.14` and above. To build on your own, use either: - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary. - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost` diff --git a/build.sh b/build.sh index 98633cd..b5d4659 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ function build { GOOS=$3 GOARCH=$4 - if ! go version | egrep -q 'go(1\.1[23456])' ; then + if ! go version | egrep -q 'go(1\.1[456])' ; then echo "go version must be 1.14 or above" exit 1 fi diff --git a/script/ensure-go-installed b/script/ensure-go-installed index 32ade46..98ccdc1 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,7 +1,7 @@ #!/bin/bash PREFERRED_GO_VERSION=go1.14.4 -SUPPORTED_GO_VERSIONS='go1.1[23456]' +SUPPORTED_GO_VERSIONS='go1.1[456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg GO_PKG_DARWIN_SHA=b518f21f823759ee30faddb1f623810a432499f050c9338777523d9c8551c62c From 6c7b4736e15e0fc8c1768395d16cc514c95036a0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 22 Jul 2020 12:33:02 +0300 Subject: [PATCH 171/176] Support a complete ALTER TABLE statement in --alter --- go/cmd/gh-ost/main.go | 23 ++++++--- go/logic/migrator.go | 4 +- go/sql/parser.go | 83 +++++++++++++++++++++++++++------ go/sql/parser_test.go | 106 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 178 insertions(+), 38 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 33fc6f4..57c6ff8 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -14,6 +14,7 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" + "github.com/github/gh-ost/go/sql" _ "github.com/go-sql-driver/mysql" "github.com/outbrain/golib/log" @@ -172,15 +173,25 @@ func main() { log.SetLevel(log.ERROR) } - if migrationContext.DatabaseName == "" { - log.Fatalf("--database must be provided and database name must not be empty") - } - if migrationContext.OriginalTableName == "" { - log.Fatalf("--table must be provided and table name must not be empty") - } if migrationContext.AlterStatement == "" { log.Fatalf("--alter must be provided and statement must not be empty") } + parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement) + + if migrationContext.DatabaseName == "" { + if parser.HasExplicitSchema() { + migrationContext.DatabaseName = parser.GetExplicitSchema() + } else { + log.Fatalf("--database must be provided and database name must not be empty, or --alter must specify database name") + } + } + if migrationContext.OriginalTableName == "" { + if parser.HasExplicitTable() { + migrationContext.OriginalTableName = parser.GetExplicitTable() + } else { + log.Fatalf("--table must be provided and table name must not be empty, or --alter must specify table name") + } + } migrationContext.Noop = !(*executeFlag) if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica { log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive") diff --git a/go/logic/migrator.go b/go/logic/migrator.go index b1a238f..cff64e1 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -62,7 +62,7 @@ const ( // Migrator is the main schema migration flow manager. type Migrator struct { - parser *sql.Parser + parser *sql.AlterTableParser inspector *Inspector applier *Applier eventsStreamer *EventsStreamer @@ -90,7 +90,7 @@ type Migrator struct { func NewMigrator(context *base.MigrationContext) *Migrator { migrator := &Migrator{ migrationContext: context, - parser: sql.NewParser(), + parser: sql.NewAlterTableParser(), ghostTableMigrated: make(chan bool), firstThrottlingCollected: make(chan bool, 3), rowCopyComplete: make(chan error), diff --git a/go/sql/parser.go b/go/sql/parser.go index 447cbcc..d050fcd 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -12,26 +12,47 @@ import ( ) var ( - sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')") - renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) - dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) - renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) + sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')") + renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) + dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) + renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) + alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{ + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`), + } + alterTableExplicitTableRegexps = []*regexp.Regexp{ + regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`), + } ) -type Parser struct { +type AlterTableParser struct { columnRenameMap map[string]string droppedColumns map[string]bool isRenameTable bool + + alterTokens []string + + explicitSchema string + explicitTable string } -func NewParser() *Parser { - return &Parser{ +func NewAlterTableParser() *AlterTableParser { + return &AlterTableParser{ columnRenameMap: make(map[string]string), droppedColumns: make(map[string]bool), } } -func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) { +func NewParserFromAlterStatement(alterStatement string) *AlterTableParser { + parser := NewAlterTableParser() + parser.ParseAlterStatement(alterStatement) + return parser +} + +func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) { terminatingQuote := rune(0) f := func(c rune) bool { switch { @@ -58,13 +79,13 @@ func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []stri return tokens, nil } -func (this *Parser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) { +func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) { strippedStatement = alterStatement strippedStatement = sanitizeQuotesRegexp.ReplaceAllString(strippedStatement, "''") return strippedStatement } -func (this *Parser) parseAlterToken(alterToken string) (err error) { +func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) { { // rename allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1) @@ -97,16 +118,33 @@ func (this *Parser) parseAlterToken(alterToken string) (err error) { return nil } -func (this *Parser) ParseAlterStatement(alterStatement string) (err error) { +func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) { + + for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps { + if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + this.explicitSchema = submatch[1] + this.explicitTable = submatch[2] + alterStatement = submatch[3] + break + } + } + for _, alterTableRegexp := range alterTableExplicitTableRegexps { + if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + this.explicitTable = submatch[1] + alterStatement = submatch[2] + break + } + } alterTokens, _ := this.tokenizeAlterStatement(alterStatement) for _, alterToken := range alterTokens { alterToken = this.sanitizeQuotesFromAlterStatement(alterToken) this.parseAlterToken(alterToken) + this.alterTokens = append(this.alterTokens, alterToken) } return nil } -func (this *Parser) GetNonTrivialRenames() map[string]string { +func (this *AlterTableParser) GetNonTrivialRenames() map[string]string { result := make(map[string]string) for column, renamed := range this.columnRenameMap { if column != renamed { @@ -116,14 +154,29 @@ func (this *Parser) GetNonTrivialRenames() map[string]string { return result } -func (this *Parser) HasNonTrivialRenames() bool { +func (this *AlterTableParser) HasNonTrivialRenames() bool { return len(this.GetNonTrivialRenames()) > 0 } -func (this *Parser) DroppedColumnsMap() map[string]bool { +func (this *AlterTableParser) DroppedColumnsMap() map[string]bool { return this.droppedColumns } -func (this *Parser) IsRenameTable() bool { +func (this *AlterTableParser) IsRenameTable() bool { return this.isRenameTable } +func (this *AlterTableParser) GetExplicitSchema() string { + return this.explicitSchema +} + +func (this *AlterTableParser) HasExplicitSchema() bool { + return this.GetExplicitSchema() != "" +} + +func (this *AlterTableParser) GetExplicitTable() string { + return this.explicitTable +} + +func (this *AlterTableParser) HasExplicitTable() bool { + return this.GetExplicitTable() != "" +} diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 5d38130..626fe75 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -19,7 +19,7 @@ func init() { func TestParseAlterStatement(t *testing.T) { statement := "add column t int, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) @@ -27,7 +27,7 @@ func TestParseAlterStatement(t *testing.T) { func TestParseAlterStatementTrivialRename(t *testing.T) { statement := "add column t int, change ts ts timestamp, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) @@ -37,7 +37,7 @@ func TestParseAlterStatementTrivialRename(t *testing.T) { func TestParseAlterStatementTrivialRenames(t *testing.T) { statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb" - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) @@ -58,7 +58,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { } for _, statement := range statements { - parser := NewParser() + parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) renames := parser.GetNonTrivialRenames() @@ -69,7 +69,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { } func TestTokenizeAlterStatement(t *testing.T) { - parser := NewParser() + parser := NewAlterTableParser() { alterStatement := "add column t int" tokens, _ := parser.tokenizeAlterStatement(alterStatement) @@ -108,7 +108,7 @@ func TestTokenizeAlterStatement(t *testing.T) { } func TestSanitizeQuotesFromAlterStatement(t *testing.T) { - parser := NewParser() + parser := NewAlterTableParser() { alterStatement := "add column e enum('a','b','c')" strippedStatement := parser.sanitizeQuotesFromAlterStatement(alterStatement) @@ -124,7 +124,7 @@ func TestSanitizeQuotesFromAlterStatement(t *testing.T) { func TestParseAlterStatementDroppedColumns(t *testing.T) { { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -132,7 +132,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["b"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop key c_idx, drop column `d`" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -141,7 +141,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["d"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop key c_idx, drop column `d`, drop `e`, drop primary key, drop foreign key fk_1" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -151,7 +151,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { test.S(t).ExpectTrue(parser.droppedColumns["e"]) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, drop bad statement, add column i int" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) @@ -163,38 +163,114 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { func TestParseAlterStatementRenameTable(t *testing.T) { { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectFalse(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "drop column b, rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "engine=innodb rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } { - parser := NewParser() + parser := NewAlterTableParser() statement := "rename as something_else, engine=innodb" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) test.S(t).ExpectTrue(parser.isRenameTable) } } + +func TestParseAlterStatementExplicitTable(t *testing.T) { + + { + parser := NewAlterTableParser() + statement := "drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm with spaces`.`tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm with spaces") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm`.`tbl with spaces` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl with spaces") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table `scm`.tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table scm.`tbl` drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } + { + parser := NewAlterTableParser() + statement := "alter table scm.tbl drop column b" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) + } +} From c9249f2b71ba30d13cbf0ead2717f287786e780f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 23 Jul 2020 11:38:05 +0300 Subject: [PATCH 172/176] Updating and using AlterTableOptions --- go/base/context.go | 7 ++++--- go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 2 +- go/sql/parser.go | 18 ++++++++++++------ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 1030463..cee66ef 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -77,9 +77,10 @@ func NewThrottleCheckResult(throttle bool, reason string, reasonHint ThrottleRea type MigrationContext struct { Uuid string - DatabaseName string - OriginalTableName string - AlterStatement string + DatabaseName string + OriginalTableName string + AlterStatement string + AlterStatementOptions string // anything following the 'ALTER TABLE [schema.]table' from AlterStatement CountTableRows bool ConcurrentCountTableRows bool diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 31729a5..d287d11 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -177,6 +177,7 @@ func main() { log.Fatalf("--alter must be provided and statement must not be empty") } parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement) + migrationContext.AlterStatementOptions = parser.GetAlterStatementOptions() if migrationContext.DatabaseName == "" { if parser.HasExplicitSchema() { diff --git a/go/logic/applier.go b/go/logic/applier.go index 926023f..3b1b9bf 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -190,7 +190,7 @@ func (this *Applier) AlterGhost() error { query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s %s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetGhostTableName()), - this.migrationContext.AlterStatement, + this.migrationContext.AlterStatementOptions, ) this.migrationContext.Log.Infof("Altering ghost table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), diff --git a/go/sql/parser.go b/go/sql/parser.go index d050fcd..f7333a7 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -33,7 +33,8 @@ type AlterTableParser struct { droppedColumns map[string]bool isRenameTable bool - alterTokens []string + alterStatementOptions string + alterTokens []string explicitSchema string explicitTable string @@ -120,22 +121,23 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) { func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) { + this.alterStatementOptions = alterStatement for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps { - if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { this.explicitSchema = submatch[1] this.explicitTable = submatch[2] - alterStatement = submatch[3] + this.alterStatementOptions = submatch[3] break } } for _, alterTableRegexp := range alterTableExplicitTableRegexps { - if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 { + if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { this.explicitTable = submatch[1] - alterStatement = submatch[2] + this.alterStatementOptions = submatch[2] break } } - alterTokens, _ := this.tokenizeAlterStatement(alterStatement) + alterTokens, _ := this.tokenizeAlterStatement(this.alterStatementOptions) for _, alterToken := range alterTokens { alterToken = this.sanitizeQuotesFromAlterStatement(alterToken) this.parseAlterToken(alterToken) @@ -180,3 +182,7 @@ func (this *AlterTableParser) GetExplicitTable() string { func (this *AlterTableParser) HasExplicitTable() bool { return this.GetExplicitTable() != "" } + +func (this *AlterTableParser) GetAlterStatementOptions() string { + return this.alterStatementOptions +} From 731df3cd152656ac21781bbdcf43e397542042e6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 23 Jul 2020 14:04:14 +0300 Subject: [PATCH 173/176] comments --- go/sql/parser.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/sql/parser.go b/go/sql/parser.go index f7333a7..ebb8b38 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -17,13 +17,19 @@ var ( dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{ + // ALTER TABLE `scm`.`tbl` something regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE `scm`.tbl something regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`), + // ALTER TABLE scm.`tbl` something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE scm.tbl something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`), } alterTableExplicitTableRegexps = []*regexp.Regexp{ + // ALTER TABLE `tbl` something regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), + // ALTER TABLE tbl something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`), } ) From ae4dd1867a230c2864538d3e45adbf2a2804c4ff Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 29 Jul 2020 15:06:13 +0300 Subject: [PATCH 174/176] extra unit test checks --- go/sql/parser_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 626fe75..79faa63 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -22,6 +22,7 @@ func TestParseAlterStatement(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) } @@ -30,6 +31,7 @@ func TestParseAlterStatementTrivialRename(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 1) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") @@ -40,6 +42,7 @@ func TestParseAlterStatementTrivialRenames(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 2) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") @@ -61,6 +64,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) renames := parser.GetNonTrivialRenames() test.S(t).ExpectEquals(len(renames), 2) test.S(t).ExpectEquals(renames["i"], "count") @@ -136,6 +140,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) { statement := "drop column b, drop key c_idx, drop column `d`" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectEquals(len(parser.droppedColumns), 2) test.S(t).ExpectTrue(parser.droppedColumns["b"]) test.S(t).ExpectTrue(parser.droppedColumns["d"]) @@ -181,6 +186,7 @@ func TestParseAlterStatementRenameTable(t *testing.T) { statement := "drop column b, rename as something_else" err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectTrue(parser.isRenameTable) } { @@ -208,6 +214,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "") test.S(t).ExpectEquals(parser.explicitTable, "") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -217,6 +224,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -226,6 +234,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -235,6 +244,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm with spaces") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -244,6 +254,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl with spaces") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -253,6 +264,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -262,6 +274,7 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } { @@ -271,6 +284,17 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.explicitSchema, "scm") test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b") test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"})) } + { + parser := NewAlterTableParser() + statement := "alter table scm.tbl drop column b, add index idx(i)" + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.explicitSchema, "scm") + test.S(t).ExpectEquals(parser.explicitTable, "tbl") + test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b, add index idx(i)") + test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b", "add index idx(i)"})) + } } From 25d28855b27bb4519d6951eb236118b1c0e44a15 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 19 Aug 2020 20:21:40 +0200 Subject: [PATCH 175/176] Copy of PR https://github.com/github/gh-ost/pull/861 --- Dockerfile.packaging | 2 +- Dockerfile.test | 2 +- README.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 74aa190..9c5cd29 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,6 @@ # -FROM golang:1.14.4 +FROM golang:1.14.7 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index 3b3bfe5..8f56be3 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.14.4 +FROM golang:1.14.7 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/README.md b/README.md index 7b2bc1d..d4a17e9 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,4 @@ Generally speaking, `master` branch is stable, but only [releases](https://githu - [@shlomi-noach](https://github.com/shlomi-noach) - [@jessbreckenridge](https://github.com/jessbreckenridge) - [@gtowey](https://github.com/gtowey) +- [@timvaillancourt](https://github.com/timvaillancourt) From 42bd9187c1cf93b37438ea6dae9ca0a93d14c863 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 19 Aug 2020 20:27:11 +0200 Subject: [PATCH 176/176] Use 1.14.7 --- script/ensure-go-installed | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/ensure-go-installed b/script/ensure-go-installed index 98ccdc1..baa5bd7 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,13 +1,13 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.14.4 +PREFERRED_GO_VERSION=go1.14.7 SUPPORTED_GO_VERSIONS='go1.1[456]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg -GO_PKG_DARWIN_SHA=b518f21f823759ee30faddb1f623810a432499f050c9338777523d9c8551c62c +GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz -GO_PKG_LINUX_SHA=aed845e4185a0b2a3c3d5e1d0a35491702c55889192bb9c30e67a3de6849c067 +GO_PKG_LINUX_SHA=4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5 export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" cd $ROOTDIR