Move localtests to golang+docker
This commit is contained in:
parent
7320fda848
commit
b45947e974
.github/workflows
.gitignorego.modgo.sumgo
localtests
vendor
35
.github/workflows/replica-tests.yml
vendored
35
.github/workflows/replica-tests.yml
vendored
@ -1,6 +1,6 @@
|
||||
name: migration tests
|
||||
|
||||
on: [pull_request]
|
||||
on:
|
||||
- pull_request
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@ -8,17 +8,30 @@ jobs:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21]
|
||||
tests:
|
||||
- image: mysql:5.7
|
||||
engine: innodb
|
||||
- image: mysql:8.0
|
||||
engine: innodb
|
||||
- image: percona:5.7
|
||||
engine: innodb
|
||||
- image: percona:8.0
|
||||
engine: innodb
|
||||
- image: percona:5.7
|
||||
engine: rocksdb
|
||||
- image: percona:8.0
|
||||
engine: rocksdb
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.17
|
||||
|
||||
- name: migration tests
|
||||
- name: generate mysql environment file
|
||||
env:
|
||||
TEST_MYSQL_VERSION: ${{ matrix.version }}
|
||||
run: script/cibuild-gh-ost-replica-tests
|
||||
TEST_STORAGE_ENGINE: "${{ matrix.tests.engine }}"
|
||||
run: localtests/mysql-env.sh
|
||||
|
||||
- name: run localtests
|
||||
env:
|
||||
TEST_DOCKER_IMAGE: "${{ matrix.tests.image }}"
|
||||
run: docker-compose -f localtests/docker-compose.yml up --abort-on-container-exit --no-log-prefix tests
|
||||
|
||||
|
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,5 +1,6 @@
|
||||
/.gopath/
|
||||
/bin/
|
||||
/libexec/
|
||||
/localtests/mysql.env
|
||||
/.vendor/
|
||||
.idea/
|
||||
|
4
go.mod
4
go.mod
@ -6,10 +6,11 @@ require (
|
||||
github.com/go-ini/ini v1.62.0
|
||||
github.com/go-mysql-org/go-mysql v1.3.0
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
|
||||
github.com/openark/golib v0.0.0-20210531070646-355f37940af8
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
|
||||
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
|
||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
|
||||
golang.org/x/text v0.3.6
|
||||
)
|
||||
|
||||
@ -21,7 +22,6 @@ require (
|
||||
github.com/smartystreets/goconvey v1.6.4 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
|
||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/ini.v1 v1.62.0 // indirect
|
||||
)
|
||||
|
8
go.sum
8
go.sum
@ -17,6 +17,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
|
||||
@ -81,8 +83,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
@ -96,14 +96,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
|
||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
|
104
go/cmd/gh-ost-localtests/main.go
Normal file
104
go/cmd/gh-ost-localtests/main.go
Normal file
@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"github.com/github/gh-ost/go/localtests"
|
||||
)
|
||||
|
||||
var AppVersion string
|
||||
|
||||
func envStringVarOrDefault(envVar, defaultVal string) string {
|
||||
if val := os.Getenv(envVar); val != "" {
|
||||
return val
|
||||
}
|
||||
return defaultVal
|
||||
}
|
||||
|
||||
func main() {
|
||||
// flags
|
||||
var printVersion, testNoop bool
|
||||
var testName string
|
||||
var cnf localtests.Config
|
||||
flag.StringVar(&cnf.Host, "host", localtests.DefaultHost, "mysql host")
|
||||
flag.Int64Var(&cnf.Port, "port", localtests.DefaultPort, "mysql port")
|
||||
flag.StringVar(&cnf.Username, "username", localtests.DefaultUsername, "mysql username")
|
||||
flag.StringVar(&cnf.Password, "password", localtests.DefaultPassword, "mysql password")
|
||||
flag.StringVar(&cnf.TestsDir, "tests-dir", "/etc/localtests", "path to localtests directory")
|
||||
flag.StringVar(&testName, "test", "", "run a single test by name (default: run all tests)")
|
||||
flag.BoolVar(&testNoop, "test-noop", false, "run a single noop migration, eg: --alter='ENGINE=InnoDB'")
|
||||
flag.StringVar(&cnf.StorageEngine, "storage-engine", envStringVarOrDefault("TEST_STORAGE_ENGINE", "innodb"), "mysql storage engine")
|
||||
flag.StringVar(&cnf.GhostBinary, "binary", "gh-ost", "path to gh-ost binary")
|
||||
flag.StringVar(&cnf.MysqlBinary, "mysql-binary", "mysql", "path to mysql binary")
|
||||
flag.BoolVar(&printVersion, "version", false, "print version and exit")
|
||||
flag.Parse()
|
||||
|
||||
// print version
|
||||
if printVersion {
|
||||
fmt.Println(AppVersion)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// connect to replica
|
||||
replica, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true",
|
||||
cnf.Username,
|
||||
cnf.Password,
|
||||
cnf.Host,
|
||||
cnf.Port,
|
||||
))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer replica.Close()
|
||||
|
||||
// connect to primary
|
||||
primary, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true",
|
||||
cnf.Username,
|
||||
cnf.Password,
|
||||
"primary", // TODO: fix me
|
||||
cnf.Port,
|
||||
))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer primary.Close()
|
||||
|
||||
// start tester
|
||||
tester := localtests.NewTester(cnf, primary, replica)
|
||||
if err = tester.WaitForMySQLAvailable(); err != nil {
|
||||
log.Fatalf("Failed to setup MySQL database servers: %+v", err)
|
||||
}
|
||||
|
||||
// find tests
|
||||
var tests []localtests.Test
|
||||
if testNoop {
|
||||
tests = []localtests.Test{
|
||||
{
|
||||
Name: "noop",
|
||||
ExtraArgs: []string{
|
||||
fmt.Sprintf("--alter='ENGINE=%s'", cnf.StorageEngine),
|
||||
},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
tests, err = tester.ReadTests(testName)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read tests: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// run tests
|
||||
for _, test := range tests {
|
||||
log.Println("------------------------------------------------------------------------------------------------------------")
|
||||
log.Printf("Loading test %q at %s/%s", test.Name, cnf.TestsDir, test.Name)
|
||||
if err = tester.RunTest(test); err != nil {
|
||||
log.Fatalf("Failed to run test %s: %+v", test.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
196
go/localtests/test.go
Normal file
196
go/localtests/test.go
Normal file
@ -0,0 +1,196 @@
|
||||
package localtests
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// Test represents a single test.
|
||||
type Test struct {
|
||||
Name string
|
||||
Path string
|
||||
CreateSQLFile string
|
||||
DestroySQLFile string
|
||||
ExtraArgs []string
|
||||
ExpectedFailure string
|
||||
SQLMode *string
|
||||
IgnoreVersions []string
|
||||
ValidateOrderBy string
|
||||
ValidateColumns []string
|
||||
ValidateOrigColumns []string
|
||||
//
|
||||
origPrimaryInfo *MysqlInfo
|
||||
}
|
||||
|
||||
func (test *Test) prepareDBPrimary(primary *sql.DB) (err error) {
|
||||
test.origPrimaryInfo, err = getMysqlHostInfo(primary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if test.SQLMode != nil {
|
||||
if err = setDBGlobalSqlMode(primary, *test.SQLMode); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[%s] sql_mode set to %q on primary", test.Name, *test.SQLMode)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (test *Test) resetDBPrimary(config Config, primary *sql.DB) {
|
||||
if test.SQLMode != nil && test.origPrimaryInfo != nil {
|
||||
log.Printf("[%s] resetting primary to sql_mode: %s", test.Name, test.origPrimaryInfo.SQLMode)
|
||||
if err := setDBGlobalSqlMode(primary, test.origPrimaryInfo.SQLMode); err != nil {
|
||||
log.Printf("[%s] failed to reset primary to sql_mode: %+v", test.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
if test.DestroySQLFile != "" {
|
||||
log.Printf("[%s] running destroy.sql file", test.Name)
|
||||
stdin, stderr, err := execSQLFile(config, test.DestroySQLFile)
|
||||
if err != nil {
|
||||
log.Printf("[%s] failed to destroy test schema %s%s%s: %+v", test.Name, failedEmoji, failedEmoji, failedEmoji, stderr.String())
|
||||
log.Printf("[%s] destroy.sql: %s", test.Name, stdin.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare inits a test and runs a 'mysql' client/shell command to populate
|
||||
// the test schema. The create.sql file is read by golang and passed to
|
||||
// 'mysql' over stdin.
|
||||
func (test *Test) Prepare(config Config, primary *sql.DB) error {
|
||||
if test.CreateSQLFile == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := test.prepareDBPrimary(primary); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stdin, stderr, err := execSQLFile(config, test.CreateSQLFile)
|
||||
if err != nil {
|
||||
log.Printf("[%s] failed to prepare test schema %s%s%s: %+v", test.Name, failedEmoji, failedEmoji, failedEmoji, stderr.String())
|
||||
log.Printf("[%s] create.sql: %s", test.Name, stdin.String())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Migrate runs the migration test.
|
||||
func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) {
|
||||
defer test.resetDBPrimary(config, primary)
|
||||
|
||||
replicaInfo, err := getMysqlHostInfo(replica)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
flags := []string{
|
||||
fmt.Sprintf("--user=%s", config.Username),
|
||||
fmt.Sprintf("--password=%s", config.Password),
|
||||
fmt.Sprintf("--host=%s", config.Host),
|
||||
fmt.Sprintf("--port=%d", config.Port),
|
||||
fmt.Sprintf("--assume-master-host=%s:%d", PrimaryHost, replicaInfo.Port), // TODO: fix this
|
||||
fmt.Sprintf("--database=%s", testDatabase),
|
||||
fmt.Sprintf("--table=%s", testTable),
|
||||
fmt.Sprintf("--chunk-size=%d", testChunkSize),
|
||||
fmt.Sprintf("--default-retries=%d", testDefaultRetries),
|
||||
fmt.Sprintf("--throttle-query=%s", throttleQuery),
|
||||
fmt.Sprintf("--throttle-flag-file=%s", throttleFlagFile),
|
||||
fmt.Sprintf("--serve-socket-file=%s", testSocketFile),
|
||||
fmt.Sprintf("--storage-engine=%s", config.StorageEngine),
|
||||
"--allow-on-master",
|
||||
"--assume-rbr",
|
||||
"--debug",
|
||||
"--exact-rowcount",
|
||||
"--execute",
|
||||
"--initially-drop-old-table",
|
||||
"--initially-drop-ghost-table",
|
||||
"--initially-drop-socket-file",
|
||||
"--stack",
|
||||
"--verbose",
|
||||
}
|
||||
if !flagsSliceContainsAlter(test.ExtraArgs) {
|
||||
test.ExtraArgs = append(test.ExtraArgs, fmt.Sprintf(`--alter=ENGINE=%s`, config.StorageEngine))
|
||||
}
|
||||
if len(test.ExtraArgs) > 0 {
|
||||
flags = append(flags, test.ExtraArgs...)
|
||||
}
|
||||
|
||||
log.Printf("[%s] running gh-ost command with extra args: %+v", test.Name, test.ExtraArgs)
|
||||
|
||||
var output, stderr bytes.Buffer
|
||||
cmd := exec.Command(config.GhostBinary, flags...)
|
||||
cmd.Stderr = &stderr
|
||||
cmd.Stdout = &output
|
||||
|
||||
if err = cmd.Run(); err != nil {
|
||||
if isExpectedFailureOutput(&stderr, test.ExpectedFailure) {
|
||||
return nil
|
||||
}
|
||||
output.Write(stderr.Bytes())
|
||||
log.Printf("[%s] test failed: %+v", test.Name, output.String())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
/*
|
||||
func getPrimaryOrUniqueKey(db *sql.DB, database, table string) (string, error) {
|
||||
return "id", nil // TODO: fix this
|
||||
}
|
||||
*/
|
||||
|
||||
// Validate performs a validation of the migration test results.
|
||||
func (test *Test) Validate(config Config, primary, replica *sql.DB) error {
|
||||
if len(test.ValidateColumns) == 0 || len(test.ValidateOrigColumns) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
primaryKey, err := getPrimaryOrUniqueKey(replica, testDatabase, testTable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var query string
|
||||
var maxPrimaryKeyVal interface{}
|
||||
if maxPrimaryKeyVal == nil {
|
||||
query = fmt.Sprintf("select * from %s.%s limit 10", testDatabase, testTable)
|
||||
} else {
|
||||
query = fmt.Sprintf("select * from %s.%s where %s > %+v limit 10",
|
||||
testDatabase, testTable, primaryKey, maxPrimaryKeyVal,
|
||||
)
|
||||
}
|
||||
var rowMap sqlutils.RowMap
|
||||
err = sqlutils.QueryRowsMap(replica, query, func(m sqlutils.RowMap) error {
|
||||
for _, col := range test.ValidateColumns {
|
||||
if val, found := m[col]; found {
|
||||
rowMap[col] = val
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
values := make([]interface{}, 0)
|
||||
for range test.ValidateOrigColumns {
|
||||
var val interface{}
|
||||
values = append(values, &val)
|
||||
}
|
||||
maxPrimaryKeyVal = values[0]
|
||||
|
||||
for rows.Next() {
|
||||
if err = rows.Scan(values...); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, value := range values {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
log.Printf("[%s] row value for %q col: %d", test.Name, test.ValidateOrigColumns[i], value)
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
return nil
|
||||
}
|
198
go/localtests/tester.go
Normal file
198
go/localtests/tester.go
Normal file
@ -0,0 +1,198 @@
|
||||
package localtests
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/shlex"
|
||||
)
|
||||
|
||||
const (
|
||||
PrimaryHost = "primary"
|
||||
DefaultHost = "replica"
|
||||
DefaultPort int64 = 3306
|
||||
DefaultUsername = "gh-ost"
|
||||
DefaultPassword = "gh-ost"
|
||||
testDatabase = "test"
|
||||
testTable = "gh_ost_test"
|
||||
testSocketFile = "/tmp/gh-ost.test.sock"
|
||||
testChunkSize int64 = 10
|
||||
testDefaultRetries int64 = 3
|
||||
throttleFlagFile = "/tmp/gh-ost-test.ghost.throttle.flag"
|
||||
throttleQuery = "select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc"
|
||||
//
|
||||
failedEmoji = "\u274C"
|
||||
successEmoji = "\u2705"
|
||||
)
|
||||
|
||||
// Config represents the configuration.
|
||||
type Config struct {
|
||||
Host string
|
||||
Port int64
|
||||
Username string
|
||||
Password string
|
||||
GhostBinary string
|
||||
MysqlBinary string
|
||||
StorageEngine string
|
||||
TestsDir string
|
||||
}
|
||||
|
||||
type Tester struct {
|
||||
config Config
|
||||
primary *sql.DB
|
||||
replica *sql.DB
|
||||
}
|
||||
|
||||
func NewTester(config Config, primary, replica *sql.DB) *Tester {
|
||||
return &Tester{
|
||||
config: config,
|
||||
primary: primary,
|
||||
replica: replica,
|
||||
}
|
||||
}
|
||||
|
||||
// WaitForMySQLAvailable waits for MySQL to become ready for
|
||||
// testing on both the primary and replica.
|
||||
func (t *Tester) WaitForMySQLAvailable() error {
|
||||
interval := 2 * time.Second
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-time.After(5 * time.Minute):
|
||||
return errors.New("timed out waiting for mysql")
|
||||
case <-ticker.C:
|
||||
if err := func() error {
|
||||
primaryGTIDExec, err := pingAndGetGTIDExecuted(t.primary, interval)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
replicaGTIDExec, err := pingAndGetGTIDExecuted(t.replica, interval)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !replicaGTIDExec.Contain(primaryGTIDExec) {
|
||||
return fmt.Errorf("Replica/primary GTID not equal: %s != %s",
|
||||
replicaGTIDExec.String(), primaryGTIDExec.String(),
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
log.Printf("Waiting for MySQL primary/replica to init: %+v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("MySQL primary/replica is available %s", successEmoji)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ReadTests reads test configurations from a directory. As single test isee
|
||||
// returned when specificTestName is specified.
|
||||
func (t *Tester) ReadTests(specificTestName string) (tests []Test, err error) {
|
||||
subdirs, err := ioutil.ReadDir(t.config.TestsDir)
|
||||
if err != nil {
|
||||
return tests, err
|
||||
}
|
||||
|
||||
for _, subdir := range subdirs {
|
||||
test := Test{
|
||||
Name: subdir.Name(),
|
||||
Path: filepath.Join(t.config.TestsDir, subdir.Name()),
|
||||
}
|
||||
|
||||
stat, err := os.Stat(test.Path)
|
||||
if err != nil || !stat.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
if specificTestName != "" && !strings.EqualFold(test.Name, specificTestName) {
|
||||
continue
|
||||
}
|
||||
|
||||
test.CreateSQLFile = filepath.Join(test.Path, "create.sql")
|
||||
if _, err = os.Stat(test.CreateSQLFile); err != nil {
|
||||
log.Printf("Failed to find create.sql file %q: %+v", test.CreateSQLFile, err)
|
||||
return tests, err
|
||||
}
|
||||
|
||||
destroySQLFile := filepath.Join(test.Path, "destroy.sql")
|
||||
if _, err = os.Stat(destroySQLFile); err == nil {
|
||||
test.DestroySQLFile = destroySQLFile
|
||||
}
|
||||
|
||||
expectFailureFile := filepath.Join(test.Path, "expect_failure")
|
||||
test.ExpectedFailure, _ = readTestFile(expectFailureFile)
|
||||
|
||||
sqlModeFile := filepath.Join(test.Path, "sql_mode")
|
||||
if sqlMode, err := readTestFile(sqlModeFile); err == nil {
|
||||
test.SQLMode = &sqlMode
|
||||
}
|
||||
|
||||
orderByFile := filepath.Join(test.Path, "order_by")
|
||||
test.ValidateOrderBy, _ = readTestFile(orderByFile)
|
||||
|
||||
origColumnsFile := filepath.Join(test.Path, "orig_columns")
|
||||
if origColumns, err := readTestFile(origColumnsFile); err == nil {
|
||||
origColumns = strings.Replace(origColumns, " ", "", -1)
|
||||
test.ValidateOrigColumns = strings.Split(origColumns, ",")
|
||||
}
|
||||
|
||||
ghostColumnsFile := filepath.Join(test.Path, "ghost_columns")
|
||||
if ghostColumns, err := readTestFile(ghostColumnsFile); err == nil {
|
||||
ghostColumns = strings.Replace(ghostColumns, " ", "", -1)
|
||||
test.ValidateColumns = strings.Split(ghostColumns, ",")
|
||||
}
|
||||
|
||||
extraArgsFile := filepath.Join(test.Path, "extra_args")
|
||||
if _, err = os.Stat(extraArgsFile); err == nil {
|
||||
extraArgsStr, err := readTestFile(extraArgsFile)
|
||||
if err != nil {
|
||||
log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err)
|
||||
return tests, err
|
||||
}
|
||||
if test.ExtraArgs, err = shlex.Split(extraArgsStr); err != nil {
|
||||
log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err)
|
||||
return tests, err
|
||||
}
|
||||
}
|
||||
|
||||
tests = append(tests, test)
|
||||
}
|
||||
|
||||
return tests, err
|
||||
}
|
||||
|
||||
// RunTest prepares and runs a single test.
|
||||
func (t *Tester) RunTest(test Test) (err error) {
|
||||
if err = test.Prepare(t.config, t.primary); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[%s] prepared test %s", test.Name, successEmoji)
|
||||
|
||||
if err = test.Migrate(t.config, t.primary, t.replica); err != nil {
|
||||
log.Printf("[%s] failed to migrate test %s%s%s", test.Name, failedEmoji,
|
||||
failedEmoji, failedEmoji)
|
||||
return err
|
||||
}
|
||||
log.Printf("[%s] successfully migrated test %s", test.Name, successEmoji)
|
||||
|
||||
if err = test.Validate(t.config, t.primary, t.replica); err != nil {
|
||||
log.Printf("[%s] failed to validate test %s%s%s", test.Name, failedEmoji,
|
||||
failedEmoji, failedEmoji)
|
||||
return err
|
||||
}
|
||||
log.Printf("[%s] successfully validated test %s", test.Name, successEmoji)
|
||||
|
||||
return err
|
||||
}
|
135
go/localtests/utils.go
Normal file
135
go/localtests/utils.go
Normal file
@ -0,0 +1,135 @@
|
||||
package localtests
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-mysql-org/go-mysql/mysql"
|
||||
)
|
||||
|
||||
func execSQLFile(config Config, sqlFile string) (stdin, stderr bytes.Buffer, err error) {
|
||||
defaultsFile, err := writeMysqlClientDefaultsFile(config)
|
||||
if err != nil {
|
||||
return stderr, stdin, err
|
||||
}
|
||||
defer os.Remove(defaultsFile)
|
||||
|
||||
flags := []string{
|
||||
fmt.Sprintf("--defaults-file=%s", defaultsFile),
|
||||
fmt.Sprintf("--host=%s", PrimaryHost), // TODO: fix this
|
||||
fmt.Sprintf("--port=%d", config.Port),
|
||||
"--default-character-set=utf8mb4",
|
||||
testDatabase,
|
||||
}
|
||||
|
||||
f, err := os.Open(sqlFile)
|
||||
if err != nil {
|
||||
return stderr, stdin, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
cmd := exec.Command(config.MysqlBinary, flags...)
|
||||
cmd.Stdin = io.TeeReader(f, &stdin)
|
||||
cmd.Stderr = &stderr
|
||||
cmd.Stdout = os.Stdout
|
||||
|
||||
return stdin, stderr, cmd.Run()
|
||||
}
|
||||
|
||||
func flagsSliceContainsAlter(flags []string) bool {
|
||||
for _, flag := range flags {
|
||||
if strings.HasPrefix(flag, "--alter") || strings.HasPrefix(flag, "-alter") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type MysqlInfo struct {
|
||||
Host string
|
||||
Port int64
|
||||
Version string
|
||||
VersionComment string
|
||||
SQLMode string
|
||||
}
|
||||
|
||||
func getMysqlHostInfo(db *sql.DB) (*MysqlInfo, error) {
|
||||
var info MysqlInfo
|
||||
res := db.QueryRow("select @@hostname, @@port, @@version, @@version_comment, @@global.sql_mode")
|
||||
if res.Err() != nil {
|
||||
return nil, res.Err()
|
||||
}
|
||||
err := res.Scan(&info.Host, &info.Port, &info.Version, &info.VersionComment, &info.SQLMode)
|
||||
return &info, err
|
||||
}
|
||||
|
||||
func isExpectedFailureOutput(output io.Reader, expectedFailure string) bool {
|
||||
scanner := bufio.NewScanner(output)
|
||||
for scanner.Scan() {
|
||||
if !strings.Contains(scanner.Text(), "FATAL") {
|
||||
continue
|
||||
} else if strings.Contains(scanner.Text(), expectedFailure) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func pingAndGetGTIDExecuted(db *sql.DB, timeout time.Duration) (*mysql.UUIDSet, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
row := db.QueryRowContext(ctx, "select @@global.gtid_executed")
|
||||
if err := row.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var uuidSet string
|
||||
if err := row.Scan(&uuidSet); err != nil {
|
||||
return nil, err
|
||||
} else if uuidSet == "" {
|
||||
return nil, errors.New("gtid_executed is undefined")
|
||||
}
|
||||
return mysql.ParseUUIDSet(uuidSet)
|
||||
}
|
||||
|
||||
func readTestFile(file string) (string, error) {
|
||||
bytes, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(bytes)), nil
|
||||
}
|
||||
|
||||
func setDBGlobalSqlMode(db *sql.DB, sqlMode string) (err error) {
|
||||
_, err = db.Exec("set @@global.sql_mode=?", sqlMode)
|
||||
return err
|
||||
}
|
||||
|
||||
func writeMysqlClientDefaultsFile(config Config) (string, error) {
|
||||
defaultsFile, err := os.CreateTemp("", "gh-ost-localtests.my.cnf")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer defaultsFile.Close()
|
||||
|
||||
_, err = defaultsFile.Write([]byte(fmt.Sprintf(
|
||||
"[client]\nuser=%s\npassword=%s\n",
|
||||
config.Username, config.Password,
|
||||
)))
|
||||
return defaultsFile.Name(), err
|
||||
}
|
@ -135,6 +135,12 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) {
|
||||
|
||||
func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) {
|
||||
this.alterStatementOptions = alterStatement
|
||||
for _, trimQuote := range []string{`'`, `"`} {
|
||||
if strings.HasPrefix(this.alterStatementOptions, trimQuote) && strings.HasSuffix(this.alterStatementOptions, trimQuote) {
|
||||
this.alterStatementOptions = strings.TrimPrefix(this.alterStatementOptions, trimQuote)
|
||||
this.alterStatementOptions = strings.TrimSuffix(this.alterStatementOptions, trimQuote)
|
||||
}
|
||||
}
|
||||
for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps {
|
||||
if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 {
|
||||
this.explicitSchema = submatch[1]
|
||||
|
@ -6,6 +6,7 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@ -18,13 +19,40 @@ func init() {
|
||||
}
|
||||
|
||||
func TestParseAlterStatement(t *testing.T) {
|
||||
statement := "add column t int, engine=innodb"
|
||||
parser := NewAlterTableParser()
|
||||
err := parser.ParseAlterStatement(statement)
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
|
||||
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
|
||||
test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
|
||||
// plain alter
|
||||
{
|
||||
statement := "add column t int, engine=innodb"
|
||||
parser := NewAlterTableParser()
|
||||
err := parser.ParseAlterStatement(statement)
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
|
||||
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
|
||||
test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
|
||||
}
|
||||
// single-quoted alter
|
||||
{
|
||||
statement := "add column t int, engine=innodb"
|
||||
parser := NewAlterTableParser()
|
||||
err := parser.ParseAlterStatement(fmt.Sprintf(`'%s'`, statement))
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
|
||||
}
|
||||
// single-quoted w/comment alter
|
||||
{
|
||||
statement := "add column t int 'single-quoted comment'"
|
||||
parser := NewAlterTableParser()
|
||||
err := parser.ParseAlterStatement(fmt.Sprintf(`'%s'`, statement))
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
|
||||
}
|
||||
// double-quoted alter
|
||||
{
|
||||
statement := "add column t int, engine=innodb"
|
||||
parser := NewAlterTableParser()
|
||||
err := parser.ParseAlterStatement(fmt.Sprintf(`"%s"`, statement))
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAlterStatementTrivialRename(t *testing.T) {
|
||||
|
22
localtests/Dockerfile
Normal file
22
localtests/Dockerfile
Normal file
@ -0,0 +1,22 @@
|
||||
FROM golang:1.17 AS build
|
||||
LABEL maintainer="github@github.com"
|
||||
|
||||
COPY . /go/src/github.com/github/gh-ost
|
||||
WORKDIR /go/src/github.com/github/gh-ost
|
||||
|
||||
RUN go build -o gh-ost go/cmd/gh-ost/main.go
|
||||
RUN go build -o gh-ost-localtests go/cmd/gh-ost-localtests/main.go
|
||||
|
||||
|
||||
|
||||
FROM debian:buster-slim AS image
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y default-mysql-client
|
||||
RUN rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /go/src/github.com/github/gh-ost/gh-ost /usr/local/bin/gh-ost
|
||||
COPY --from=build /go/src/github.com/github/gh-ost/gh-ost-localtests /usr/local/bin/gh-ost-localtests
|
||||
COPY --from=build /go/src/github.com/github/gh-ost/localtests /etc/localtests
|
||||
|
||||
ENTRYPOINT ["gh-ost-localtests"]
|
25
localtests/docker-compose.yml
Normal file
25
localtests/docker-compose.yml
Normal file
@ -0,0 +1,25 @@
|
||||
version: "3.6"
|
||||
services:
|
||||
tests:
|
||||
build:
|
||||
context: "../"
|
||||
dockerfile: "localtests/Dockerfile"
|
||||
env_file: "mysql.env"
|
||||
depends_on:
|
||||
- "primary"
|
||||
- "replica"
|
||||
primary:
|
||||
image: ${TEST_DOCKER_IMAGE}
|
||||
command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --event-scheduler=ON --log-bin --log-slave-updates --server-id=1"
|
||||
env_file: "mysql.env"
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro"
|
||||
replica:
|
||||
image: ${TEST_DOCKER_IMAGE}
|
||||
command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --log-bin --log-slave-updates --read-only=ON --server-id=2"
|
||||
env_file: "mysql.env"
|
||||
depends_on:
|
||||
- "primary"
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro"
|
||||
- "./init-replica.sql:/docker-entrypoint-initdb.d/02-init-replica.sql:ro"
|
9
localtests/init-replica.sql
Normal file
9
localtests/init-replica.sql
Normal file
@ -0,0 +1,9 @@
|
||||
STOP SLAVE;
|
||||
RESET SLAVE;
|
||||
RESET MASTER;
|
||||
|
||||
CHANGE MASTER TO MASTER_HOST='primary', MASTER_USER='gh-ost', MASTER_PASSWORD='gh-ost', MASTER_PORT=3306,
|
||||
MASTER_AUTO_POSITION=1, MASTER_CONNECT_RETRY=1;
|
||||
|
||||
START SLAVE;
|
||||
SET @@GLOBAL.read_only=ON;
|
5
localtests/init.sql
Normal file
5
localtests/init.sql
Normal file
@ -0,0 +1,5 @@
|
||||
CREATE DATABASE IF NOT EXISTS `test`;
|
||||
|
||||
CREATE USER IF NOT EXISTS `gh-ost`@`%`;
|
||||
SET PASSWORD FOR `gh-ost`@`%` = PASSWORD('gh-ost');
|
||||
GRANT ALL ON *.* TO `gh-ost`@`%`;
|
@ -1 +1 @@
|
||||
--alter='add column `index` int unsigned' \
|
||||
--alter='add column `index` int unsigned'
|
||||
|
15
localtests/mysql-env.sh
Executable file
15
localtests/mysql-env.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
DIR=$(readlink -f $(dirname $0))
|
||||
FILE=$DIR/mysql.env
|
||||
|
||||
(
|
||||
echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true'
|
||||
|
||||
echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}"
|
||||
if [ "$TEST_STORAGE_ENGINE" == "rocksdb" ]; then
|
||||
echo 'INIT_ROCKSDB=true'
|
||||
fi
|
||||
) | tee $FILE
|
||||
|
||||
echo "Wrote env file to $FILE"
|
@ -1 +1 @@
|
||||
--throttle-query='select false' \
|
||||
--throttle-query='select false'
|
||||
|
202
vendor/github.com/google/shlex/COPYING
generated
vendored
Normal file
202
vendor/github.com/google/shlex/COPYING
generated
vendored
Normal file
@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
2
vendor/github.com/google/shlex/README
generated
vendored
Normal file
2
vendor/github.com/google/shlex/README
generated
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
go-shlex is a simple lexer for go that supports shell-style quoting,
|
||||
commenting, and escaping.
|
416
vendor/github.com/google/shlex/shlex.go
generated
vendored
Normal file
416
vendor/github.com/google/shlex/shlex.go
generated
vendored
Normal file
@ -0,0 +1,416 @@
|
||||
/*
|
||||
Copyright 2012 Google Inc. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
Package shlex implements a simple lexer which splits input in to tokens using
|
||||
shell-style rules for quoting and commenting.
|
||||
|
||||
The basic use case uses the default ASCII lexer to split a string into sub-strings:
|
||||
|
||||
shlex.Split("one \"two three\" four") -> []string{"one", "two three", "four"}
|
||||
|
||||
To process a stream of strings:
|
||||
|
||||
l := NewLexer(os.Stdin)
|
||||
for ; token, err := l.Next(); err != nil {
|
||||
// process token
|
||||
}
|
||||
|
||||
To access the raw token stream (which includes tokens for comments):
|
||||
|
||||
t := NewTokenizer(os.Stdin)
|
||||
for ; token, err := t.Next(); err != nil {
|
||||
// process token
|
||||
}
|
||||
|
||||
*/
|
||||
package shlex
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TokenType is a top-level token classification: A word, space, comment, unknown.
|
||||
type TokenType int
|
||||
|
||||
// runeTokenClass is the type of a UTF-8 character classification: A quote, space, escape.
|
||||
type runeTokenClass int
|
||||
|
||||
// the internal state used by the lexer state machine
|
||||
type lexerState int
|
||||
|
||||
// Token is a (type, value) pair representing a lexographical token.
|
||||
type Token struct {
|
||||
tokenType TokenType
|
||||
value string
|
||||
}
|
||||
|
||||
// Equal reports whether tokens a, and b, are equal.
|
||||
// Two tokens are equal if both their types and values are equal. A nil token can
|
||||
// never be equal to another token.
|
||||
func (a *Token) Equal(b *Token) bool {
|
||||
if a == nil || b == nil {
|
||||
return false
|
||||
}
|
||||
if a.tokenType != b.tokenType {
|
||||
return false
|
||||
}
|
||||
return a.value == b.value
|
||||
}
|
||||
|
||||
// Named classes of UTF-8 runes
|
||||
const (
|
||||
spaceRunes = " \t\r\n"
|
||||
escapingQuoteRunes = `"`
|
||||
nonEscapingQuoteRunes = "'"
|
||||
escapeRunes = `\`
|
||||
commentRunes = "#"
|
||||
)
|
||||
|
||||
// Classes of rune token
|
||||
const (
|
||||
unknownRuneClass runeTokenClass = iota
|
||||
spaceRuneClass
|
||||
escapingQuoteRuneClass
|
||||
nonEscapingQuoteRuneClass
|
||||
escapeRuneClass
|
||||
commentRuneClass
|
||||
eofRuneClass
|
||||
)
|
||||
|
||||
// Classes of lexographic token
|
||||
const (
|
||||
UnknownToken TokenType = iota
|
||||
WordToken
|
||||
SpaceToken
|
||||
CommentToken
|
||||
)
|
||||
|
||||
// Lexer state machine states
|
||||
const (
|
||||
startState lexerState = iota // no runes have been seen
|
||||
inWordState // processing regular runes in a word
|
||||
escapingState // we have just consumed an escape rune; the next rune is literal
|
||||
escapingQuotedState // we have just consumed an escape rune within a quoted string
|
||||
quotingEscapingState // we are within a quoted string that supports escaping ("...")
|
||||
quotingState // we are within a string that does not support escaping ('...')
|
||||
commentState // we are within a comment (everything following an unquoted or unescaped #
|
||||
)
|
||||
|
||||
// tokenClassifier is used for classifying rune characters.
|
||||
type tokenClassifier map[rune]runeTokenClass
|
||||
|
||||
func (typeMap tokenClassifier) addRuneClass(runes string, tokenType runeTokenClass) {
|
||||
for _, runeChar := range runes {
|
||||
typeMap[runeChar] = tokenType
|
||||
}
|
||||
}
|
||||
|
||||
// newDefaultClassifier creates a new classifier for ASCII characters.
|
||||
func newDefaultClassifier() tokenClassifier {
|
||||
t := tokenClassifier{}
|
||||
t.addRuneClass(spaceRunes, spaceRuneClass)
|
||||
t.addRuneClass(escapingQuoteRunes, escapingQuoteRuneClass)
|
||||
t.addRuneClass(nonEscapingQuoteRunes, nonEscapingQuoteRuneClass)
|
||||
t.addRuneClass(escapeRunes, escapeRuneClass)
|
||||
t.addRuneClass(commentRunes, commentRuneClass)
|
||||
return t
|
||||
}
|
||||
|
||||
// ClassifyRune classifiees a rune
|
||||
func (t tokenClassifier) ClassifyRune(runeVal rune) runeTokenClass {
|
||||
return t[runeVal]
|
||||
}
|
||||
|
||||
// Lexer turns an input stream into a sequence of tokens. Whitespace and comments are skipped.
|
||||
type Lexer Tokenizer
|
||||
|
||||
// NewLexer creates a new lexer from an input stream.
|
||||
func NewLexer(r io.Reader) *Lexer {
|
||||
|
||||
return (*Lexer)(NewTokenizer(r))
|
||||
}
|
||||
|
||||
// Next returns the next word, or an error. If there are no more words,
|
||||
// the error will be io.EOF.
|
||||
func (l *Lexer) Next() (string, error) {
|
||||
for {
|
||||
token, err := (*Tokenizer)(l).Next()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch token.tokenType {
|
||||
case WordToken:
|
||||
return token.value, nil
|
||||
case CommentToken:
|
||||
// skip comments
|
||||
default:
|
||||
return "", fmt.Errorf("Unknown token type: %v", token.tokenType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tokenizer turns an input stream into a sequence of typed tokens
|
||||
type Tokenizer struct {
|
||||
input bufio.Reader
|
||||
classifier tokenClassifier
|
||||
}
|
||||
|
||||
// NewTokenizer creates a new tokenizer from an input stream.
|
||||
func NewTokenizer(r io.Reader) *Tokenizer {
|
||||
input := bufio.NewReader(r)
|
||||
classifier := newDefaultClassifier()
|
||||
return &Tokenizer{
|
||||
input: *input,
|
||||
classifier: classifier}
|
||||
}
|
||||
|
||||
// scanStream scans the stream for the next token using the internal state machine.
|
||||
// It will panic if it encounters a rune which it does not know how to handle.
|
||||
func (t *Tokenizer) scanStream() (*Token, error) {
|
||||
state := startState
|
||||
var tokenType TokenType
|
||||
var value []rune
|
||||
var nextRune rune
|
||||
var nextRuneType runeTokenClass
|
||||
var err error
|
||||
|
||||
for {
|
||||
nextRune, _, err = t.input.ReadRune()
|
||||
nextRuneType = t.classifier.ClassifyRune(nextRune)
|
||||
|
||||
if err == io.EOF {
|
||||
nextRuneType = eofRuneClass
|
||||
err = nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch state {
|
||||
case startState: // no runes read yet
|
||||
{
|
||||
switch nextRuneType {
|
||||
case eofRuneClass:
|
||||
{
|
||||
return nil, io.EOF
|
||||
}
|
||||
case spaceRuneClass:
|
||||
{
|
||||
}
|
||||
case escapingQuoteRuneClass:
|
||||
{
|
||||
tokenType = WordToken
|
||||
state = quotingEscapingState
|
||||
}
|
||||
case nonEscapingQuoteRuneClass:
|
||||
{
|
||||
tokenType = WordToken
|
||||
state = quotingState
|
||||
}
|
||||
case escapeRuneClass:
|
||||
{
|
||||
tokenType = WordToken
|
||||
state = escapingState
|
||||
}
|
||||
case commentRuneClass:
|
||||
{
|
||||
tokenType = CommentToken
|
||||
state = commentState
|
||||
}
|
||||
default:
|
||||
{
|
||||
tokenType = WordToken
|
||||
value = append(value, nextRune)
|
||||
state = inWordState
|
||||
}
|
||||
}
|
||||
}
|
||||
case inWordState: // in a regular word
|
||||
{
|
||||
switch nextRuneType {
|
||||
case eofRuneClass:
|
||||
{
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
}
|
||||
case spaceRuneClass:
|
||||
{
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
}
|
||||
case escapingQuoteRuneClass:
|
||||
{
|
||||
state = quotingEscapingState
|
||||
}
|
||||
case nonEscapingQuoteRuneClass:
|
||||
{
|
||||
state = quotingState
|
||||
}
|
||||
case escapeRuneClass:
|
||||
{
|
||||
state = escapingState
|
||||
}
|
||||
default:
|
||||
{
|
||||
value = append(value, nextRune)
|
||||
}
|
||||
}
|
||||
}
|
||||
case escapingState: // the rune after an escape character
|
||||
{
|
||||
switch nextRuneType {
|
||||
case eofRuneClass:
|
||||
{
|
||||
err = fmt.Errorf("EOF found after escape character")
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
}
|
||||
default:
|
||||
{
|
||||
state = inWordState
|
||||
value = append(value, nextRune)
|
||||
}
|
||||
}
|
||||
}
|
||||
case escapingQuotedState: // the next rune after an escape character, in double quotes
|
||||
{
|
||||
switch nextRuneType {
|
||||
case eofRuneClass:
|
||||
{
|
||||
err = fmt.Errorf("EOF found after escape character")
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
}
|
||||
default:
|
||||
{
|
||||
state = quotingEscapingState
|
||||
value = append(value, nextRune)
|
||||
}
|
||||
}
|
||||
}
|
||||
case quotingEscapingState: // in escaping double quotes
|
||||
{
|
||||
switch nextRuneType {
|
||||
case eofRuneClass:
|
||||
{
|
||||
err = fmt.Errorf("EOF found when expecting closing quote")
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
}
|
||||
case escapingQuoteRuneClass:
|
||||
{
|
||||
state = inWordState
|
||||
}
|
||||
case escapeRuneClass:
|
||||
{
|
||||
state = escapingQuotedState
|
||||
}
|
||||
default:
|
||||
{
|
||||
value = append(value, nextRune)
|
||||
}
|
||||
}
|
||||
}
|
||||
case quotingState: // in non-escaping single quotes
|
||||
{
|
||||
switch nextRuneType {
|
||||
case eofRuneClass:
|
||||
{
|
||||
err = fmt.Errorf("EOF found when expecting closing quote")
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
}
|
||||
case nonEscapingQuoteRuneClass:
|
||||
{
|
||||
state = inWordState
|
||||
}
|
||||
default:
|
||||
{
|
||||
value = append(value, nextRune)
|
||||
}
|
||||
}
|
||||
}
|
||||
case commentState: // in a comment
|
||||
{
|
||||
switch nextRuneType {
|
||||
case eofRuneClass:
|
||||
{
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
}
|
||||
case spaceRuneClass:
|
||||
{
|
||||
if nextRune == '\n' {
|
||||
state = startState
|
||||
token := &Token{
|
||||
tokenType: tokenType,
|
||||
value: string(value)}
|
||||
return token, err
|
||||
} else {
|
||||
value = append(value, nextRune)
|
||||
}
|
||||
}
|
||||
default:
|
||||
{
|
||||
value = append(value, nextRune)
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
{
|
||||
return nil, fmt.Errorf("Unexpected state: %v", state)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Next returns the next token in the stream.
|
||||
func (t *Tokenizer) Next() (*Token, error) {
|
||||
return t.scanStream()
|
||||
}
|
||||
|
||||
// Split partitions a string into a slice of strings.
|
||||
func Split(s string) ([]string, error) {
|
||||
l := NewLexer(strings.NewReader(s))
|
||||
subStrings := make([]string, 0)
|
||||
for {
|
||||
word, err := l.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return subStrings, nil
|
||||
}
|
||||
return subStrings, err
|
||||
}
|
||||
subStrings = append(subStrings, word)
|
||||
}
|
||||
}
|
5
vendor/modules.txt
vendored
5
vendor/modules.txt
vendored
@ -11,6 +11,9 @@ github.com/go-mysql-org/go-mysql/utils
|
||||
# github.com/go-sql-driver/mysql v1.6.0
|
||||
## explicit; go 1.10
|
||||
github.com/go-sql-driver/mysql
|
||||
# github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
|
||||
## explicit; go 1.13
|
||||
github.com/google/shlex
|
||||
# github.com/openark/golib v0.0.0-20210531070646-355f37940af8
|
||||
## explicit; go 1.16
|
||||
github.com/openark/golib/log
|
||||
@ -37,8 +40,6 @@ github.com/siddontang/go-log/loggers
|
||||
# go.uber.org/atomic v1.7.0
|
||||
## explicit; go 1.13
|
||||
go.uber.org/atomic
|
||||
# golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
|
||||
## explicit; go 1.11
|
||||
# golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
|
||||
## explicit; go 1.11
|
||||
golang.org/x/net/context
|
||||
|
Loading…
Reference in New Issue
Block a user