octoleo/gh-ost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Tim Vaillancourt 2022-12-10 20:14:41 +01:00
parent ad19d1e9c3
commit 115fcfb9e8
7 changed files with 98 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,6 +1,6 @@
/.gopath/
/bin/
/libexec/
/localtests/mysql.env
/localtests/tests.env
/.vendor/
.idea/

0
go/cmd/gh-ost-localtests/main.go → go/cmd/gh-ost-tester/main.go
View File

95
go/localtests/test.go
View File

@ -8,6 +8,10 @@ import (
"log"
"os"
"os/exec"
"strings"
"sync"
"github.com/openark/golib/sqlutils"
)
// Test represents a single test.
@ -133,11 +137,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) {
fmt.Printf("::group::%s gh-ost output\n", test.Name)
}
if err = cmd.Start(); err != nil {
return err
}
err = cmd.Wait()
err = cmd.Run()
if isGitHubActions() {
fmt.Println("::endgroup::")
@ -145,6 +145,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) {
if err != nil {
if isExpectedFailureOutput(&stderr, test.ExpectedFailure) {
log.Printf("[%s] test got expected failure: %s", test.ExpectedFailure)
return nil
}
log.Printf("[%s] test failed: %+v", test.Name, stderr.String())
@ -152,61 +153,61 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) {
return err
}
/*
func getPrimaryOrUniqueKey(db *sql.DB, database, table string) (string, error) {
func getTablePrimaryKey(db *sql.DB, database, table string) (string, error) {
return "id", nil // TODO: fix this
}
*/
type validationResult struct {
Source string
Rows sqlutils.RowMap
}
// Validate performs a validation of the migration test results.
func (test *Test) Validate(config Config, primary, replica *sql.DB) error {
func (test *Test) Validate(config Config, db *sql.DB) error {
if len(test.ValidateColumns) == 0 || len(test.ValidateOrigColumns) == 0 {
return nil
}
/*
primaryKey, err := getPrimaryOrUniqueKey(replica, testDatabase, testTable)
if err != nil {
return err
}
primaryKey, err := getTablePrimaryKey(db, testDatabase, testTable)
if err != nil {
return err
}
var query string
var maxPrimaryKeyVal interface{}
if maxPrimaryKeyVal == nil {
query = fmt.Sprintf("select * from %s.%s limit 10", testDatabase, testTable)
} else {
query = fmt.Sprintf("select * from %s.%s where %s > %+v limit 10",
testDatabase, testTable, primaryKey, maxPrimaryKeyVal,
)
}
var rowMap sqlutils.RowMap
err = sqlutils.QueryRowsMap(replica, query, func(m sqlutils.RowMap) error {
for _, col := range test.ValidateColumns {
if val, found := m[col]; found {
rowMap[col] = val
}
limit := 10
orderBy := primaryKey
if test.ValidateOrderBy != "" {
orderBy = test.ValidateOrderBy
}
outChan := make(chan validationResult, 2)
getTableMap := func(wg *sync.WaitGroup, database, table string, columns []string, outChan chan validationResult) {
defer wg.Done()
query := fmt.Sprintf("select %s from %s.%s order by %s limit %d",
strings.Join(columns, ", "),
database, table,
orderBy, limit,
)
err := sqlutils.QueryRowsMap(db, query, func(m sqlutils.RowMap) error {
outChan <- validationResult{
Source: table,
Rows: m,
}
return nil
})
values := make([]interface{}, 0)
for range test.ValidateOrigColumns {
var val interface{}
values = append(values, &val)
if err != nil {
log.Printf("[%s] failed to validate table %s: %+v", test.Name, table, err)
}
maxPrimaryKeyVal = values[0]
}
for rows.Next() {
if err = rows.Scan(values...); err != nil {
return err
}
for i, value := range values {
if value == nil {
continue
}
log.Printf("[%s] row value for %q col: %d", test.Name, test.ValidateOrigColumns[i], value)
}
}
*/
var wg sync.WaitGroup
go getTableMap(&wg, testDatabase, testTable, test.ValidateColumns, outChan)
go getTableMap(&wg, testDatabase, fmt.Sprintf("_%s_del", testTable), test.ValidateOrigColumns, outChan)
wg.Add(2)
wg.Wait()
for result := range outChan {
log.Printf("[%s] result for %s: %+v", test.Name, result.Source, result.Rows)
}
return nil
}

12
go/localtests/tester.go
View File

@ -114,8 +114,10 @@ func (t *Tester) ReadTests(specificTestName string) (tests []Test, err error) {
for _, subdir := range subdirs {
test := Test{
Name: subdir.Name(),
Path: filepath.Join(t.config.TestsDir, subdir.Name()),
Name: subdir.Name(),
Path: filepath.Join(t.config.TestsDir, subdir.Name()),
ValidateColumns: []string{"*"},
ValidateOrigColumns: []string{"*"},
}
stat, err := os.Stat(test.Path)
@ -147,7 +149,9 @@ func (t *Tester) ReadTests(specificTestName string) (tests []Test, err error) {
}
orderByFile := filepath.Join(test.Path, "order_by")
test.ValidateOrderBy, _ = readTestFile(orderByFile)
if validateOrderBy, err := readTestFile(orderByFile); err == nil {
test.ValidateOrderBy = validateOrderBy
}
origColumnsFile := filepath.Join(test.Path, "orig_columns")
if origColumns, err := readTestFile(origColumnsFile); err == nil {
@ -194,7 +198,7 @@ func (t *Tester) RunTest(test Test) (err error) {
}
log.Printf("[%s] successfully migrated test %s", test.Name, successEmoji)
if err = test.Validate(t.config, t.primary, t.replica); err != nil {
if err = test.Validate(t.config, t.replica); err != nil {
log.Printf("[%s] failed to validate test %s%s%s", test.Name, failedEmoji,
failedEmoji, failedEmoji)
return err

6
localtests/Dockerfile
View File

@ -5,7 +5,7 @@ COPY . /go/src/github.com/github/gh-ost
WORKDIR /go/src/github.com/github/gh-ost
RUN go build -o gh-ost go/cmd/gh-ost/main.go
RUN go build -o gh-ost-localtests go/cmd/gh-ost-localtests/main.go
RUN go build -o gh-ost-tester go/cmd/gh-ost-tester/main.go
@ -16,7 +16,7 @@ RUN apt-get install -y default-mysql-client
RUN rm -rf /var/lib/apt/lists/*
COPY --from=build /go/src/github.com/github/gh-ost/gh-ost /usr/local/bin/gh-ost
COPY --from=build /go/src/github.com/github/gh-ost/gh-ost-localtests /usr/local/bin/gh-ost-localtests
COPY --from=build /go/src/github.com/github/gh-ost/gh-ost-tester /usr/local/bin/gh-ost-tester
COPY --from=build /go/src/github.com/github/gh-ost/localtests /etc/localtests
ENTRYPOINT ["gh-ost-localtests"]
ENTRYPOINT ["gh-ost-tester"]

8
localtests/docker-compose.yml
View File

@ -4,22 +4,24 @@ services:
build:
context: "../"
dockerfile: "localtests/Dockerfile"
env_file: "mysql.env"
env_file: "tests.env"
depends_on:
- "primary"
- "replica"
primary:
image: ${TEST_DOCKER_IMAGE}
command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --event-scheduler=ON --log-bin --log-slave-updates --server-id=1"
env_file: "mysql.env"
env_file: "tests.env"
volumes:
- "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro"
replica:
image: ${TEST_DOCKER_IMAGE}
command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --log-bin --log-slave-updates --read-only=ON --server-id=2"
env_file: "mysql.env"
env_file: "tests.env"
depends_on:
- "primary"
ports:
- "3306:3306"
volumes:
- "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro"
- "./init-replica.sql:/docker-entrypoint-initdb.d/02-init-replica.sql:ro"

48
localtests/tests.sh
View File

@ -9,29 +9,47 @@ if [ -z "$TEST_DOCKER_IMAGE" ]; then
exit 1
fi
# generate mysql.env file for containers
[ ! -z "$GITHUB_ACTION" ] && echo "::group::generate mysql env"
(
echo "GITHUB_ACTION=${GITHUB_ACTION}"
echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true'
echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}"
[ "$TEST_STORAGE_ENGINE" == "rocksdb" ] && echo 'INIT_ROCKSDB=true'
) | tee $LOCALTESTS_DIR/mysql.env
echo "Wrote env file to $LOCALTESTS_DIR/mysql.env"
[ ! -z "$GITHUB_ACTION" ] && echo "::endgroup::"
if [ -z "$TEST_STORAGE_ENGINE" ]; then
TEST_STORAGE_ENGINE=innodb
fi
# generate env file for containers
generate_test_env() {
[ ! -z "$GITHUB_ACTION" ] && echo "::group::generate mysql env"
(
echo "GITHUB_ACTION=${GITHUB_ACTION}"
echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true'
echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}"
[ "$TEST_STORAGE_ENGINE" == "rocksdb" ] && echo 'INIT_ROCKSDB=true'
) | tee $LOCALTESTS_DIR/tests.env
echo "Wrote env file to $LOCALTESTS_DIR/tests.env"
[ ! -z "$GITHUB_ACTION" ] && echo "::endgroup::"
return 0
}
# prebuild the test docker image
prebuild_docker_image() {
echo "::group::docker image build"
docker-compose -f localtests/docker-compose.yml build
echo "::endgroup::"
}
###
generate_test_env
# conditional pre-build
EXTRA_UP_FLAGS=
if [ -z "$GITHUB_ACTION" ]; then
EXTRA_UP_FLAGS="--build"
else
echo "::group::docker image build"
docker-compose -f localtests/docker-compose.yml build
echo "::endgroup::"
prebuild_docker_image
fi
# this will start the test container and the
# mysql primary/replica w/docker-compose
# start test container and mysql primary/replica
# with docker-compose
docker-compose -f localtests/docker-compose.yml up \
--abort-on-container-exit \
--no-log-prefix \