From 88d6163ce2af234f4ede7eb6b8531da8d7d3cf8b Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 Aug 2017 11:20:58 +0300 Subject: [PATCH 1/6] adding json 5.7 tests --- localtests/json57/create.sql | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 localtests/json57/create.sql diff --git a/localtests/json57/create.sql b/localtests/json57/create.sql new file mode 100644 index 0000000..83dfe79 --- /dev/null +++ b/localtests/json57/create.sql @@ -0,0 +1,21 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + j json, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, '"sometext"'); + insert into gh_ost_test values (null, '{"key":"val"}'); + insert into gh_ost_test values (null, '{"is-it": true, "count": 3, "elements": []}'); +end ;; From a43796d5c579ef6a5d85f0b8259cb1bd71468097 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 Aug 2017 11:50:50 +0300 Subject: [PATCH 2/6] added JSON UPDATE and DELETE tests --- localtests/json57dml/create.sql | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 localtests/json57dml/create.sql diff --git a/localtests/json57dml/create.sql b/localtests/json57dml/create.sql new file mode 100644 index 0000000..2cda51e --- /dev/null +++ b/localtests/json57dml/create.sql @@ -0,0 +1,27 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + updated tinyint not null default 0, + j json, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, i, j) values (null, 11, '"sometext"'); + insert into gh_ost_test (id, i, j) values (null, 13, '{"key":"val"}'); + insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); + + update gh_ost_test set j = '{"updated": 11}', updated = 1 where i = 11 and updated = 0; + update gh_ost_test set j = json_set(j, '$.count', 13), updated = 1 where i = 13 and updated = 0; + delete from gh_ost_test where i = 17; +end ;; From 2174588ebf7abd4d8246c4d4befbd9abaa6612b5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 Aug 2017 11:51:20 +0300 Subject: [PATCH 3/6] Supporting JSON types --- go/logic/inspect.go | 5 +++++ go/sql/builder.go | 4 ++++ go/sql/types.go | 1 + 3 files changed, 10 insertions(+) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 5049193..b0088d5 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -545,6 +545,11 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL columnsList.GetColumn(columnName).Type = sql.DateTimeColumnType } } + if strings.Contains(columnType, "json") { + for _, columnsList := range columnsLists { + columnsList.GetColumn(columnName).Type = sql.JSONColumnType + } + } if strings.HasPrefix(columnType, "enum") { for _, columnsList := range columnsLists { columnsList.GetColumn(columnName).Type = sql.EnumColumnType diff --git a/go/sql/builder.go b/go/sql/builder.go index 0b1ac9e..a60999f 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -38,6 +38,8 @@ func buildColumnsPreparedValues(columns *ColumnList) []string { var token string if column.timezoneConversion != nil { token = fmt.Sprintf("convert_tz(?, '%s', '%s')", column.timezoneConversion.ToTimezone, "+00:00") + } else if column.Type == JSONColumnType { + token = "convert(? using utf8mb4)" } else { token = "?" } @@ -106,6 +108,8 @@ func BuildSetPreparedClause(columns *ColumnList) (result string, err error) { var setToken string if column.timezoneConversion != nil { setToken = fmt.Sprintf("%s=convert_tz(?, '%s', '%s')", EscapeName(column.Name), column.timezoneConversion.ToTimezone, "+00:00") + } else if column.Type == JSONColumnType { + setToken = fmt.Sprintf("%s=convert(? using utf8mb4)", EscapeName(column.Name)) } else { setToken = fmt.Sprintf("%s=?", EscapeName(column.Name)) } diff --git a/go/sql/types.go b/go/sql/types.go index 9f4f8e7..fd6b01d 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -20,6 +20,7 @@ const ( DateTimeColumnType = iota EnumColumnType = iota MediumIntColumnType = iota + JSONColumnType = iota ) const maxMediumintUnsigned int32 = 16777215 From 6a5ace184266493b6279369367b26c913bc926ce Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 Aug 2017 12:02:57 +0300 Subject: [PATCH 4/6] a slighly more interesting test --- localtests/json57dml/create.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localtests/json57dml/create.sql b/localtests/json57dml/create.sql index 2cda51e..da8cd57 100644 --- a/localtests/json57dml/create.sql +++ b/localtests/json57dml/create.sql @@ -22,6 +22,6 @@ begin insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); update gh_ost_test set j = '{"updated": 11}', updated = 1 where i = 11 and updated = 0; - update gh_ost_test set j = json_set(j, '$.count', 13), updated = 1 where i = 13 and updated = 0; + update gh_ost_test set j = json_set(j, '$.count', 13, '$.id', id), updated = 1 where i = 13 and updated = 0; delete from gh_ost_test where i = 17; end ;; From 5a8ed7b6e9a5f6b1d538b099077555cd1972d963 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 Aug 2017 12:10:47 +0300 Subject: [PATCH 5/6] clarifying JSON limitation in docs --- doc/requirements-and-limitations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index c961706..5bfccc1 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -28,7 +28,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - MySQL 5.7 generated columns are not supported. They may be supported in the future. -- MySQL 5.7 `JSON` columns are not supported. They are likely to be supported shortly. +- MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY` - The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md) - The migration key must not include columns with NULL values. This means either: From 7c38a50b664e12b7987e6c72cb72b94e8050fe27 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 Aug 2017 12:12:27 +0300 Subject: [PATCH 6/6] POINT limitation --- doc/requirements-and-limitations.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 5bfccc1..e12f3f0 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -28,6 +28,8 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - MySQL 5.7 generated columns are not supported. They may be supported in the future. +- MySQL 5.7 `POINT` column type is not supported. + - MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY` - The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md)