From c5b76e6deb375dd963b53255db74d647d8954e50 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 13 Jan 2022 15:18:34 +0100 Subject: [PATCH 1/2] Add signed and unsigned integer types to forked ason library to fix issue of integers being parsed as float64 by the source binlog parser. This results in larger integers being stored as floats on the target and sent with scientific notation in vstream events. Signed-off-by: Rohit Nayak --- go.mod | 2 +- go.sum | 4 +- go/mysql/binlog_event_json.go | 198 +++++++++++++----- go/mysql/binlog_event_json_test.go | 87 +++++++- .../vreplication/unsharded_init_data.sql | 2 +- .../vreplication/vreplication_test.go | 14 +- .../vreplication/vplayer_flaky_test.go | 23 +- 7 files changed, 249 insertions(+), 81 deletions(-) diff --git a/go.mod b/go.mod index facf8bbf177..44ba27462d5 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,6 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 - github.com/spyzhov/ajson v0.7.2 github.com/stretchr/testify v1.8.1 github.com/tchap/go-patricia v2.3.0+incompatible github.com/tidwall/gjson v1.12.1 @@ -112,6 +111,7 @@ require ( github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 + github.com/rohit-nayak-ps/ajson v0.7.3 golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 modernc.org/sqlite v1.20.3 ) diff --git a/go.sum b/go.sum index e47c7dda131..2c53f6ab37a 100644 --- a/go.sum +++ b/go.sum @@ -690,6 +690,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rohit-nayak-ps/ajson v0.7.3 h1:dJG+5sWk5Q0tn3tl1ltWPU3V9vA0AAsCKMJ3m/CYN1M= +github.com/rohit-nayak-ps/ajson v0.7.3/go.mod h1:ApWx7TjNA9X23L+BVqfKKlasFe1Hknqapgz0Ue9uhi8= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -734,8 +736,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= -github.com/spyzhov/ajson v0.7.2 h1:kyl+ovUoId/RSBbSbCm31xyQvPixA6Sxgvb0eWyt1Ko= -github.com/spyzhov/ajson v0.7.2/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzyqMuVA= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/go/mysql/binlog_event_json.go b/go/mysql/binlog_event_json.go index 82b53311c0f..23e5d748447 100644 --- a/go/mysql/binlog_event_json.go +++ b/go/mysql/binlog_event_json.go @@ -23,7 +23,7 @@ import ( "vitess.io/vitess/go/vt/log" - "github.com/spyzhov/ajson" + "github.com/rohit-nayak-ps/ajson" querypb "vitess.io/vitess/go/vt/proto/query" ) @@ -153,7 +153,7 @@ func (jh *BinlogJSON) register(typ jsonDataType, Plugin jsonPlugin) { func (jh *BinlogJSON) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { Plugin := jh.plugins[typ] if Plugin == nil { - return nil, fmt.Errorf("Plugin not found for type %d", typ) + return nil, fmt.Errorf("plugin not found for type %d", typ) } return Plugin.getNode(typ, data, pos) } @@ -316,59 +316,157 @@ type intPlugin struct { var _ jsonPlugin = (*intPlugin)(nil) -func (ih intPlugin) getVal(typ jsonDataType, data []byte, pos int) (value float64) { +func (ipl intPlugin) getVal(typ jsonDataType, data []byte, pos int) (value int64) { var val uint64 - var val2 float64 - size := ih.sizes[typ] + var val2 int64 + size := ipl.sizes[typ] for i := 0; i < size; i++ { val = val + uint64(data[pos+i])<<(8*i) } switch typ { case jsonInt16: - val2 = float64(int16(val)) - case jsonUint16: - val2 = float64(uint16(val)) + val2 = int64(int16(val)) case jsonInt32: - val2 = float64(int32(val)) - case jsonUint32: - val2 = float64(uint32(val)) + val2 = int64(int32(val)) case jsonInt64: - val2 = float64(int64(val)) - case jsonUint64: - val2 = float64(val) - case jsonDouble: - val2 = math.Float64frombits(val) + val2 = int64(val) } return val2 } -func (ih intPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { - val := ih.getVal(typ, data, pos) - node = ajson.NumericNode("", val) +func (ipl intPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { + val := ipl.getVal(typ, data, pos) + node = ajson.IntegerNode("", val) return node, nil } func newIntPlugin() *intPlugin { - ih := &intPlugin{ + ipl := &intPlugin{ info: &jsonPluginInfo{ name: "Int", - types: []jsonDataType{jsonInt64, jsonInt32, jsonInt16, jsonUint16, jsonUint32, jsonUint64, jsonDouble}, + types: []jsonDataType{jsonInt64, jsonInt32, jsonInt16}, + }, + sizes: make(map[jsonDataType]int), + } + ipl.sizes = map[jsonDataType]int{ + jsonInt64: 8, + jsonInt32: 4, + jsonInt16: 2, + } + for _, typ := range ipl.info.types { + binlogJSON.register(typ, ipl) + } + return ipl +} + +//endregion + +//region uint plugin + +func init() { + newUintPlugin() +} + +type uintPlugin struct { + info *jsonPluginInfo + sizes map[jsonDataType]int +} + +var _ jsonPlugin = (*uintPlugin)(nil) + +func (upl uintPlugin) getVal(typ jsonDataType, data []byte, pos int) (value uint64) { + var val uint64 + var val2 uint64 + size := upl.sizes[typ] + for i := 0; i < size; i++ { + val = val + uint64(data[pos+i])<<(8*i) + } + switch typ { + case jsonUint16: + val2 = uint64(uint16(val)) + case jsonUint32: + val2 = uint64(uint32(val)) + case jsonUint64: + val2 = val + } + return val2 +} + +func (upl uintPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { + val := upl.getVal(typ, data, pos) + node = ajson.UnsignedIntegerNode("", val) + return node, nil +} + +func newUintPlugin() *uintPlugin { + upl := &uintPlugin{ + info: &jsonPluginInfo{ + name: "Uint", + types: []jsonDataType{jsonUint16, jsonUint32, jsonUint64}, }, sizes: make(map[jsonDataType]int), } - ih.sizes = map[jsonDataType]int{ + upl.sizes = map[jsonDataType]int{ jsonUint64: 8, - jsonInt64: 8, jsonUint32: 4, - jsonInt32: 4, jsonUint16: 2, - jsonInt16: 2, + } + for _, typ := range upl.info.types { + binlogJSON.register(typ, upl) + } + return upl +} + +//endregion + +//region float plugin + +func init() { + newFloatPlugin() +} + +type floatPlugin struct { + info *jsonPluginInfo + sizes map[jsonDataType]int +} + +var _ jsonPlugin = (*floatPlugin)(nil) + +func (flp floatPlugin) getVal(typ jsonDataType, data []byte, pos int) (value float64) { + var val uint64 + var val2 float64 + size := flp.sizes[typ] + for i := 0; i < size; i++ { + val = val + uint64(data[pos+i])<<(8*i) + } + switch typ { + case jsonDouble: + val2 = math.Float64frombits(val) + } + return val2 +} + +func (flp floatPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { + val := flp.getVal(typ, data, pos) + node = ajson.NumericNode("", val) + return node, nil +} + +func newFloatPlugin() *floatPlugin { + fp := &floatPlugin{ + info: &jsonPluginInfo{ + name: "Float", + types: []jsonDataType{jsonDouble}, + }, + sizes: make(map[jsonDataType]int), + } + fp.sizes = map[jsonDataType]int{ jsonDouble: 8, } - for _, typ := range ih.info.types { - binlogJSON.register(typ, ih) + for _, typ := range fp.info.types { + binlogJSON.register(typ, fp) } - return ih + return fp } //endregion @@ -385,7 +483,7 @@ type literalPlugin struct { var _ jsonPlugin = (*literalPlugin)(nil) -func (lh literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (lpl literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { val := jsonDataLiteral(data[pos]) switch val { case jsonNullLiteral: @@ -401,14 +499,14 @@ func (lh literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *a } func newLiteralPlugin() *literalPlugin { - lh := &literalPlugin{ + lpl := &literalPlugin{ info: &jsonPluginInfo{ name: "Literal", types: []jsonDataType{jsonLiteral}, }, } - binlogJSON.register(jsonLiteral, lh) - return lh + binlogJSON.register(jsonLiteral, lpl) + return lpl } //endregion @@ -427,7 +525,7 @@ var _ jsonPlugin = (*opaquePlugin)(nil) // other types are stored as catch-all opaque types: documentation on these is scarce. // we currently know about (and support) date/time/datetime/decimal. -func (oh opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (opl opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { dataType := data[pos] start := 3 // account for length of stored value end := start + 8 // all currently supported opaque data types are 8 bytes in size @@ -484,14 +582,14 @@ func (oh opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj } func newOpaquePlugin() *opaquePlugin { - oh := &opaquePlugin{ + opl := &opaquePlugin{ info: &jsonPluginInfo{ name: "Opaque", types: []jsonDataType{jsonOpaque}, }, } - binlogJSON.register(jsonOpaque, oh) - return oh + binlogJSON.register(jsonOpaque, opl) + return opl } //endregion @@ -508,7 +606,7 @@ type stringPlugin struct { var _ jsonPlugin = (*stringPlugin)(nil) -func (sh stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (spl stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { size, pos := readVariableLength(data, pos) node = ajson.StringNode("", string(data[pos:pos+size])) @@ -516,14 +614,14 @@ func (sh stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj } func newStringPlugin() *stringPlugin { - sh := &stringPlugin{ + spl := &stringPlugin{ info: &jsonPluginInfo{ name: "String", types: []jsonDataType{jsonString}, }, } - binlogJSON.register(jsonString, sh) - return sh + binlogJSON.register(jsonString, spl) + return spl } //endregion @@ -542,7 +640,7 @@ var _ jsonPlugin = (*arrayPlugin)(nil) // arrays are stored thus: // | type_identifier(one of [2,3]) | elem count | obj size | list of offsets+lengths of values | actual values | -func (ah arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (apl arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { jlog("JSON Array %s, len %d", jsonDataTypeToString(uint(typ)), len(data)) var nodes []*ajson.Node var elem *ajson.Node @@ -565,15 +663,15 @@ func (ah arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajs } func newArrayPlugin() *arrayPlugin { - ah := &arrayPlugin{ + apl := &arrayPlugin{ info: &jsonPluginInfo{ name: "Array", types: []jsonDataType{jsonSmallArray, jsonLargeArray}, }, } - binlogJSON.register(jsonSmallArray, ah) - binlogJSON.register(jsonLargeArray, ah) - return ah + binlogJSON.register(jsonSmallArray, apl) + binlogJSON.register(jsonLargeArray, apl) + return apl } //endregion @@ -592,7 +690,7 @@ var _ jsonPlugin = (*objectPlugin)(nil) // objects are stored thus: // | type_identifier(0/1) | elem count | obj size | list of offsets+lengths of keys | list of offsets+lengths of values | actual keys | actual values | -func (oh objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (opl objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { jlog("JSON Type is %s, len %d", jsonDataTypeToString(uint(typ)), len(data)) // "large" decides number of bytes used to specify element count and total object size: 4 bytes for large, 2 for small @@ -640,15 +738,15 @@ func (oh objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj } func newObjectPlugin() *objectPlugin { - oh := &objectPlugin{ + opl := &objectPlugin{ info: &jsonPluginInfo{ name: "Object", types: []jsonDataType{jsonSmallObject, jsonLargeObject}, }, } - binlogJSON.register(jsonSmallObject, oh) - binlogJSON.register(jsonLargeObject, oh) - return oh + binlogJSON.register(jsonSmallObject, opl) + binlogJSON.register(jsonLargeObject, opl) + return opl } //endregion diff --git a/go/mysql/binlog_event_json_test.go b/go/mysql/binlog_event_json_test.go index 711965386ed..5cc61440084 100644 --- a/go/mysql/binlog_event_json_test.go +++ b/go/mysql/binlog_event_json_test.go @@ -18,50 +18,65 @@ package mysql import ( "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/require" ) func TestJSONTypes(t *testing.T) { + // most of these test cases have been taken from open source java/python adapters + // like https://github.com/shyiko/mysql-binlog-connector-java/pull/119/files testcases := []struct { + name string data []byte expected string isMap bool }{{ + name: "null", data: []byte{}, expected: `null`, }, { + name: "map, string value", data: []byte{0, 1, 0, 14, 0, 11, 0, 1, 0, 12, 12, 0, 97, 1, 98}, expected: `{"a":"b"}`, }, { + name: "map, int value", data: []byte{0, 1, 0, 12, 0, 11, 0, 1, 0, 5, 2, 0, 97}, expected: `{"a":2}`, }, { + name: "map, object value", data: []byte{0, 1, 0, 29, 0, 11, 0, 4, 0, 0, 15, 0, 97, 115, 100, 102, 1, 0, 14, 0, 11, 0, 3, 0, 5, 123, 0, 102, 111, 111}, expected: `{"asdf":{"foo":123}}`, }, { + name: "list of ints", data: []byte{2, 2, 0, 10, 0, 5, 1, 0, 5, 2, 0}, expected: `[1,2]`, }, { + name: "list of maps", data: []byte{0, 4, 0, 60, 0, 32, 0, 1, 0, 33, 0, 1, 0, 34, 0, 2, 0, 36, 0, 2, 0, 12, 38, 0, 12, 40, 0, 12, 42, 0, 2, 46, 0, 97, 99, 97, 98, 98, 99, 1, 98, 1, 100, 3, 97, 98, 99, 2, 0, 14, 0, 12, 10, 0, 12, 12, 0, 1, 120, 1, 121}, expected: `{"a":"b","c":"d","ab":"abc","bc":["x","y"]}`, isMap: true, }, { + name: "list with one string", data: []byte{2, 1, 0, 37, 0, 12, 8, 0, 0, 4, 104, 101, 114, 101}, expected: `["here"]`, }, { + name: "list varied", data: []byte{2, 3, 0, 37, 0, 12, 13, 0, 2, 18, 0, 12, 33, 0, 4, 104, 101, 114, 101, 2, 0, 15, 0, 12, 10, 0, 12, 12, 0, 1, 73, 2, 97, 109, 3, 33, 33, 33}, expected: `["here",["I","am"],"!!!"]`, }, { + name: "string", data: []byte{12, 13, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103}, expected: `"scalar string"`, }, { + name: "map, long string value", data: []byte{0, 1, 0, 149, 0, 11, 0, 6, 0, 12, 17, 0, 115, 99, 111, 112, 101, 115, 130, 1, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 69, 65, 65, 65, 65, 65, 65, 69, 65, 65, 65, 65, 65, 65, 56, 65, 65, 65, 66, 103, 65, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 67, 65, 65, 65, 65, 65, 65, 65, 65, 65, 84, 216, 142, 184}, expected: `{"scopes":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAEAAAAAAEAAAAAA8AAABgAAAAAABAAAACAAAAAAAAA"}`, }, { // repeat the same string 10 times, to test the case where length of string // requires 2 bytes to store + name: "long string", data: []byte{12, 130, 1, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103, @@ -75,89 +90,139 @@ func TestJSONTypes(t *testing.T) { 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103}, expected: `"scalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar string"`, }, { + name: "bool true", data: []byte{4, 1}, expected: `true`, }, { + name: "bool false", data: []byte{4, 2}, expected: `false`, }, { + name: "bool null", data: []byte{4, 0}, expected: `null`, }, { + name: "uint16 max", + data: []byte{6, 255, 255}, + expected: `65535`, + }, { + name: "uint32 max", + data: []byte{8, 255, 255, 255, 255}, + expected: `4294967295`, + }, { + name: "uint64 max", + data: []byte{10, 255, 255, 255, 255, 255, 255, 255, 255}, + expected: `18446744073709551615`, + }, { + name: "int16 -1", data: []byte{5, 255, 255}, expected: `-1`, }, { - data: []byte{6, 1, 0}, - expected: `1`, + name: "int32 -1", + data: []byte{7, 255, 255, 255, 255}, + expected: `-1`, + }, { + name: "int64 -1", + data: []byte{9, 255, 255, 255, 255, 255, 255, 255, 255}, + expected: `-1`, }, { + name: "int16 max", data: []byte{5, 255, 127}, expected: `32767`, }, { + name: "int32 max", + data: []byte{7, 255, 255, 255, 127}, + expected: `2147483647`, + }, { + name: "int64 max", + data: []byte{9, 255, 255, 255, 255, 255, 255, 255, 127}, + expected: `9223372036854775807`, + }, { + name: "uint16/1", + data: []byte{6, 1, 0}, + expected: `1`, + }, { + name: "int32/32768", data: []byte{7, 0, 128, 0, 0}, expected: `32768`, }, { + name: "int16/neg", data: []byte{5, 0, 128}, expected: `-32768`, }, { + name: "int32/neg", data: []byte{7, 255, 127, 255, 255}, expected: `-32769`, }, { - data: []byte{7, 255, 255, 255, 127}, - expected: `2.147483647e+09`, - }, { + name: "uint32", data: []byte{8, 0, 128, 0, 0}, expected: `32768`, }, { + name: "int64", data: []byte{9, 0, 0, 0, 128, 0, 0, 0, 0}, - expected: `2.147483648e+09`, + expected: `2147483648`, }, { + name: "int32/neg", data: []byte{7, 0, 0, 0, 128}, - expected: `-2.147483648e+09`, + expected: `-2147483648`, }, { + name: "int64/neg", data: []byte{9, 255, 255, 255, 127, 255, 255, 255, 255}, - expected: `-2.147483649e+09`, + expected: `-2147483649`, }, { + name: "uint64", data: []byte{10, 255, 255, 255, 255, 255, 255, 255, 255}, - expected: `1.8446744073709552e+19`, + expected: `18446744073709551615`, }, { + name: "int64/neg", data: []byte{9, 0, 0, 0, 0, 0, 0, 0, 128}, - expected: `-9.223372036854776e+18`, + expected: `-9223372036854775808`, }, { + name: "double", data: []byte{11, 110, 134, 27, 240, 249, 33, 9, 64}, expected: `3.14159`, }, { + name: "empty map", data: []byte{0, 0, 0, 4, 0}, expected: `{}`, }, { + name: "empty list", data: []byte{2, 0, 0, 4, 0}, expected: `[]`, }, { // opaque, datetime + name: "datetime", data: []byte{15, 12, 8, 0, 0, 0, 25, 118, 31, 149, 25}, expected: `"2015-01-15 23:24:25.000000"`, }, { // opaque, time + name: "time", data: []byte{15, 11, 8, 0, 0, 0, 25, 118, 1, 0, 0}, expected: `"23:24:25.000000"`, }, { // opaque, time + name: "time2", data: []byte{15, 11, 8, 192, 212, 1, 25, 118, 1, 0, 0}, expected: `"23:24:25.120000"`, }, { // opaque, date + name: "date", data: []byte{15, 10, 8, 0, 0, 0, 0, 0, 30, 149, 25}, expected: `"2015-01-15"`, }, { // opaque, decimal + name: "decimal", data: []byte{15, 246, 8, 13, 4, 135, 91, 205, 21, 4, 210}, expected: `1.234567891234e+08`, }, { // opaque, bit field. Not yet implemented. + name: "bitfield: unimplemented", data: []byte{15, 16, 2, 202, 254}, expected: `opaque type 16 is not supported yet, data [2 202 254]`, }} for _, tc := range testcases { - t.Run(tc.expected, func(t *testing.T) { + name := fmt.Sprintf("%s (%s)", tc.name, tc.expected) + t.Run(name, func(t *testing.T) { val, err := getJSONValue(tc.data) if err != nil { require.Equal(t, tc.expected, err.Error()) diff --git a/go/test/endtoend/vreplication/unsharded_init_data.sql b/go/test/endtoend/vreplication/unsharded_init_data.sql index b12aaa9bf79..019945609db 100644 --- a/go/test/endtoend/vreplication/unsharded_init_data.sql +++ b/go/test/endtoend/vreplication/unsharded_init_data.sql @@ -1,6 +1,6 @@ insert into customer(cid, name, typ, sport, meta) values(1, 'Jøhn "❤️" Rizzolo',1,'football,baseball','{}'); insert into customer(cid, name, typ, sport, meta) values(2, 'Paül','soho','cricket',convert(x'7b7d' using utf8mb4)); -insert into customer(cid, name, typ, sport) values(3, 'ringo','enterprise',''); +insert into customer(cid, name, typ, sport, meta) values(3, 'ringo','enterprise','',null); insert into merchant(mname, category) values('Monoprice', 'eléctronics'); insert into merchant(mname, category) values('newegg', 'elec†ronics'); insert into product(pid, description) values(1, 'keyböard ⌨️'); diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 1875ef6d8f5..4858dbbc749 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -736,9 +736,11 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl query := "select cid from customer" require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", query, query)) - insertQuery1 := "insert into customer(cid, name) values(1001, 'tempCustomer1')" - matchInsertQuery1 := "insert into customer(cid, `name`) values (:vtg1, :vtg2)" - require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1)) + insertQuery1 := "insert into customer(cid, name, meta) values(1001, 'tempCustomer1', '{\"a\": 1629849600, \"b\": 930701976723823}')" + + matchInsertQuery0 := "insert into customer(cid, `name`) values (:vtg1, :vtg2)" + matchInsertQuery1 := "insert into customer(cid, `name`, meta) values (:vtg1, :vtg2, :vtg3)" + validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1) // confirm that the backticking of table names in the routing rules works tbls := []string{"Lead", "Lead-1"} @@ -800,12 +802,12 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl require.Contains(t, output, "'customer.bmd5'") insertQuery1 = "insert into customer(cid, name) values(1002, 'tempCustomer5')" - require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1)) + require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery0)) // both inserts go into 80-, this tests the edge-case where a stream (-80) has no relevant new events after the previous switch insertQuery1 = "insert into customer(cid, name) values(1003, 'tempCustomer6')" - require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery1, matchInsertQuery1)) + require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery1, matchInsertQuery0)) insertQuery1 = "insert into customer(cid, name) values(1004, 'tempCustomer7')" - require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery1, matchInsertQuery1)) + require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery1, matchInsertQuery0)) waitForNoWorkflowLag(t, vc, targetKs, workflow) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index f508c632c9f..931b99ee6df 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -27,13 +27,16 @@ import ( "testing" "time" - "github.com/spyzhov/ajson" + "vitess.io/vitess/go/mysql" + + "github.com/rohit-nayak-ps/ajson" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/log" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" qh "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/queryhistory" ) @@ -1512,21 +1515,21 @@ func TestPlayerTypes(t *testing.T) { }} if enableJSONColumnTesting { testcases = append(testcases, testcase{ - input: "insert into vitess_json(val1,val2,val3,val4,val5) values (null,'{}','123','{\"a\":[42,100]}', '{\"foo\":\"bar\"}')", + input: "insert into vitess_json(val1,val2,val3,val4,val5) values (null,'{}','1629849600','{\"a\":[42,-1,3.1415,-128,127,-9223372036854775808,9223372036854775807,18446744073709551615]}', '{\"foo\":\"bar\"}')", output: "insert into vitess_json(id,val1,val2,val3,val4,val5) values (1," + - "convert(null using utf8mb4)," + "convert('{}' using utf8mb4)," + "convert('123' using utf8mb4)," + - "convert('{\\\"a\\\":[42,100]}' using utf8mb4)," + "convert('{\\\"foo\\\":\\\"bar\\\"}' using utf8mb4))", + "convert(null using utf8mb4)," + "convert('{}' using utf8mb4)," + "convert('1629849600' using utf8mb4)," + + "convert('{\\\"a\\\":[42,-1,3.1415,-128,127,-9223372036854775808,9223372036854775807,18446744073709551615]}' using utf8mb4)," + "convert('{\\\"foo\\\":\\\"bar\\\"}' using utf8mb4))", table: "vitess_json", data: [][]string{ - {"1", "", "{}", "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, + {"1", "", "{}", "1629849600", `{"a": [42, -1, 3.1415, -128, 127, -9223372036854775808, 9223372036854775807, 18446744073709551615]}`, `{"foo": "bar"}`}, }, }) testcases = append(testcases, testcase{ - input: "update vitess_json set val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4)", - output: "update vitess_json set val1=convert(null using utf8mb4), val2=convert('{}' using utf8mb4), val3=convert('123' using utf8mb4), val4=convert('{\\\"a\\\":[98,123]}' using utf8mb4), val5=convert('{}' using utf8mb4) where id=1", + input: "update vitess_json set val4 = '{\"a\": [-9223372036854775808, -2147483648]}', val5 = convert(x'7b7d' using utf8mb4)", + output: "update vitess_json set val1=convert(null using utf8mb4), val2=convert('{}' using utf8mb4), val3=convert('1629849600' using utf8mb4), val4=convert('{\\\"a\\\":[-9223372036854775808,-2147483648]}' using utf8mb4), val5=convert('{}' using utf8mb4) where id=1", table: "vitess_json", data: [][]string{ - {"1", "", "{}", "123", `{"a": [98, 123]}`, `{}`}, + {"1", "", "{}", "1629849600", `{"a": [-9223372036854775808, -2147483648]}`, `{}`}, }, }) } From 196f8f0fbc43ecc4defe8392f598fa881d507229 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 16 Mar 2023 13:16:47 +0100 Subject: [PATCH 2/2] Use go.mod reference to the ajson module to make it easier to switch to the upstream module once changes are made there Signed-off-by: Rohit Nayak --- go.mod | 4 +++- go.sum | 4 ++-- go/mysql/binlog_event_json.go | 2 +- .../vttablet/tabletmanager/vreplication/vplayer_flaky_test.go | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 44ba27462d5..8a2a3f839e2 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 - github.com/rohit-nayak-ps/ajson v0.7.3 + github.com/spyzhov/ajson v0.8.0 golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 modernc.org/sqlite v1.20.3 ) @@ -216,3 +216,5 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) + +replace github.com/spyzhov/ajson v0.8.0 => github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c diff --git a/go.sum b/go.sum index 2c53f6ab37a..3c33c669e84 100644 --- a/go.sum +++ b/go.sum @@ -690,8 +690,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rohit-nayak-ps/ajson v0.7.3 h1:dJG+5sWk5Q0tn3tl1ltWPU3V9vA0AAsCKMJ3m/CYN1M= -github.com/rohit-nayak-ps/ajson v0.7.3/go.mod h1:ApWx7TjNA9X23L+BVqfKKlasFe1Hknqapgz0Ue9uhi8= +github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c h1:Y/4qcogoZA2WUtLWMk/yXfJSpaIG3mK3r9Lw4kaARL4= +github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzyqMuVA= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/go/mysql/binlog_event_json.go b/go/mysql/binlog_event_json.go index 23e5d748447..e055b8866ca 100644 --- a/go/mysql/binlog_event_json.go +++ b/go/mysql/binlog_event_json.go @@ -23,7 +23,7 @@ import ( "vitess.io/vitess/go/vt/log" - "github.com/rohit-nayak-ps/ajson" + "github.com/spyzhov/ajson" querypb "vitess.io/vitess/go/vt/proto/query" ) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 931b99ee6df..60378c8924a 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -29,7 +29,7 @@ import ( "vitess.io/vitess/go/mysql" - "github.com/rohit-nayak-ps/ajson" + "github.com/spyzhov/ajson" "github.com/stretchr/testify/require" "vitess.io/vitess/go/vt/log"