From 29a00eec8ebf2d34eb00e46522ea2cbbb21a81ab Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 17 Oct 2020 16:59:51 +0400 Subject: [PATCH 1/5] Lazily decode json document --- document/array.go | 7 ++++- document/create.go | 36 ++++++++++++++++++---- document/create_test.go | 63 +++++++++++++++++++++++++++++++++++++++ document/document.go | 2 +- document/document_test.go | 6 ++-- 5 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 document/create_test.go diff --git a/document/array.go b/document/array.go index 58bf06830..b1ee610b8 100644 --- a/document/array.go +++ b/document/array.go @@ -68,6 +68,11 @@ type ValueBuffer []Value // NewValueBuffer creates a buffer of values. func NewValueBuffer(values ...Value) ValueBuffer { + if len(values) == 0 { + // If called with no values return a non-nil slice. + return ValueBuffer{} + } + return ValueBuffer(values) } @@ -141,7 +146,7 @@ func (vb *ValueBuffer) Copy(a Array) error { return err } - err = vb.Replace(i, NewArrayValue(&buf)) + err = vb.Replace(i, NewArrayValue(buf)) if err != nil { return err } diff --git a/document/create.go b/document/create.go index ce4014b94..57d47e2ae 100644 --- a/document/create.go +++ b/document/create.go @@ -9,16 +9,40 @@ import ( "reflect" "strings" "time" + + "github.com/buger/jsonparser" ) -// NewFromJSON creates a document from a JSON object. -func NewFromJSON(data []byte) (Document, error) { - var fb FieldBuffer - err := fb.UnmarshalJSON(data) +// NewFromJSON creates a document from raw JSON data. +// The returned document will lazily decode the data. +// If data is not a valid json object, calls to Iterate of GetByField will +// return an error. +func NewFromJSON(data []byte) Document { + return &jsonEncodedDocument{data} +} + +type jsonEncodedDocument struct { + data []byte +} + +func (j jsonEncodedDocument) Iterate(fn func(field string, value Value) error) error { + return jsonparser.ObjectEach(j.data, func(key, value []byte, dataType jsonparser.ValueType, offset int) error { + v, err := parseJSONValue(dataType, value) + if err != nil { + return err + } + + return fn(string(key), v) + }) +} + +func (j jsonEncodedDocument) GetByField(field string) (Value, error) { + v, dt, _, err := jsonparser.Get(j.data, field) if err != nil { - return nil, err + return Value{}, err } - return &fb, nil + + return parseJSONValue(dt, v) } // NewFromMap creates a document from a map. diff --git a/document/create_test.go b/document/create_test.go new file mode 100644 index 000000000..fce22532a --- /dev/null +++ b/document/create_test.go @@ -0,0 +1,63 @@ +package document_test + +import ( + "testing" + + "github.com/genjidb/genji/document" + "github.com/stretchr/testify/require" +) + +func TestNewFromJSON(t *testing.T) { + tests := []struct { + name string + data string + expected *document.FieldBuffer + fails bool + }{ + {"empty object", "{}", document.NewFieldBuffer(), false}, + {"empty object, missing closing bracket", "{", nil, true}, + {"classic object", `{"a": 1, "b": true, "c": "hello", "d": [1, 2, 3], "e": {"f": "g"}}`, + document.NewFieldBuffer(). + Add("a", document.NewIntegerValue(1)). + Add("b", document.NewBoolValue(true)). + Add("c", document.NewTextValue("hello")). + Add("d", document.NewArrayValue(document.NewValueBuffer(). + Append(document.NewIntegerValue(1)). + Append(document.NewIntegerValue(2)). + Append(document.NewIntegerValue(3)))). + Add("e", document.NewDocumentValue(document.NewFieldBuffer().Add("f", document.NewTextValue("g")))), + false}, + {"string values", `{"a": "hello ciao"}`, document.NewFieldBuffer().Add("a", document.NewTextValue("hello ciao")), false}, + {"+integer values", `{"a": 1000}`, document.NewFieldBuffer().Add("a", document.NewIntegerValue(1000)), false}, + {"-integer values", `{"a": -1000}`, document.NewFieldBuffer().Add("a", document.NewIntegerValue(-1000)), false}, + {"+float values", `{"a": 10000000000.0}`, document.NewFieldBuffer().Add("a", document.NewDoubleValue(10000000000)), false}, + {"-float values", `{"a": -10000000000.0}`, document.NewFieldBuffer().Add("a", document.NewDoubleValue(-10000000000)), false}, + {"bool values", `{"a": true, "b": false}`, document.NewFieldBuffer().Add("a", document.NewBoolValue(true)).Add("b", document.NewBoolValue(false)), false}, + {"empty arrays", `{"a": []}`, document.NewFieldBuffer().Add("a", document.NewArrayValue(document.NewValueBuffer())), false}, + {"nested arrays", `{"a": [[1, 2]]}`, document.NewFieldBuffer(). + Add("a", document.NewArrayValue( + document.NewValueBuffer(). + Append(document.NewArrayValue( + document.NewValueBuffer(). + Append(document.NewIntegerValue(1)). + Append(document.NewIntegerValue(2)))))), false}, + {"missing comma", `{"a": 1 "b": 2}`, nil, true}, + {"missing closing brackets", `{"a": 1, "b": 2`, nil, true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + d := document.NewFromJSON([]byte(test.data)) + + fb := document.NewFieldBuffer() + err := fb.Copy(d) + + if test.fails { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, *test.expected, *fb) + } + }) + } +} diff --git a/document/document.go b/document/document.go index 648a32d4e..7b9c930ef 100644 --- a/document/document.go +++ b/document/document.go @@ -290,7 +290,7 @@ func (fb *FieldBuffer) Copy(d Document) error { return err } - fb.fields[i].Value = NewArrayValue(&buf) + fb.fields[i].Value = NewArrayValue(buf) } } diff --git a/document/document_test.go b/document/document_test.go index ec5784cda..768fe412f 100644 --- a/document/document_test.go +++ b/document/document_test.go @@ -83,7 +83,6 @@ func TestFieldBuffer(t *testing.T) { }) t.Run("Set", func(t *testing.T) { - tests := []struct { name string data string @@ -118,9 +117,8 @@ func TestFieldBuffer(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var fb document.FieldBuffer - d, err := document.NewFromJSON([]byte(tt.data)) - require.NoError(t, err) - err = fb.Copy(d) + d := document.NewFromJSON([]byte(tt.data)) + err := fb.Copy(d) require.NoError(t, err) p, err := parser.ParsePath(tt.path) require.NoError(t, err) From 2242d4a2d2c0dac6a7b345c0ec7f47a85df59f76 Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 17 Oct 2020 17:27:57 +0400 Subject: [PATCH 2/5] Add benchmark --- document/create_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/document/create_test.go b/document/create_test.go index fce22532a..ab82ce667 100644 --- a/document/create_test.go +++ b/document/create_test.go @@ -61,3 +61,15 @@ func TestNewFromJSON(t *testing.T) { }) } } + +func BenchmarkJSONToDocument(b *testing.B) { + data := []byte(`{"_id":"5f8aefb8e443c6c13afdb305","index":0,"guid":"42c2719e-3371-4b2f-b855-d302a8b7eab0","isActive":true,"balance":"$1,064.79","picture":"http://placehold.it/32x32","age":40,"eyeColor":"blue","name":"Adele Webb","gender":"female","company":"EXTRAGEN","email":"adelewebb@extragen.com","phone":"+1 (964) 409-2397","address":"970 Charles Place, Watrous, Texas, 2522","about":"Amet non do ullamco duis velit sunt esse et cillum nisi mollit ea magna. Tempor ut occaecat proident laborum velit nisi et excepteur exercitation non est labore. Laboris pariatur enim proident et. Qui minim enim et incididunt incididunt adipisicing tempor. Occaecat adipisicing sint ex ut exercitation exercitation voluptate. Laboris adipisicing ut cillum eu cillum est sunt amet Lorem quis pariatur.\r\n","registered":"2016-05-25T10:36:44 -04:00","latitude":64.57112,"longitude":176.136138,"tags":["velit","minim","eiusmod","est","eu","voluptate","deserunt"],"friends":[{"id":0,"name":"Mathis Robertson"},{"id":1,"name":"Cecilia Donaldson"},{"id":2,"name":"Joann Goodwin"}],"greeting":"Hello, Adele Webb! You have 2 unread messages.","favoriteFruit":"apple"}`) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + d := document.NewFromJSON(data) + d.Iterate(func(string, document.Value) error { + return nil + }) + } +} From 0bfce2f43fa39308c2470c57cf18061a864cbbac Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 17 Oct 2020 18:04:38 +0400 Subject: [PATCH 3/5] Fix calls to document.NewFromJSON --- database/table.go | 2 +- sql/query/expr/expr_test.go | 4 +--- sql/query/expr/path_test.go | 6 ++---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/database/table.go b/database/table.go index 515a5ac03..2552d0a90 100644 --- a/database/table.go +++ b/database/table.go @@ -514,7 +514,7 @@ func validateConstraint(d document.Document, c *FieldConstraint) error { } case document.ArrayValue: // if it's an array, we can assume it's a ValueBuffer - buf := parent.V.(*document.ValueBuffer) + buf := parent.V.(document.ValueBuffer) frag := c.Path[len(c.Path)-1] if frag.FieldName != "" { diff --git a/sql/query/expr/expr_test.go b/sql/query/expr/expr_test.go index cbd4da659..bf12df816 100644 --- a/sql/query/expr/expr_test.go +++ b/sql/query/expr/expr_test.go @@ -13,13 +13,11 @@ import ( ) var doc document.Document = func() document.Document { - d, _ := document.NewFromJSON([]byte(`{ + return document.NewFromJSON([]byte(`{ "a": 1, "b": {"foo bar": [1, 2]}, "c": [1, {"foo": "bar"}, [1, 2]] }`)) - - return d }() var stackWithDoc = expr.EvalStack{ diff --git a/sql/query/expr/path_test.go b/sql/query/expr/path_test.go index a52dfd4b3..5730853df 100644 --- a/sql/query/expr/path_test.go +++ b/sql/query/expr/path_test.go @@ -5,7 +5,6 @@ import ( "github.com/genjidb/genji/document" "github.com/genjidb/genji/sql/query/expr" - "github.com/stretchr/testify/require" ) func TestPathExpr(t *testing.T) { @@ -16,7 +15,7 @@ func TestPathExpr(t *testing.T) { }{ {"a", document.NewIntegerValue(1), false}, {"b", func() document.Value { - d, _ := document.NewFromJSON([]byte(`{"foo bar": [1, 2]}`)) + d := document.NewFromJSON([]byte(`{"foo bar": [1, 2]}`)) return document.NewDocumentValue(d) }(), false}, @@ -30,12 +29,11 @@ func TestPathExpr(t *testing.T) { {"d", nullLitteral, false}, } - d, err := document.NewFromJSON([]byte(`{ + d := document.NewFromJSON([]byte(`{ "a": 1, "b": {"foo bar": [1, 2]}, "c": [1, {"foo": "bar"}, [1, 2]] }`)) - require.NoError(t, err) for _, test := range tests { t.Run(test.expr, func(t *testing.T) { From 8882f89dea71b8de5a605f9e7e3a39b8e26dc660 Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 17 Oct 2020 18:18:57 +0400 Subject: [PATCH 4/5] Fix GetByField behaviour --- document/create.go | 3 +++ document/create_test.go | 11 +++++++++++ sql/query/expr/path_test.go | 8 ++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/document/create.go b/document/create.go index 57d47e2ae..e17750cc6 100644 --- a/document/create.go +++ b/document/create.go @@ -38,6 +38,9 @@ func (j jsonEncodedDocument) Iterate(fn func(field string, value Value) error) e func (j jsonEncodedDocument) GetByField(field string) (Value, error) { v, dt, _, err := jsonparser.Get(j.data, field) + if dt == jsonparser.NotExist { + return Value{}, ErrFieldNotFound + } if err != nil { return Value{}, err } diff --git a/document/create_test.go b/document/create_test.go index ab82ce667..d86cf2a88 100644 --- a/document/create_test.go +++ b/document/create_test.go @@ -60,6 +60,17 @@ func TestNewFromJSON(t *testing.T) { } }) } + + t.Run("GetByField", func(t *testing.T) { + d := document.NewFromJSON([]byte(`{"a": 1000}`)) + + v, err := d.GetByField("a") + require.NoError(t, err) + require.Equal(t, document.NewIntegerValue(1000), v) + + v, err = d.GetByField("b") + require.Equal(t, document.ErrFieldNotFound, err) + }) } func BenchmarkJSONToDocument(b *testing.B) { diff --git a/sql/query/expr/path_test.go b/sql/query/expr/path_test.go index 5730853df..c8f3a2677 100644 --- a/sql/query/expr/path_test.go +++ b/sql/query/expr/path_test.go @@ -1,10 +1,12 @@ package expr_test import ( + "encoding/json" "testing" "github.com/genjidb/genji/document" "github.com/genjidb/genji/sql/query/expr" + "github.com/stretchr/testify/require" ) func TestPathExpr(t *testing.T) { @@ -15,8 +17,10 @@ func TestPathExpr(t *testing.T) { }{ {"a", document.NewIntegerValue(1), false}, {"b", func() document.Value { - d := document.NewFromJSON([]byte(`{"foo bar": [1, 2]}`)) - return document.NewDocumentValue(d) + fb := document.NewFieldBuffer() + err := json.Unmarshal([]byte(`{"foo bar": [1, 2]}`), fb) + require.NoError(t, err) + return document.NewDocumentValue(fb) }(), false}, {"b.`foo bar`[0]", document.NewIntegerValue(1), false}, From fa832795913f99f630ba6eeaddaa5a4518cc13ad Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 17 Oct 2020 23:10:14 +0400 Subject: [PATCH 5/5] Fix typo --- document/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/document/create.go b/document/create.go index e17750cc6..07a3e1e5c 100644 --- a/document/create.go +++ b/document/create.go @@ -15,7 +15,7 @@ import ( // NewFromJSON creates a document from raw JSON data. // The returned document will lazily decode the data. -// If data is not a valid json object, calls to Iterate of GetByField will +// If data is not a valid json object, calls to Iterate or GetByField will // return an error. func NewFromJSON(data []byte) Document { return &jsonEncodedDocument{data}