From 2f441547c3c78337043763cf7cbf67dbfd9955cd Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 13 Mar 2021 13:41:38 +0400 Subject: [PATCH 1/3] Build index upon creation --- database/catalog.go | 23 +++++++++++++++++-- database/catalog_test.go | 48 +++++++++++++++++++++++++++++----------- database/transaction.go | 7 +++++- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/database/catalog.go b/database/catalog.go index ba95915dd..498fe815a 100644 --- a/database/catalog.go +++ b/database/catalog.go @@ -171,7 +171,22 @@ func (c *Catalog) CreateIndex(tx *Transaction, opts IndexInfo) error { return err } - return tx.getIndexStore().Insert(opts) + err = tx.getIndexStore().Insert(opts) + if err != nil { + return err + } + + idx, err := c.GetIndex(tx, opts.IndexName) + if err != nil { + return err + } + + tb, err := c.GetTable(tx, opts.TableName) + if err != nil { + return err + } + + return c.buildIndex(tx, idx, tb) } // GetIndex returns an index by name. @@ -289,7 +304,11 @@ func (c *Catalog) ReIndex(tx *Transaction, indexName string) error { return err } - return tb.Iterate(func(d document.Document) error { + return c.buildIndex(tx, idx, tb) +} + +func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error { + return table.Iterate(func(d document.Document) error { v, err := idx.Info.Path.GetValueFromDocument(d) if err == document.ErrFieldNotFound { return nil diff --git a/database/catalog_test.go b/database/catalog_test.go index 4aa32cc7d..fca8116d1 100644 --- a/database/catalog_test.go +++ b/database/catalog_test.go @@ -287,25 +287,55 @@ func TestCatalogCreate(t *testing.T) { } func TestTxCreateIndex(t *testing.T) { - t.Run("Should create an index and return it", func(t *testing.T) { + t.Run("Should create an index, build it and return it", func(t *testing.T) { db, cleanup := newTestDB(t) defer cleanup() catalog := db.Catalog() update(t, db, func(tx *database.Transaction) error { - return catalog.CreateTable(tx, "test", nil) + err := catalog.CreateTable(tx, "test", nil) + if err != nil { + return err + } + + tb, err := tx.GetTable("test") + require.NoError(t, err) + + for i := int64(0); i < 10; i++ { + _, err = tb.Insert(document.NewFieldBuffer(). + Add("a", document.NewIntegerValue(i)). + Add("b", document.NewIntegerValue(i*10)), + ) + require.NoError(t, err) + } + + return nil }) clone := catalog.Clone() update(t, db, func(tx *database.Transaction) error { err := catalog.CreateIndex(tx, database.IndexInfo{ - IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"), + IndexName: "idx_a", TableName: "test", Path: parsePath(t, "a"), }) require.NoError(t, err) - idx, err := tx.GetIndex("idxFoo") + idx, err := tx.GetIndex("idx_a") require.NoError(t, err) require.NotNil(t, idx) + + var i int + err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error { + var buf bytes.Buffer + err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i))) + require.NoError(t, err) + enc := buf.Bytes() + require.Equal(t, enc, v) + i++ + return nil + }) + require.Equal(t, 10, i) + require.NoError(t, err) + return errDontCommit }) @@ -491,7 +521,7 @@ func TestCatalogReIndex(t *testing.T) { require.Equal(t, clone, catalog) }) - t.Run("Should reindex the right index", func(t *testing.T) { + t.Run("Should reindex the index", func(t *testing.T) { db, cleanup := newTestDB(t) defer cleanup() @@ -523,14 +553,6 @@ func TestCatalogReIndex(t *testing.T) { idx, err = tx.GetIndex("b") require.NoError(t, err) - i = 0 - err = idx.AscendGreaterOrEqual(document.Value{Type: document.IntegerValue}, func(val, key []byte) error { - i++ - return nil - }) - require.NoError(t, err) - require.Zero(t, i) - return errDontCommit }) diff --git a/database/transaction.go b/database/transaction.go index 1d719cfbc..2dbad2cba 100644 --- a/database/transaction.go +++ b/database/transaction.go @@ -132,7 +132,12 @@ func (tx *Transaction) DropTable(name string) error { // CreateIndex creates an index with the given name. // If it already exists, returns ErrIndexAlreadyExists. func (tx *Transaction) CreateIndex(opts IndexInfo) error { - return tx.db.catalog.CreateIndex(tx, opts) + err := tx.db.catalog.CreateIndex(tx, opts) + if err != nil { + return err + } + + return tx.ReIndex(opts.IndexName) } // GetIndex returns an index by name. From 55c7bbefdbdafa6dc116133aeb025f77e86f2377 Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 13 Mar 2021 13:44:16 +0400 Subject: [PATCH 2/3] Improve error message --- database/catalog.go | 7 ++++++- database/index.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/database/catalog.go b/database/catalog.go index 498fe815a..a5e3522fd 100644 --- a/database/catalog.go +++ b/database/catalog.go @@ -317,7 +317,12 @@ func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error { return err } - return idx.Set(v, d.(document.Keyer).RawKey()) + err = idx.Set(v, d.(document.Keyer).RawKey()) + if err != nil { + return fmt.Errorf("error while building the index: %w", err) + } + + return nil }) } diff --git a/database/index.go b/database/index.go index 6c474157f..a1384d4a0 100644 --- a/database/index.go +++ b/database/index.go @@ -17,7 +17,7 @@ const ( var ( // ErrIndexDuplicateValue is returned when a value is already associated with a key - ErrIndexDuplicateValue = errors.New("duplicate") + ErrIndexDuplicateValue = errors.New("duplicate value") ) // An Index associates encoded values with keys. From d1d0519ea8e927d8c86cb3d83d3a865af6880281 Mon Sep 17 00:00:00 2001 From: Asdine El Hrychy Date: Sat, 13 Mar 2021 13:55:05 +0400 Subject: [PATCH 3/3] Fix reindex tests --- query/reindex_test.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/query/reindex_test.go b/query/reindex_test.go index e5a491db6..0f0548ea6 100644 --- a/query/reindex_test.go +++ b/query/reindex_test.go @@ -32,16 +32,35 @@ func TestReIndex(t *testing.T) { CREATE TABLE test1; CREATE TABLE test2; - INSERT INTO test1(a, b) VALUES (1, 'a'), (2, 'b'); - INSERT INTO test2(a, b) VALUES (3, 'c'), (4, 'd'); - CREATE INDEX idx_test1_a ON test1(a); CREATE INDEX idx_test1_b ON test1(b); CREATE INDEX idx_test2_a ON test2(a); CREATE INDEX idx_test2_b ON test2(b); + + INSERT INTO test1(a, b) VALUES (1, 'a'), (2, 'b'); + INSERT INTO test2(a, b) VALUES (3, 'c'), (4, 'd'); `) require.NoError(t, err) + // truncate all indexes + err = db.Update(func(tx *genji.Tx) error { + c := tx.DB().Catalog() + for _, idxName := range c.ListIndexes("") { + idx, err := c.GetIndex(tx.Transaction, idxName) + if err != nil { + return err + } + + err = idx.Truncate() + if err != nil { + return err + } + } + + return nil + }) + require.NoError(t, err) + err = db.Exec(test.query) if test.fails { require.Error(t, err) @@ -50,9 +69,7 @@ func TestReIndex(t *testing.T) { require.NoError(t, err) err = db.View(func(tx *genji.Tx) error { - idxList := tx.ListIndexes() - - for _, idxName := range idxList { + for _, idxName := range tx.ListIndexes() { idx, err := tx.GetIndex(idxName) require.NoError(t, err) @@ -82,5 +99,4 @@ func TestReIndex(t *testing.T) { require.NoError(t, err) }) } - }