DB Access Layer Merges: GetTablePattern ... (#103)

Added the following APIs:

ExistKeysPattern()
GetTablePattern()
Key.IsAllKeyPattern()
Go UT, and Benchmark Tests for the same
TranslibDBScriptFail error type to translib/tlerr package

translib/db/db_key.go

@@ -101,6 +101,15 @@ func (k Key) Matches(pattern Key) bool {
 	return true
 }
 
+// IsAllKeyPattern returns true if it is an all key wildcard pattern.
+// (i.e. A key with a single component "*")
+func (k *Key) IsAllKeyPattern() bool {
+	if (len(k.Comp) == 1) && (k.Comp[0] == "*") {
+		return true
+	}
+	return false
+}
+
 // patternMatch checks if the value matches a key pattern.
 // vIndex and pIndex are start positions of value and pattern strings to match.
 // Mimics redis pattern matcher - i.e, glob like pattern matcher which

translib/db/db_keys_pattern.go

@@ -0,0 +1,200 @@
+////////////////////////////////////////////////////////////////////////////////
+//                                                                            //
+//  Copyright 2022 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
+//  its subsidiaries.                                                         //
+//                                                                            //
+//  Licensed under the Apache License, Version 2.0 (the "License");           //
+//  you may not use this file except in compliance with the License.          //
+//  You may obtain a copy of the License at                                   //
+//                                                                            //
+//     http://www.apache.org/licenses/LICENSE-2.0                             //
+//                                                                            //
+//  Unless required by applicable law or agreed to in writing, software       //
+//  distributed under the License is distributed on an "AS IS" BASIS,         //
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  //
+//  See the License for the specific language governing permissions and       //
+//  limitations under the License.                                            //
+//                                                                            //
+////////////////////////////////////////////////////////////////////////////////
+
+package db
+
+import (
+	"time"
+
+	"github.com/Azure/sonic-mgmt-common/translib/tlerr"
+	"github.com/go-redis/redis/v7"
+	"github.com/golang/glog"
+)
+
+////////////////////////////////////////////////////////////////////////////////
+//  Exported Types                                                            //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+//  Exported Functions                                                        //
+////////////////////////////////////////////////////////////////////////////////
+
+// ExistKeysPattern checks if a key pattern exists in a table
+// Note:
+//  1. Statistics do not capture when the CAS Tx Cache results in a quicker
+//     response. This is to avoid mis-interpreting per-connection statistics.
+func (d *DB) ExistKeysPattern(ts *TableSpec, pat Key) (bool, error) {
+
+	var err error
+	var exists bool
+
+	// ExistsKeysPatternHits
+	// Time Start
+	var cacheHit bool
+	var now time.Time
+	var dur time.Duration
+	var stats Stats
+
+	if (d == nil) || (d.client == nil) {
+		return exists, tlerr.TranslibDBConnectionReset{}
+	}
+
+	if d.dbStatsConfig.TimeStats {
+		now = time.Now()
+	}
+
+	if glog.V(3) {
+		glog.Info("ExistKeysPattern: Begin: ", "ts: ", ts, "pat: ", pat)
+	}
+
+	defer func() {
+		if err != nil {
+			glog.Error("ExistKeysPattern: ts: ", ts, " err: ", err)
+		}
+		if glog.V(3) {
+			glog.Info("ExistKeysPattern: End: ts: ", ts, " exists: ", exists)
+		}
+	}()
+
+	// If pseudoDB then follow the path of !IsWriteDisabled with Tx Cache. TBD.
+
+	if !d.Opts.IsWriteDisabled {
+
+		// If Write is enabled, then just call GetKeysPattern() and check
+		// for now.
+
+		// Optimization: Check the DBL CAS Tx Cache for added entries.
+		for k := range d.txTsEntryMap[ts.Name] {
+			key := d.redis2key(ts, k)
+			if key.Matches(pat) {
+				if len(d.txTsEntryMap[ts.Name][k].Field) > 0 {
+					exists = true
+					break
+				}
+				// Removed entries/fields, can't help much, since we'd have
+				// to compare against the DB retrieved keys anyway
+			}
+		}
+
+		if !exists {
+			var getKeys []Key
+			if getKeys, err = d.GetKeysPattern(ts, pat); (err == nil) && len(getKeys) > 0 {
+
+				exists = true
+			}
+		}
+
+	} else if d.dbCacheConfig.PerConnection &&
+		d.dbCacheConfig.isCacheTable(ts.Name) {
+
+		// Check PerConnection cache first, [Found = SUCCESS return]
+		var keys []Key
+		if table, ok := d.cache.Tables[ts.Name]; ok {
+			if keys, ok = table.patterns[d.key2redis(ts, pat)]; ok && len(keys) > 0 {
+
+				exists = true
+				cacheHit = true
+
+			}
+		}
+	}
+
+	// Run Lua script [Found = SUCCESS return]
+	if d.Opts.IsWriteDisabled && !exists {
+
+		//glog.Info("ExistKeysPattern: B4= ", luaScriptExistsKeysPatterns.Hash())
+
+		var luaExists interface{}
+		if luaExists, err = luaScriptExistsKeysPatterns.Run(d.client,
+			[]string{d.key2redis(ts, pat)}).Result(); err == nil {
+
+			if existsString, ok := luaExists.(string); !ok {
+				err = tlerr.TranslibDBScriptFail{
+					Description: "Unexpected response"}
+			} else if existsString == "true" {
+				exists = true
+			} else if existsString != "false" {
+				err = tlerr.TranslibDBScriptFail{Description: existsString}
+			}
+		}
+
+		//glog.Info("ExistKeysPattern: AF= ", luaScriptExistsKeysPatterns.Hash())
+	}
+
+	// Time End, Time, Peak
+	if d.dbStatsConfig.TableStats {
+		stats = d.stats.Tables[ts.Name]
+	} else {
+		stats = d.stats.AllTables
+	}
+
+	stats.Hits++
+	stats.ExistsKeyPatternHits++
+	if cacheHit {
+		stats.ExistsKeyPatternCacheHits++
+	}
+
+	if d.dbStatsConfig.TimeStats {
+		dur = time.Since(now)
+
+		if dur > stats.Peak {
+			stats.Peak = dur
+		}
+		stats.Time += dur
+
+		if dur > stats.ExistsKeyPatternPeak {
+			stats.ExistsKeyPatternPeak = dur
+		}
+		stats.ExistsKeyPatternTime += dur
+	}
+
+	if d.dbStatsConfig.TableStats {
+		d.stats.Tables[ts.Name] = stats
+	} else {
+		d.stats.AllTables = stats
+	}
+
+	return exists, err
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//  Internal Functions                                                        //
+////////////////////////////////////////////////////////////////////////////////
+
+var luaScriptExistsKeysPatterns *redis.Script
+
+func init() {
+	// Register the Lua Script
+	luaScriptExistsKeysPatterns = redis.NewScript(`
+		for i,k in pairs(redis.call('KEYS', KEYS[1])) do
+			return 'true'
+		end
+		return 'false'
+	`)
+
+	// Alternate Lua Script
+	// luaScriptExistsKeysPatterns = redis.NewScript(`
+	// 	if #redis.call('KEYS', KEYS[1]) > 0 then
+	// 		return 'true'
+	// 	else
+	// 		return 'false'
+	// 	end
+	// `)
+
+}

translib/db/db_keys_pattern_test.go

@@ -0,0 +1,128 @@
+////////////////////////////////////////////////////////////////////////////////
+//                                                                            //
+//  Copyright 2022 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
+//  its subsidiaries.                                                         //
+//                                                                            //
+//  Licensed under the Apache License, Version 2.0 (the "License");           //
+//  you may not use this file except in compliance with the License.          //
+//  You may obtain a copy of the License at                                   //
+//                                                                            //
+//     http://www.apache.org/licenses/LICENSE-2.0                             //
+//                                                                            //
+//  Unless required by applicable law or agreed to in writing, software       //
+//  distributed under the License is distributed on an "AS IS" BASIS,         //
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  //
+//  See the License for the specific language governing permissions and       //
+//  limitations under the License.                                            //
+//                                                                            //
+////////////////////////////////////////////////////////////////////////////////
+
+package db
+
+import (
+	"testing"
+)
+
+func BenchmarkExistKeysPattern(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || !exists {
+			b.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
+		}
+	}
+}
+
+func BenchmarkGetKeysPattern(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		if _, e := db.GetKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil {
+			b.Errorf("GetKeysPattern() returns err: %v", e)
+		}
+	}
+}
+
+func TestGetKeysPattern(t *testing.T) {
+	if keys, e := db.GetKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || len(keys) == 0 {
+		t.Errorf("GetKeysPattern() returns len(keys) == 0 || err: %v", e)
+	}
+}
+
+func TestExistKeysPattern(t *testing.T) {
+	if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || !exists {
+		t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
+	}
+}
+
+func TestExistKeysPatternSinglular(t *testing.T) {
+	if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"KEY1"}}); e != nil || !exists {
+		t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
+	}
+}
+
+func TestExistKeysPatternGeneric(t *testing.T) {
+	if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"KEY*"}}); e != nil || !exists {
+		t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
+	}
+}
+
+func TestExistKeysPatternEmpty(t *testing.T) {
+	if exists, e := db.ExistKeysPattern(&TableSpec{Name: "UNLIKELY_23"},
+		Key{Comp: []string{"*"}}); e != nil || exists {
+		t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
+	}
+
+	if exists, e := db.ExistKeysPattern(&ts, Key{Comp: []string{"UNKNOWN"}}); e != nil || exists {
+		t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
+	}
+}
+
+func TestExistKeysPatternRW(t *testing.T) {
+	dbRW, err := newDB(ConfigDB)
+	if err != nil {
+		t.Fatalf("TestExistKeysPatternRW: newDB() for RW fails err = %v\n", err)
+	}
+	t.Cleanup(func() { dbRW.DeleteDB() })
+
+	if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"*"}}); e != nil || !exists {
+		t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
+	}
+}
+
+func TestExistKeysPatternSinglularRW(t *testing.T) {
+	dbRW, err := newDB(ConfigDB)
+	if err != nil {
+		t.Fatalf("TestExistKeysPatternSinglularRW: newDB() for RW fails err = %v\n", err)
+	}
+	t.Cleanup(func() { dbRW.DeleteDB() })
+
+	if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"KEY1"}}); e != nil || !exists {
+		t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
+	}
+}
+
+func TestExistKeysPatternGenericRW(t *testing.T) {
+	dbRW, err := newDB(ConfigDB)
+	if err != nil {
+		t.Fatalf("TestExistKeysPatternGenericRW: newDB() for RW fails err = %v\n", err)
+	}
+	t.Cleanup(func() { dbRW.DeleteDB() })
+
+	if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"KEY*"}}); e != nil || !exists {
+		t.Errorf("ExistKeysPattern() returns !exists || err: %v", e)
+	}
+}
+
+func TestExistKeysPatternEmptyRW(t *testing.T) {
+	dbRW, err := newDB(ConfigDB)
+	if err != nil {
+		t.Fatalf("TestExistKeysPatternEmptyRW: newDB() for RW fails err = %v\n", err)
+	}
+	t.Cleanup(func() { dbRW.DeleteDB() })
+
+	if exists, e := dbRW.ExistKeysPattern(&TableSpec{Name: "UNLIKELY_23"},
+		Key{Comp: []string{"*"}}); e != nil || exists {
+		t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
+	}
+
+	if exists, e := dbRW.ExistKeysPattern(&ts, Key{Comp: []string{"UNKNOWN"}}); e != nil || exists {
+		t.Errorf("ExistKeysPattern() returns exists || err: %v", e)
+	}
+}