Skip to content

Commit

Permalink
fix bugs and enhance Transformer (#35)
Browse files Browse the repository at this point in the history
Optimize transformer for GET performance
For any request URI and its children, table-name & db-key are queried for the first time and cached internally by the transformer infra. This cache block will be subsequently referred by the infra to avoid redundant callback invocations while the yang model tree is being traversed for GET operation. The cache block is local & valid for a single transaction.
Data from Redis DB is queried only once for a table & cached in the infra for further reference. This cache is local & valid for a single transaction
On-change, sample based subscription support for gNMI
On receipt of subscription request uri, the transformer infra translates it to corresponding Redis Db-table and key, using YANG extensions and provides this info along with other subscription parameters to the translib, which takes care of sending notifications on-change of table/key. YANG extensions introduced : "subscribe-preference", "subscribe-on-change", "subscribe-min-interval".
rpc callback support
RPC statements defined in both OpenConfig and SONiC yang can be annotated with YANG extension “rpc-callback” to invoke the callback to take an action for the given rpc node
YANG default value support
Infra has been enhanced to fill the oc yang defaults by referring internal yang-metadata for terminal nodes, if it is not present in the request payload. These defaults will be pushed along with other payload data when a resource is created.
RFC compliance support - resource check etc.
Parent resource is checked for CRUD and GET requests for the incoming URI, before processing the request. If the parent resource is not available in DB, the transaction is rejected with 404 / "Resource not found" error.
Cascade delete to de-configure all related nodes
Cascade delete enables dependent config clean up for a DELETE request, based on SONiC yang table dependency.
YANG extension "cascade-delete" can be annotated to a node that can be deleted along with all dependent nodes.
value xfmr callback
The value-xfmr callback supports data conversion before writing into the DB for CRUD request and after reading from DB for GET request. Value-xfmr annotated for a leaf will also be invoked, for all other leaves having a leaf-ref to it.
The value-xfmr callback is valid only for the SONIC yang.
Operation and tables ordering, dependent tables watch list
For a set of DB table/instances, grouped by operation, resulting from processing a request URI, infra supports the following :
CRUD Operation ordering,
sequencing tables before operating based on sonic-yang/Redis dependencies
providing translib, with a list of Redis-tables to watch for changes, when performing operations on dependent tables.
Yang model capability
Provide yang module name, organization and version information to translib in order to serve model capability request.
Miscellaneous bug fixes
  • Loading branch information
kwangsuk authored Nov 11, 2020
1 parent 49e5a22 commit 4acb2c3
Show file tree
Hide file tree
Showing 18 changed files with 6,455 additions and 2,233 deletions.
688 changes: 543 additions & 145 deletions translib/common_app.go

Large diffs are not rendered by default.

15 changes: 11 additions & 4 deletions translib/tlerr/tlerr.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,19 @@ func (e TranslibSyntaxValidationError) Error() string {
}

type TranslibUnsupportedClientVersion struct {
ClientVersion string
ServerVersion string
ServerBaseVersion string
ClientVersion string
ServerVersion string
ServerBaseVersion string
}

func (e TranslibUnsupportedClientVersion) Error() string {
return p.Sprintf("Unsupported client version %s", e.ClientVersion)
return p.Sprintf("Unsupported client version %s", e.ClientVersion)
}

type TranslibXfmrRetError struct {
XlateFailDelReq bool
}

func (e TranslibXfmrRetError) Error() string {
return p.Sprintf("Translib transformer return %s", e.XlateFailDelReq)
}
63 changes: 45 additions & 18 deletions translib/transformer/transformer.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ package transformer
import (
"fmt"
"github.com/openconfig/goyang/pkg/yang"
"github.com/openconfig/ygot/ygot"
"os"
"strings"
"bufio"
Expand All @@ -30,15 +29,8 @@ import (
)

var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models path

var entries = map[string]*yang.Entry{}

//Interface for xfmr methods
type xfmrInterface interface {
tableXfmr(s *ygot.GoStruct, t *interface{}) (string, error)
keyXfmr(s *ygot.GoStruct, t *interface{}) (string, error)
fieldXfmr(s *ygot.GoStruct, t *interface{}) (string, error)
}
var ModelsListFile = "models_list"
var TblInfoJsonFile = "sonic_table_info.json"

func reportIfError(errs []error) {
if len(errs) > 0 {
Expand All @@ -50,15 +42,15 @@ func reportIfError(errs []error) {

func getOcModelsList () ([]string) {
var fileList []string
file, err := os.Open(YangPath + "models_list")
file, err := os.Open(YangPath + ModelsListFile)
if err != nil {
return fileList
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fileEntry := scanner.Text()
if strings.HasPrefix(fileEntry, "#") != true {
if !strings.HasPrefix(fileEntry, "#") {
_, err := os.Stat(YangPath + fileEntry)
if err != nil {
continue
Expand Down Expand Up @@ -86,9 +78,9 @@ func getDefaultModelsList () ([]string) {

func init() {
initYangModelsPath()
yangFiles := []string{}
initRegex()
ocList := getOcModelsList()
yangFiles = getDefaultModelsList()
yangFiles := getDefaultModelsList()
yangFiles = append(yangFiles, ocList...)
fmt.Println("Yang model List:", yangFiles)
err := loadYangModules(yangFiles...)
Expand All @@ -105,7 +97,7 @@ func initYangModelsPath() {
YangPath = path
}

fmt.Println("Yang models path:", YangPath)
fmt.Println("Yang modles path:", YangPath)
}

func loadYangModules(files ...string) error {
Expand Down Expand Up @@ -147,15 +139,19 @@ func loadYangModules(files ...string) error {
}
}

sonic_entries := make([]*yang.Entry, len(names))
sonic_entries := make([]*yang.Entry, 0)
oc_entries := make(map[string]*yang.Entry)
oc_annot_entries := make([]*yang.Entry, len(names))
sonic_annot_entries := make([]*yang.Entry, len(names))
oc_annot_entries := make([]*yang.Entry, 0)
sonic_annot_entries := make([]*yang.Entry, 0)

for _, n := range names {
if strings.Contains(n, "annot") && strings.Contains(n, "sonic") {
sonic_annot_entries = append(sonic_annot_entries, yang.ToEntry(mods[n]))
} else if strings.Contains(n, "annot") {
yangMdlNmDt := strings.Split(n, "-annot")
if len(yangMdlNmDt) > 0 {
addMdlCpbltEntry(yangMdlNmDt[0])
}
oc_annot_entries = append(oc_annot_entries, yang.ToEntry(mods[n]))
} else if strings.Contains(n, "sonic") {
sonic_entries = append(sonic_entries, yang.ToEntry(mods[n]))
Expand All @@ -164,6 +160,37 @@ func loadYangModules(files ...string) error {
}
}

// populate model capabilities data
for yngMdlNm := range(xMdlCpbltMap) {
org := ""
ver := ""
ocVerSet := false
yngEntry := oc_entries[yngMdlNm]
if (yngEntry != nil) {
// OC yang has version in standard extension oc-ext:openconfig-version
if strings.HasPrefix(yngMdlNm, "openconfig-") {
for _, ext := range yngEntry.Exts {
dataTagArr := strings.Split(ext.Keyword, ":")
tagType := dataTagArr[len(dataTagArr)-1]
if tagType == "openconfig-version" {
ver = ext.NName()
fmt.Printf("Found version %v for yang module %v", ver, yngMdlNm)
if len(strings.TrimSpace(ver)) > 0 {
ocVerSet = true
}
break
}

}
}
}
if ((strings.HasPrefix(yngMdlNm, "ietf-")) || (!ocVerSet)) {
// as per RFC7895 revision date to be used as version
ver = mods[yngMdlNm].Current() //gives the most recent revision date for yang module
}
org = mods[yngMdlNm].Organization.Name
addMdlCpbltData(yngMdlNm, ver, org)
}
dbMapBuild(sonic_entries)
annotDbSpecMap(sonic_annot_entries)
annotToDbMapBuild(oc_annot_entries)
Expand Down
39 changes: 39 additions & 0 deletions translib/transformer/xconst.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ const (
YANG_CONTAINER = "container"
YANG_LEAF = "leaf"
YANG_LEAF_LIST = "leaf-list"
YANG_CHOICE = "choice"
YANG_CASE = "case"

YANG_ANNOT_DB_NAME = "db-name"
YANG_ANNOT_TABLE_NAME = "table-name"
Expand All @@ -46,6 +48,43 @@ const (

XPATH_SEP_FWD_SLASH = "/"
XFMR_EMPTY_STRING = ""
XFMR_NONE_STRING = "NONE"
SONIC_TABLE_INDEX = 2
SONIC_LIST_INDEX = 3
SONIC_FIELD_INDEX = 4
SONIC_MDL_PFX = "sonic"
OC_MDL_PFX = "openconfig-"
IETF_MDL_PFX = "ietf-"
IANA_MDL_PFX = "iana-"
YTDB_KEY_XFMR_RET_ARGS = 2
YTDB_KEY_XFMR_RET_VAL_INDX = 0
YTDB_KEY_XFMR_RET_ERR_INDX = 1
YTDB_SBT_XFMR_RET_ARGS = 2
YTDB_SBT_XFMR_RET_VAL_INDX = 0
YTDB_SBT_XFMR_RET_ERR_INDX = 1
YTDB_FLD_XFMR_RET_ARGS = 2
YTDB_FLD_XFMR_RET_VAL_INDX = 0
YTDB_FLD_XFMR_RET_ERR_INDX = 1
DBTY_KEY_XFMR_RET_ARGS = 2
DBTY_KEY_XFMR_RET_VAL_INDX = 0
DBTY_KEY_XFMR_RET_ERR_INDX = 1
DBTY_FLD_XFMR_RET_ARGS = 2
DBTY_FLD_XFMR_RET_VAL_INDX = 0
DBTY_FLD_XFMR_RET_ERR_INDX = 1
SUBSC_SBT_XFMR_RET_ARGS = 2
SUBSC_SBT_XFMR_RET_VAL_INDX = 0
SUBSC_SBT_XFMR_RET_ERR_INDX = 1
DBTY_SBT_XFMR_RET_ERR_INDX = 0
TBL_XFMR_RET_ARGS = 2
TBL_XFMR_RET_VAL_INDX = 0
TBL_XFMR_RET_ERR_INDX = 1
POST_XFMR_RET_ARGS = 2
POST_XFMR_RET_VAL_INDX = 0
POST_XFMR_RET_ERR_INDX = 1
PRE_XFMR_RET_ARGS = 1
PRE_XFMR_RET_ERR_INDX = 0

XFMR_INVALID = -1
XFMR_DISABLE = 0
XFMR_ENABLE = 1
)
Loading

0 comments on commit 4acb2c3

Please sign in to comment.