Skip to content

Commit

Permalink
Query Parameter support in transformer infra for REST GET operation (#…
Browse files Browse the repository at this point in the history 
…102)

* Transformer infra enhacements to support singleton container
	-Changes in DbSpec creation, inherit db-name from sonic table, use cvl db-name annotation for sonic yang, utility function changes
	-Support exclusion of sonic yangs xfmr support from models list

* Sonic yang singleton container support in GET and SET/CRU code flow

* Added a container in openconfig-test-xfmr.yang that maps to a sonic yang table containing a singleton container and UT cases exercising this mapping using table-name and key-name annotations

* Add sonic singleton container in sonic yang and UT cases for CRUD and GET

* Update unit test README file with the build tag information

* Initial Query Params infra support in translib and common app

* Support for depth query parameter in transformer infra

* Content query parameter support in xfmr infra

* Add fields query parameters support

* Query parameter pruning API integration and error handling

* 1)Handle query-parameter content mismatch check for list node since translib fills ygot in request iteslf and when there is onctent mismatch one cannot return that ygot and has to return empty.2)Handle singleton container case for fields query-parameter processing.

* added query-parameter support in transformer test infra

* fixed build error due to rebase/merge-conflict resolution

* Add UT cases for depth and content query parameters support in infra

* Add UT cases for depth and content query parameters support in infra for sonic yang

* Add OC yang fields query parameters unit tests

* Added sonic yang fields query-parameter UT cases

* Infra enhancement for handling OC Yang list/config/key-leaf & list/state/key-leaf allowing application annotation to be skipped and also avoiding translating into db key-attribute in value hash

* Add composite key handling changes for 1:1 OC to sonic key mapping

* Run format check on test yang UT file, Move composite Key handling inside keyCreate function

* when filling the list-keys in GET flow, update the list-keys map used as an optimized way to fill list-key leaves inside the config and state containers instead of extracting from the URI string

---------

Co-authored-by: ranjinidn <ranjini_nagaraj@dell.com>
Co-authored-by: ranjinidn <51423501+ranjinidn@users.noreply.github.com>
  • Loading branch information
@amrutasali @ranjinidn
3 people authored Sep 27, 2023
1 parent 42ca0a6 commit cdd83d7
Show file tree
Hide file tree
Showing 21 changed files with 2,068 additions and 293 deletions.
2 changes: 2 additions & 0 deletions translib/api_tests_app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ func (app *apiTests) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) (
resp["message"] = app.echoMsg
resp["path"] = app.path
resp["depth"] = app.depth
resp["content"] = app.content
resp["fields"] = app.fields

gr.Payload, err = json.Marshal(&resp)
return gr, err
Expand Down
18 changes: 14 additions & 4 deletions translib/app_interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,20 @@ type appData struct {
// These include RESTCONF query parameters like - depth, fields etc.
type appOptions struct {

// depth limits subtree levels in the response data.
// 0 indicates unlimited depth.
// Valid for GET API only.
depth uint
// depth limits subtree levels in the response data.
// 0 indicates unlimited depth.
// Valid for GET API only.
depth uint

// content query parameter value receved from the URI
// possible value is one of 'config', 'nonconfig','all','state' or 'operational'
// Valid for GET API only.
content string

//fields query parameters
// paths of the fields that needs to be filtered in GET payload response
// Valid for GET API only.
fields []string

// deleteEmptyEntry indicates if the db entry should be deleted upon
// deletion of last field. This is a non standard option.
Expand Down
40 changes: 33 additions & 7 deletions translib/common_app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,24 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType)
origXfmrYgotRoot, _ := ygot.DeepCopy((*app.ygotRoot).(ygot.GoStruct))
isEmptyPayload := false
appYgotStruct := (*app.ygotRoot).(ygot.GoStruct)
payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache)
var qParams transformer.QueryParams
qParams, err = transformer.NewQueryParams(app.depth, app.content, app.fields)
if err != nil {
log.Warning("transformer.NewQueryParams() returned : ", err)
resPayload = []byte("{}")
break
}
payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache, qParams)
if err != nil {
// target URI for list GET request with QP content!=all and node's content-type mismatches the requested content-type, return empty payload
if isEmptyPayload && qParams.IsContentEnabled() && transformer.IsListNode(app.pathInfo.Path) {
if err.Error() == transformer.QUERY_CONTENT_MISMATCH_ERR {
err = nil
}
}
if err != nil {
log.Warning("transformer.GetAndXlateFromDB() returned : ", err)
}
resPayload = payload
break
}
Expand All @@ -429,6 +445,12 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType)
resPayload = payload
break
}
if isEmptyPayload && (app.depth == 1) && !transformer.IsLeafNode(app.pathInfo.Path) && !transformer.IsLeafListNode(app.pathInfo.Path) {
// target URI for Container or list GET request with depth = 1, returns empty payload
resPayload = payload
break
}

targetObj, tgtObjCastOk := (*app.ygotTarget).(ygot.GoStruct)
if !tgtObjCastOk {
/*For ygotTarget populated by tranlib, for query on leaf level and list(without instance) level,
Expand Down Expand Up @@ -476,13 +498,17 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType)
err = tlerr.NotFound("Resource not found")
break
}
resPayload = payload
log.Info("No data available")
//TODO: Return not found error
//err = tlerr.NotFound("Resource not found")
break
if !qParams.IsEnabled() {
resPayload = payload
log.Info("No data available")
//TODO: Return not found error
//err = tlerr.NotFound("Resource not found")
break
}
}
if !qParams.IsEnabled() {
resYgot = appYgotStruct
}
resYgot = appYgotStruct
}
}
if resYgot != nil {
Expand Down
6 changes: 3 additions & 3 deletions translib/transformer/sflow_openconfig_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func Test_node_on_openconfig_sflow(t *testing.T) {
loadDB(db.ConfigDB, pre_req_map)
expected_get_json := "{\"openconfig-sampling-sflow:state\":{\"agent\":\"Ethernet8\",\"enabled\":true,\"polling-interval\":300}}"
url = "/openconfig-sampling-sflow:sampling/sflow/state"
t.Run("Test get on sflow node", processGetRequest(url, expected_get_json, false))
t.Run("Test get on sflow node", processGetRequest(url, nil, expected_get_json, false))
time.Sleep(1 * time.Second)
unloadDB(db.ConfigDB, cleanuptbl)
t.Log("\n\n+++++++++++++ Done Performing Get on Sflow node ++++++++++++")
Expand Down Expand Up @@ -154,7 +154,7 @@ func Test_node_openconfig_sflow_collector(t *testing.T) {
loadDB(db.ConfigDB, pre_req_map)
expected_get_json := "{ \"openconfig-sampling-sflow:collectors\":{\"collector\":[{\"address\":\"3.3.3.3\",\"config\":{\"address\":\"3.3.3.3\",\"network-instance\":\"default\",\"port\":6666},\"network-instance\":\"default\",\"port\":6666,\"state\":{\"address\":\"3.3.3.3\",\"network-instance\":\"default\",\"port\":6666}}]}}"
url = "/openconfig-sampling-sflow:sampling/sflow/collectors"
t.Run("Test get on collector node for sflow", processGetRequest(url, expected_get_json, false))
t.Run("Test get on collector node for sflow", processGetRequest(url, nil, expected_get_json, false))
time.Sleep(1 * time.Second)
cleanuptbl = map[string]interface{}{"SFLOW_COLLECTOR": map[string]interface{}{"3.3.3.3_6666_default": ""}}
unloadDB(db.ConfigDB, cleanuptbl)
Expand Down Expand Up @@ -215,7 +215,7 @@ func Test_node_openconfig_sflow_interface(t *testing.T) {
loadDB(db.ApplDB, non_pre_req_map)
expected_get_json := "{\"openconfig-sampling-sflow:state\":{\"enabled\":true,\"name\":\"Ethernet8\",\"sampling-rate\":30000}}"
url = "/openconfig-sampling-sflow:sampling/sflow/interfaces/interface[name=Ethernet8]/state"
t.Run("Test get on sflow interface", processGetRequest(url, expected_get_json, false))
t.Run("Test get on sflow interface", processGetRequest(url, nil, expected_get_json, false))
time.Sleep(2 * time.Second)
cleanuptbl = map[string]interface{}{"SFLOW_SESSION": map[string]interface{}{"Ethernet8": ""}}
unloadDB(db.ConfigDB, cleanuptbl)
Expand Down
12 changes: 6 additions & 6 deletions translib/transformer/sflow_sonic_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func Test_node_sonic_sflow(t *testing.T) {
// Verify global configurations
url = "/sonic-sflow:sonic-sflow/SFLOW/global"
url_body_json = "{\"sonic-sflow:global\":{\"admin_state\":\"up\",\"agent_id\":\"Ethernet4\",\"polling_interval\":100}}"
t.Run("Verify sFlow global configurations", processGetRequest(url, url_body_json, false))
t.Run("Verify sFlow global configurations", processGetRequest(url, nil, url_body_json, false))

//Delete sflow global configurations
url = "/sonic-sflow:sonic-sflow/SFLOW"
Expand All @@ -68,7 +68,7 @@ func Test_node_sonic_sflow(t *testing.T) {
//Verify deleted sflow global configuration
url = "/sonic-sflow:sonic-sflow/SFLOW"
url_body_json = "{}"
t.Run("Verify delete on sflow node", processGetRequest(url, url_body_json, false))
t.Run("Verify delete on sflow node", processGetRequest(url, nil, url_body_json, false))
}

func Test_node_sonic_sflow_collector(t *testing.T) {
Expand All @@ -82,7 +82,7 @@ func Test_node_sonic_sflow_collector(t *testing.T) {

// Verify sFlow collector configurations
url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST[name=1.1.1.1_6343_default]"
t.Run("Verify sFlow collector", processGetRequest(url, url_body_json, false))
t.Run("Verify sFlow collector", processGetRequest(url, nil, url_body_json, false))

// Set collector ip
url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST[name=1.1.1.1_6343_default]/collector_ip"
Expand All @@ -104,7 +104,7 @@ func Test_node_sonic_sflow_collector(t *testing.T) {
//Verify collector port
url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST"
url_body_json = "{}"
t.Run("Verify delete on sFlow collector", processGetRequest(url, url_body_json, false))
t.Run("Verify delete on sFlow collector", processGetRequest(url, nil, url_body_json, false))
}

func Test_node_sonic_sflow_interface(t *testing.T) {
Expand Down Expand Up @@ -137,7 +137,7 @@ func Test_node_sonic_sflow_interface(t *testing.T) {
url_body_json = "{}"
err_str := "Resource not found"
expected_err := tlerr.NotFoundError{Format: err_str}
t.Run("Verify delete on sFlow Interface", processGetRequest(url, url_body_json, true, expected_err))
t.Run("Verify delete on sFlow Interface", processGetRequest(url, nil, url_body_json, true, expected_err))

//Delete sflow global configurations
url = "/sonic-sflow:sonic-sflow/SFLOW"
Expand All @@ -147,5 +147,5 @@ func Test_node_sonic_sflow_interface(t *testing.T) {
//Verify deleted sflow global configuration
url = "/sonic-sflow:sonic-sflow/SFLOW"
url_body_json = "{}"
t.Run("Verify delete on sFlow collector", processGetRequest(url, url_body_json, false))
t.Run("Verify delete on sFlow collector", processGetRequest(url, nil, url_body_json, false))
}
18 changes: 6 additions & 12 deletions translib/transformer/test/openconfig-test-xfmr-annot.yang
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,12 @@ module openconfig-test-xfmr-annot {
}
}

deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:config/oc-test-xfmr:id {
deviate add {
sonic-ext:field-transformer "test_sensor_group_id_field_xfmr";
}
}

deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:config/oc-test-xfmr:group-colors {
deviate add {
sonic-ext:field-name "colors";
}
}

deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:state/oc-test-xfmr:id {
deviate add {
sonic-ext:field-transformer "test_sensor_group_id_field_xfmr";
}
}

deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:state/oc-test-xfmr:group-colors {
deviate add {
sonic-ext:field-name "colors";
Expand Down Expand Up @@ -90,6 +78,12 @@ module openconfig-test-xfmr-annot {
}
}

deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:test-sensor-components/oc-test-xfmr:test-sensor-component {
deviate add {
sonic-ext:table-name "TEST_SENSOR_COMPONENT_TABLE";
}
}

deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sets/oc-test-xfmr:test-set {
deviate add {
sonic-ext:table-name "TEST_SET_TABLE";
Expand Down
125 changes: 123 additions & 2 deletions translib/transformer/test/openconfig-test-xfmr.yang
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ module openconfig-test-xfmr {
}
}
}
uses test-sensor-components-top;
}
}
}
Expand Down Expand Up @@ -274,7 +275,9 @@ module openconfig-test-xfmr {
}
}
}

////////////////////////////

grouping test-sensor-group-config {
description
"Config parameters related to the test sensor groups";
Expand Down Expand Up @@ -370,6 +373,126 @@ module openconfig-test-xfmr {

}

///////////////////
grouping test-sensor-component-config {
description
"Configuration data for sensor-components";

leaf name {
type string;
description
"Device name for the sensor component. ";
}
leaf type {
type enumeration {
enum TYPE1 {
description
"Component Type 1.";
}
enum TYPE2 {
description
"Component Type 2.";
}
enum TYPE3 {
description
"Component Type 3.";
}
}
}
leaf version {
type string;
description
"Version of the Component. ";
}
leaf description {
type string;
description
"Description, or comment, for the test sensor component";
}
}

grouping test-sensor-component-state {
description
"Operational State data for sensor components";
leaf mfg-name {
type string;
description
"System-supplied identifier for the manufacturer of the component.";
}

leaf mfg-date {
type string;
description
"System-supplied representation of the component's
manufacturing date.";
}

leaf hardware-version {
type string;
description
"For hardware components, this is the hardware revision of
the component.";
}

leaf firmware-version {
type string;
description
"For hardware components, this is the version of associated
firmware that is running on the component, if applicable.";
}
}

grouping test-sensor-components-top {
container test-sensor-components {
description
"Enclosing container for test sensor component references";

list test-sensor-component {
key "name type version";
description
"List of sensor component references";

leaf name {
type leafref {
path "../config/name";
}
description
"Reference to the name of component";
}

leaf type {
type leafref {
path "../config/type";
}
description
"Reference to the type of component";
}

leaf version {
type leafref {
path "../config/version";
}
description
"Reference to the version of component";
}

container config {
description
"Configuration parameters to configure a sensor component";
uses test-sensor-component-config;
}

container state {
config false;
description
"Operational state parameters of a sensor component";
uses test-sensor-component-config;
uses test-sensor-component-state;
}
}
}
}

grouping interfaces-config {
description
"Configuration data for interface references";
Expand Down Expand Up @@ -424,10 +547,8 @@ module openconfig-test-xfmr {
}
}
}

///////////////////


// data definition statements

container test-xfmr {
Expand Down
23 changes: 23 additions & 0 deletions translib/transformer/test/sonic-test-xfmr.yang
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,29 @@ module sonic-test-xfmr {
}
}

container TEST_SENSOR_COMPONENT_TABLE {

list TEST_SENSOR_COMPONENT_TABLE_LIST {
key "name type version";

leaf name {
type string;
}

leaf type {
type string;
}

leaf version {
type string;
}

leaf description {
type string;
}
}
}

container TEST_SET_TABLE {

list TEST_SET_TABLE_LIST {
Expand Down
Loading

0 comments on commit cdd83d7

Please sign in to comment.