From 61a5478da651b363d18be819538ce6ea53cb8e68 Mon Sep 17 00:00:00 2001 From: dhchandw Date: Tue, 10 Sep 2024 15:21:24 -0400 Subject: [PATCH] in zcl-loader-dotdot.js: * making sure discriminator ref is set correctly for all strings * making sure is_signed is set correctly for all numbers * dealing with subatomics (by processing them last) * picking up atomic size from length property fixing incorrect value in zcl-loader-consecutive.test.js and adding string count test in zcl-loader.test.js --- docs/api.md | 104 ++++++++++++++++++++++++++ src-electron/zcl/zcl-loader-dotdot.js | 84 ++++++++++++++++++--- test/zcl-loader-consecutive.test.js | 64 ++++++++++------ test/zcl-loader.test.js | 23 ++++++ 4 files changed, 245 insertions(+), 30 deletions(-) diff --git a/docs/api.md b/docs/api.md index fa1510398e..03ae696c24 100644 --- a/docs/api.md +++ b/docs/api.md @@ -19624,9 +19624,11 @@ This module provides the APIs for dotdot Loading * [~prepareCommands(commands, side, types)](#module_Loader API_ Loader APIs..prepareCommands) ⇒ * [~prepareCluster(cluster, isExtension, types)](#module_Loader API_ Loader APIs..prepareCluster) ⇒ * [~prepareAtomic(type)](#module_Loader API_ Loader APIs..prepareAtomic) ⇒ + * [~prepareSubAtomic(type, atomics)](#module_Loader API_ Loader APIs..prepareSubAtomic) ⇒ * [~prepareBitmap(type, isContained)](#module_Loader API_ Loader APIs..prepareBitmap) ⇒ * [~prepareEnum(type)](#module_Loader API_ Loader APIs..prepareEnum) ⇒ * [~prepareStruct(type)](#module_Loader API_ Loader APIs..prepareStruct) ⇒ + * [~prepareSubAtomicTypes(types)](#module_Loader API_ Loader APIs..prepareSubAtomicTypes) * [~prepareTypes(zclTypes, types)](#module_Loader API_ Loader APIs..prepareTypes) * [~prepareAttributeType(attribute, types, cluster)](#module_Loader API_ Loader APIs..prepareAttributeType) * [~prepareDeviceType(deviceType)](#module_Loader API_ Loader APIs..prepareDeviceType) ⇒ @@ -19891,6 +19893,19 @@ Parses xml type into the atomic object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the atomic format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomic(type, atomics) ⇒ +Parses xml type into the sub-atomic object for insertion into the DB + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) +**Returns**: object ready for insertion into the DB + +| Param | Type | Description | +| --- | --- | --- | +| type | \* | an xml object which conforms to the sub-atomic format in the dotdot xml | +| atomics | \* | an array of atomic types | + ### Loader API: Loader APIs~prepareBitmap(type, isContained) ⇒ @@ -19928,6 +19943,17 @@ Parses xml type into the struct object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the struct format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomicTypes(types) +Parses array of xml objects that conform to the sub-atomic format in the dotdot xml + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) + +| Param | Type | +| --- | --- | +| types | \* | + ### Loader API: Loader APIs~prepareTypes(zclTypes, types) @@ -21424,9 +21450,11 @@ This module provides the APIs for new data model loading * [~prepareCommands(commands, side, types)](#module_Loader API_ Loader APIs..prepareCommands) ⇒ * [~prepareCluster(cluster, isExtension, types)](#module_Loader API_ Loader APIs..prepareCluster) ⇒ * [~prepareAtomic(type)](#module_Loader API_ Loader APIs..prepareAtomic) ⇒ + * [~prepareSubAtomic(type, atomics)](#module_Loader API_ Loader APIs..prepareSubAtomic) ⇒ * [~prepareBitmap(type, isContained)](#module_Loader API_ Loader APIs..prepareBitmap) ⇒ * [~prepareEnum(type)](#module_Loader API_ Loader APIs..prepareEnum) ⇒ * [~prepareStruct(type)](#module_Loader API_ Loader APIs..prepareStruct) ⇒ + * [~prepareSubAtomicTypes(types)](#module_Loader API_ Loader APIs..prepareSubAtomicTypes) * [~prepareTypes(zclTypes, types)](#module_Loader API_ Loader APIs..prepareTypes) * [~prepareAttributeType(attribute, types, cluster)](#module_Loader API_ Loader APIs..prepareAttributeType) * [~prepareDeviceType(deviceType)](#module_Loader API_ Loader APIs..prepareDeviceType) ⇒ @@ -21691,6 +21719,19 @@ Parses xml type into the atomic object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the atomic format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomic(type, atomics) ⇒ +Parses xml type into the sub-atomic object for insertion into the DB + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) +**Returns**: object ready for insertion into the DB + +| Param | Type | Description | +| --- | --- | --- | +| type | \* | an xml object which conforms to the sub-atomic format in the dotdot xml | +| atomics | \* | an array of atomic types | + ### Loader API: Loader APIs~prepareBitmap(type, isContained) ⇒ @@ -21728,6 +21769,17 @@ Parses xml type into the struct object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the struct format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomicTypes(types) +Parses array of xml objects that conform to the sub-atomic format in the dotdot xml + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) + +| Param | Type | +| --- | --- | +| types | \* | + ### Loader API: Loader APIs~prepareTypes(zclTypes, types) @@ -23224,9 +23276,11 @@ This module provides the APIs for ZCL/Data-Model loading. * [~prepareCommands(commands, side, types)](#module_Loader API_ Loader APIs..prepareCommands) ⇒ * [~prepareCluster(cluster, isExtension, types)](#module_Loader API_ Loader APIs..prepareCluster) ⇒ * [~prepareAtomic(type)](#module_Loader API_ Loader APIs..prepareAtomic) ⇒ + * [~prepareSubAtomic(type, atomics)](#module_Loader API_ Loader APIs..prepareSubAtomic) ⇒ * [~prepareBitmap(type, isContained)](#module_Loader API_ Loader APIs..prepareBitmap) ⇒ * [~prepareEnum(type)](#module_Loader API_ Loader APIs..prepareEnum) ⇒ * [~prepareStruct(type)](#module_Loader API_ Loader APIs..prepareStruct) ⇒ + * [~prepareSubAtomicTypes(types)](#module_Loader API_ Loader APIs..prepareSubAtomicTypes) * [~prepareTypes(zclTypes, types)](#module_Loader API_ Loader APIs..prepareTypes) * [~prepareAttributeType(attribute, types, cluster)](#module_Loader API_ Loader APIs..prepareAttributeType) * [~prepareDeviceType(deviceType)](#module_Loader API_ Loader APIs..prepareDeviceType) ⇒ @@ -23491,6 +23545,19 @@ Parses xml type into the atomic object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the atomic format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomic(type, atomics) ⇒ +Parses xml type into the sub-atomic object for insertion into the DB + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) +**Returns**: object ready for insertion into the DB + +| Param | Type | Description | +| --- | --- | --- | +| type | \* | an xml object which conforms to the sub-atomic format in the dotdot xml | +| atomics | \* | an array of atomic types | + ### Loader API: Loader APIs~prepareBitmap(type, isContained) ⇒ @@ -23528,6 +23595,17 @@ Parses xml type into the struct object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the struct format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomicTypes(types) +Parses array of xml objects that conform to the sub-atomic format in the dotdot xml + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) + +| Param | Type | +| --- | --- | +| types | \* | + ### Loader API: Loader APIs~prepareTypes(zclTypes, types) @@ -25024,9 +25102,11 @@ This module provides the APIs for for common functionality related to loading. * [~prepareCommands(commands, side, types)](#module_Loader API_ Loader APIs..prepareCommands) ⇒ * [~prepareCluster(cluster, isExtension, types)](#module_Loader API_ Loader APIs..prepareCluster) ⇒ * [~prepareAtomic(type)](#module_Loader API_ Loader APIs..prepareAtomic) ⇒ + * [~prepareSubAtomic(type, atomics)](#module_Loader API_ Loader APIs..prepareSubAtomic) ⇒ * [~prepareBitmap(type, isContained)](#module_Loader API_ Loader APIs..prepareBitmap) ⇒ * [~prepareEnum(type)](#module_Loader API_ Loader APIs..prepareEnum) ⇒ * [~prepareStruct(type)](#module_Loader API_ Loader APIs..prepareStruct) ⇒ + * [~prepareSubAtomicTypes(types)](#module_Loader API_ Loader APIs..prepareSubAtomicTypes) * [~prepareTypes(zclTypes, types)](#module_Loader API_ Loader APIs..prepareTypes) * [~prepareAttributeType(attribute, types, cluster)](#module_Loader API_ Loader APIs..prepareAttributeType) * [~prepareDeviceType(deviceType)](#module_Loader API_ Loader APIs..prepareDeviceType) ⇒ @@ -25291,6 +25371,19 @@ Parses xml type into the atomic object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the atomic format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomic(type, atomics) ⇒ +Parses xml type into the sub-atomic object for insertion into the DB + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) +**Returns**: object ready for insertion into the DB + +| Param | Type | Description | +| --- | --- | --- | +| type | \* | an xml object which conforms to the sub-atomic format in the dotdot xml | +| atomics | \* | an array of atomic types | + ### Loader API: Loader APIs~prepareBitmap(type, isContained) ⇒ @@ -25328,6 +25421,17 @@ Parses xml type into the struct object for insertion into the DB | --- | --- | --- | | type | \* | an xml object which conforms to the struct format in the dotdot xml | + + +### Loader API: Loader APIs~prepareSubAtomicTypes(types) +Parses array of xml objects that conform to the sub-atomic format in the dotdot xml + +**Kind**: inner method of [Loader API: Loader APIs](#module_Loader API_ Loader APIs) + +| Param | Type | +| --- | --- | +| types | \* | + ### Loader API: Loader APIs~prepareTypes(zclTypes, types) diff --git a/src-electron/zcl/zcl-loader-dotdot.js b/src-electron/zcl/zcl-loader-dotdot.js index 535da3862e..c2b32ddee2 100644 --- a/src-electron/zcl/zcl-loader-dotdot.js +++ b/src-electron/zcl/zcl-loader-dotdot.js @@ -456,10 +456,11 @@ function prepareCluster(cluster, types, isExtension = false) { */ function prepareAtomic(type) { let desc = type.$.name + let length = type.restriction?.[0]?.['type:length']?.[0]?.$?.value return { name: type.$.short, id: parseInt(normalizeHexValue(type.$.id)), - size: getNumBytesFromShortName(type.$.short), + size: length ?? getNumBytesFromShortName(type.$.short), description: desc, isDiscrete: type.$.discrete == 'true' ? true : false, isSigned: desc.includes('Signed'), @@ -469,6 +470,36 @@ function prepareAtomic(type) { } } +/** + * Parses xml type into the sub-atomic object for insertion into the DB + * + * @param {*} type an xml object which conforms to the sub-atomic format in the dotdot xml + * @param {*} atomics an array of atomic types + * @returns object ready for insertion into the DB + */ +function prepareSubAtomic(subAtomic, atomics) { + let relatedAtomic = atomics.find( + (atomic) => atomic.name === subAtomic.inheritsFrom + ) + if (relatedAtomic) { + return { + name: subAtomic.short, + id: parseInt(normalizeHexValue(subAtomic.id)), + size: relatedAtomic.size, + description: relatedAtomic.description, + isDiscrete: relatedAtomic.isDiscrete, + isSigned: relatedAtomic.isSigned, + isString: relatedAtomic.isString, + isLong: relatedAtomic.isLong, + isChar: relatedAtomic.isChar + } + } else { + env.log( + `Could not find related atomic type for sub-atomic type ${subAtomic.short} - Dropping this type.` + ) + } +} + /** * * Parses xml type into the bitmap object for insertion into the DB @@ -573,6 +604,17 @@ function prepareStruct(type) { return ret } +/** + * Parses array of xml objects that conform to the sub-atomic format in the dotdot xml + * + * @param {*} types + */ +function prepareSubAtomicTypes(types) { + types.subAtomics = types.subAtomics + .map((subAtomic) => prepareSubAtomic(subAtomic, types.atomics)) + .filter((subAtomic) => subAtomic != null || subAtomic != undefined) +} + /** * * Parses xml types into the types object for insertion into the DB @@ -593,14 +635,11 @@ function prepareTypes(zclTypes, types) { } else if (type.$.inheritsFrom === undefined) { types.atomics.push(prepareAtomic(type)) } else { - // TODO: Need to handle sub-atomic types, these are types that impose restrictions - // and inherit from an atomic type but are not a struct, bitmap or enum - droppedTypes.push(type.$.name) + // Sub-atomic types - these are types that impose restrictions and inherit from an atomic type but are not a struct, bitmap or enum + // Will be processed after all other types are processed + types.subAtomics.push(type.$) } }) - if (droppedTypes.length > 0) { - env.logDebug(`Dropped types in DotDot loader: ${droppedTypes}`) - } } /** @@ -721,6 +760,8 @@ function prepareDataType(a, dataType, typeMap) { a.name.toLowerCase().includes(dbEnum.zclType.string) ) { dataTypeRef = typeMap.get(dbEnum.zclType.string) + } else if (!dataType && a.isString !== undefined && a.isString == 1) { + dataTypeRef = typeMap.get(dbEnum.zclType.string) } else if ( !dataType && a.name.toLowerCase().includes(dbEnum.zclType.struct) @@ -818,7 +859,14 @@ async function processDataType(db, filePath, packageId, data, dataType) { function prepareNumber(a, dataType) { return { size: a.size, - is_signed: a.name.endsWith('u') || !a.name.includes('int') ? 0 : 1, + is_signed: + a.isSigned !== undefined + ? a.isSigned + : a.name.startsWith('u') || + a.name.endsWith('u') || + !a.name.includes('int') + ? 0 + : 1, name: a.name, cluster_code: a.cluster ? a.cluster : null, discriminator_ref: dataType @@ -1177,7 +1225,13 @@ async function loadZclData(db, ctx) { env.logDebug( `Starting to load Dotdot ZCL data in to DB for: ${ctx.metadataFile}, clusters length=${ctx.zclClusters.length}` ) - let types = { atomics: [], enums: [], bitmaps: [], structs: [] } + let types = { + atomics: [], + enums: [], + bitmaps: [], + structs: [], + subAtomics: [] + } prepareTypes(ctx.zclTypes, types) prepareTypes(ctx.zclGlobalTypes, types) let preparedClusters = [] @@ -1219,6 +1273,8 @@ async function loadZclData(db, ctx) { await queryLoader.insertAtomics(db, ctx.packageId, types.atomics) ctx.ZCLDataTypes = ['ARRAY', 'BITMAP', 'ENUM', 'NUMBER', 'STRING', 'STRUCT'] await processDataTypeDiscriminator(db, ctx.packageId, ctx.ZCLDataTypes) + // prepare sub-atomics after all other types are processed + prepareSubAtomicTypes(types) await processDataType( db, ctx.metadataFile, @@ -1247,6 +1303,14 @@ async function loadZclData(db, ctx) { types.structs, dbEnum.zclType.struct ) + // sub-atomics are in the same format as atomics, so we can use the same prepare function + await processDataType( + db, + ctx.metadataFile, + ctx.packageId, + types.subAtomics, + dbEnum.zclType.atomic + ) await processEnumsFromAtomics( db, @@ -1262,6 +1326,8 @@ async function loadZclData(db, ctx) { ) await processNumber(db, ctx.metadataFile, ctx.packageId, types.atomics) await processString(db, ctx.metadataFile, ctx.packageId, types.atomics) + await processNumber(db, ctx.metadataFile, ctx.packageId, types.subAtomics) + await processString(db, ctx.metadataFile, ctx.packageId, types.subAtomics) await processEnums(db, ctx.metadataFile, ctx.packageId, types.enums) await processBitmaps(db, ctx.metadataFile, ctx.packageId, types.bitmaps) await processStruct(db, ctx.metadataFile, ctx.packageId, types.structs) diff --git a/test/zcl-loader-consecutive.test.js b/test/zcl-loader-consecutive.test.js index 74ff03802a..52eb6cbf17 100644 --- a/test/zcl-loader-consecutive.test.js +++ b/test/zcl-loader-consecutive.test.js @@ -60,17 +60,17 @@ test( let dotdotPackageId let ctx = await zclLoader.loadZcl(db, env.builtinSilabsZclMetafile()) - let jsonPackageId = ctx.packageId + let zigbeePackageId = ctx.packageId ctx = await zclLoader.loadZcl(db, env.builtinSilabsZclMetafile()) - expect(ctx.packageId).toEqual(jsonPackageId) + expect(ctx.packageId).toEqual(zigbeePackageId) let p = await queryPackage.getPackageByPackageId(db, ctx.packageId) expect(p.version).toEqual(1) expect(p.category).toEqual('zigbee') await zclLoader.loadZcl(db, env.builtinDotdotZclMetafile()) ctx = await zclLoader.loadZcl(db, env.builtinDotdotZclMetafile()) dotdotPackageId = ctx.packageId - expect(dotdotPackageId).not.toEqual(jsonPackageId) + expect(dotdotPackageId).not.toEqual(zigbeePackageId) p = await queryPackage.getPackageByPackageId(db, ctx.packageId) expect(p.version).toEqual(1) @@ -79,9 +79,9 @@ test( dbEnum.packageType.zclProperties ) expect(rows.length).toEqual(2) - let x = await queryZcl.selectAllClusters(db, jsonPackageId) + let x = await queryZcl.selectAllClusters(db, zigbeePackageId) expect(x.length).toEqual(testUtil.totalClusterCount) - x = await queryCommand.selectAllClusterCommands(db, jsonPackageId) + x = await queryCommand.selectAllClusterCommands(db, zigbeePackageId) let unmatchedRequestCount = 0 let responsesCount = 0 @@ -104,47 +104,61 @@ test( let z = await queryCommand.selectCommandById(db, x[0].id) expect(z.label).toBe(x[0].label) - x = await queryCommand.selectAllCommandArguments(db, jsonPackageId) + x = await queryCommand.selectAllCommandArguments(db, zigbeePackageId) expect(x.length).toEqual(testUtil.totalCommandArgsCount) - x = await queryZcl.selectAllDomains(db, jsonPackageId) + x = await queryZcl.selectAllDomains(db, zigbeePackageId) expect(x.length).toEqual(testUtil.totalDomainCount) z = await queryZcl.selectDomainById(db, x[0].id) expect(z.label).toBe(x[0].label) - x = await queryZcl.selectAllEnums(db, jsonPackageId) + x = await queryZcl.selectAllEnums(db, zigbeePackageId) expect(x.length).toEqual(testUtil.totalEnumCount) - x = await queryZcl.selectAllAttributesBySide(db, 'server', jsonPackageId) + x = await queryZcl.selectAllAttributesBySide( + db, + 'server', + zigbeePackageId + ) expect(x.length).toBe(testUtil.totalServerAttributeCount) - x = await queryZcl.selectAllEnumItems(db, jsonPackageId) + x = await queryZcl.selectAllEnumItems(db, zigbeePackageId) expect(x.length).toEqual(testUtil.totalEnumItemCount) - x = await queryZcl.selectAllStructsWithItemCount(db, [jsonPackageId]) + x = await queryZcl.selectAllStructsWithItemCount(db, [zigbeePackageId]) expect(x.length).toEqual(54) - x = await queryZcl.selectAllBitmaps(db, jsonPackageId) + x = await queryZcl.selectAllBitmaps(db, zigbeePackageId) expect(x.length).toEqual(129) - x = await queryZcl.selectAllDataTypes(db, jsonPackageId) + x = await queryZcl.selectAllDataTypes(db, zigbeePackageId) expect(x.length).toEqual(440) - x = await queryZcl.selectAllNumbers(db, jsonPackageId) + x = await queryZcl.selectAllNumbers(db, zigbeePackageId) expect(x.length).toEqual(41) - x = await queryZcl.selectAllStrings(db, jsonPackageId) - expect(x.length).toEqual(5) - - x = await queryStruct.selectAllStructs(db, jsonPackageId) + x = await queryZcl.selectAllStrings(db, zigbeePackageId) + x = x.map((item) => item.name) + let strings = [ + 'octet_string', + 'char_string', + 'long_octet_string', + 'long_char_string' + ] + expect(x.length).toEqual(4) + strings.forEach((s) => { + expect(x).toContain(s) + }) + + x = await queryStruct.selectAllStructs(db, zigbeePackageId) expect(x.length).toEqual(54) - x = await queryDeviceType.selectAllDeviceTypes(db, jsonPackageId) + x = await queryDeviceType.selectAllDeviceTypes(db, zigbeePackageId) expect(x.length).toEqual(175) - x = await queryZcl.selectAllAtomics(db, jsonPackageId) + x = await queryZcl.selectAllAtomics(db, zigbeePackageId) expect(x.length).toEqual(56) x = await queryZcl.selectAllClusters(db, dotdotPackageId) @@ -162,6 +176,14 @@ test( x = await queryZcl.selectAllBitmaps(db, dotdotPackageId) expect(x.length).toEqual(69) + x = await queryZcl.selectAllStrings(db, dotdotPackageId) + x = x.map((item) => item.name) + strings = ['octstr', 'string', 'octstr16', 'string16'] + expect(x.length).toEqual(5) // 5th string is a subatomic type + strings.forEach((s) => { + expect(x).toContain(s) + }) + x = await queryZcl.selectAllEnums(db, dotdotPackageId) expect(x.length).toEqual(testUtil.totalDotDotEnums) @@ -215,7 +237,7 @@ test( rows = await queryPackage.selectAllOptionsValues( db, - jsonPackageId, + zigbeePackageId, dbEnum.sessionOption.defaultResponsePolicy ) expect(rows.length).toBe(3) diff --git a/test/zcl-loader.test.js b/test/zcl-loader.test.js index fda1d9847c..218d3e0d2b 100644 --- a/test/zcl-loader.test.js +++ b/test/zcl-loader.test.js @@ -85,6 +85,20 @@ test( expect(x.length).toEqual(54) x = await queryZcl.selectAllBitmaps(db, packageId) expect(x.length).toEqual(129) + + x = await queryZcl.selectAllStrings(db, packageId) + x = x.map((item) => item.name) + let strings = [ + 'octet_string', + 'char_string', + 'long_octet_string', + 'long_char_string' + ] + expect(x.length).toEqual(4) + strings.forEach((s) => { + expect(x).toContain(s) + }) + x = await queryDeviceType.selectAllDeviceTypes(db, packageId) expect(x.length).toEqual(175) x = await testQuery.selectCountFrom(db, 'COMMAND_ARG') @@ -291,6 +305,15 @@ test( expect(x).toEqual(630) x = await queryZcl.selectAllAtomics(db, packageId) expect(x.length).toEqual(56) + + x = await queryZcl.selectAllStrings(db, packageId) + x = x.map((item) => item.name) + let strings = ['octstr', 'string', 'octstr16', 'string16'] + expect(x.length).toEqual(5) // 5th string is a subatomic type + strings.forEach((s) => { + expect(x).toContain(s) + }) + x = await queryZcl.selectAllBitmaps(db, packageId) expect(x.length).toEqual(69) x = await queryZcl.selectAllEnums(db, packageId)