feat: allow limiting nested repeating fields (#129)

Passes the limit config through to child message decoders to allow limiting the length of nested repeating fields at runtime. E.g. ```protobuf message SubMessage { repeated uint32 repeatedField = 1; } message MyMessage { repeated SubMessage messages = 1; } ``` ```TypeScript const message = MyMessage.decode(buf, { limit: { messages: 5 // limit messages length messages$: { repeatedField: 5 // limit `repeatedField` field of all messages in the messages field } } }) ``` `
ipfs · Feb 2, 2024 · a81f997 · a81f997
1 parent 01e2b69
commit a81f997
Show file tree

Hide file tree

Showing 21 changed files with 751 additions and 67 deletions.
diff --git a/packages/protons-runtime/src/codec.ts b/packages/protons-runtime/src/codec.ts
@@ -19,11 +19,40 @@ export interface EncodeFunction<T> {
   (value: Partial<T>, writer: Writer, opts?: EncodeOptions): void
 }
 
+// protobuf types that contain multiple values
+type CollectionTypes = any[] | Map<any, any>
+
+// protobuf types that are not collections or messages
+type PrimitiveTypes = boolean | number | string | bigint | Uint8Array
+
+// recursive array/map field length limits
+type CollectionLimits <T> = {
+  [K in keyof T]: T[K] extends CollectionTypes ? number :
+    T[K] extends PrimitiveTypes ? never : Limits<T[K]>
+}
+
+// recursive array member array/map field length limits
+type ArrayElementLimits <T> = {
+  [K in keyof T as `${string & K}$`]: T[K] extends Array<infer ElementType> ?
+      (ElementType extends PrimitiveTypes ? never : Limits<ElementType>) :
+      (T[K] extends PrimitiveTypes ? never : Limits<T[K]>)
+}
+
+// recursive map value array/map field length limits
+type MapValueLimits <T> = {
+  [K in keyof T as `${string & K}$value`]: T[K] extends Map<any, infer MapValueType> ?
+      (MapValueType extends PrimitiveTypes ? never : Limits<MapValueType>) :
+      (T[K] extends PrimitiveTypes ? never : Limits<T[K]>)
+}
+
+// union of collection and array elements
+type Limits<T> = Partial<CollectionLimits<T> & ArrayElementLimits<T> & MapValueLimits<T>>
+
 export interface DecodeOptions<T> {
   /**
    * Runtime-specified limits for lengths of repeated/map fields
    */
-  limits?: Partial<Record<keyof T, number>>
+  limits?: Limits<T>
 }
 
 export interface DecodeFunction<T> {

diff --git a/packages/protons-runtime/src/decode.ts b/packages/protons-runtime/src/decode.ts
@@ -2,7 +2,7 @@ import { createReader } from './utils/reader.js'
 import type { Codec, DecodeOptions } from './codec.js'
 import type { Uint8ArrayList } from 'uint8arraylist'
 
-export function decodeMessage <T> (buf: Uint8Array | Uint8ArrayList, codec: Codec<T>, opts?: DecodeOptions<T>): T {
+export function decodeMessage <T> (buf: Uint8Array | Uint8ArrayList, codec: Pick<Codec<T>, 'decode'>, opts?: DecodeOptions<T>): T {
   const reader = createReader(buf)
 
   return codec.decode(reader, undefined, opts)

diff --git a/packages/protons-runtime/src/encode.ts b/packages/protons-runtime/src/encode.ts
@@ -1,7 +1,7 @@
 import { createWriter } from './utils/writer.js'
 import type { Codec } from './codec.js'
 
-export function encodeMessage <T> (message: T, codec: Codec<T>): Uint8Array {
+export function encodeMessage <T> (message: Partial<T>, codec: Pick<Codec<T>, 'encode'>): Uint8Array {
   const w = createWriter()
 
   codec.encode(message, w, {

diff --git a/packages/protons/README.md b/packages/protons/README.md
@@ -94,6 +94,80 @@ const message = MyMessage.decode(buf, {
 })
 ```
 
+#### Limiting repeating fields of nested messages at runtime
+
+Sub messages with repeating elements can be limited in a similar way:
+
+```protobuf
+message SubMessage {
+  repeated uint32 repeatedField = 1;
+}
+
+message MyMessage {
+  SubMessage message = 1;
+}
+```
+
+```TypeScript
+const message = MyMessage.decode(buf, {
+  limits: {
+    messages: {
+      repeatedField: 5 // the SubMessage can not have more than 5 repeatedField entries
+    }
+  }
+})
+```
+
+#### Limiting repeating fields of repeating messages at runtime
+
+Sub messages defined in repeating elements can be limited by appending `$` to the field name in the runtime limit options:
+
+```protobuf
+message SubMessage {
+ repeated uint32 repeatedField = 1;
+}
+
+message MyMessage {
+  repeated SubMessage messages = 1;
+}
+```
+
+```TypeScript
+const message = MyMessage.decode(buf, {
+  limits: {
+    messages: 5 // max 5x SubMessages
+    messages$: {
+      repeatedField: 5 // no SubMessage can have more than 5 repeatedField entries
+    }
+  }
+})
+```
+
+#### Limiting repeating fields of map entries at runtime
+
+Repeating fields in map entries can be limited by appending `$value` to the field name in the runtime limit options:
+
+```protobuf
+message SubMessage {
+ repeated uint32 repeatedField = 1;
+}
+
+message MyMessage {
+  map<string, SubMessage> messages = 1;
+}
+```
+
+```TypeScript
+const message = MyMessage.decode(buf, {
+  limits: {
+    messages: 5 // max 5x SubMessages in the map
+    messages$value: {
+      repeatedField: 5 // no SubMessage in the map can have more than 5 repeatedField entries
+    }
+  }
+})
+```
+
 ### Overriding 64 bit types
 
 By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s.

diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts
@@ -88,6 +88,80 @@
  * })
  * ```
  *
+ * #### Limiting repeating fields of nested messages at runtime
+ *
+ * Sub messages with repeating elements can be limited in a similar way:
+ *
+ * ```protobuf
+ * message SubMessage {
+ *   repeated uint32 repeatedField = 1;
+ * }
+ *
+ * message MyMessage {
+ *   SubMessage message = 1;
+ * }
+ * ```
+ *
+ * ```TypeScript
+ * const message = MyMessage.decode(buf, {
+ *   limits: {
+ *     messages: {
+ *       repeatedField: 5 // the SubMessage can not have more than 5 repeatedField entries
+ *     }
+ *   }
+ * })
+ * ```
+ *
+ * #### Limiting repeating fields of repeating messages at runtime
+ *
+ * Sub messages defined in repeating elements can be limited by appending `$` to the field name in the runtime limit options:
+ *
+ * ```protobuf
+ * message SubMessage {
+ *  repeated uint32 repeatedField = 1;
+ * }
+ *
+ * message MyMessage {
+ *   repeated SubMessage messages = 1;
+ * }
+ * ```
+ *
+ * ```TypeScript
+ * const message = MyMessage.decode(buf, {
+ *   limits: {
+ *     messages: 5 // max 5x SubMessages
+ *     messages$: {
+ *       repeatedField: 5 // no SubMessage can have more than 5 repeatedField entries
+ *     }
+ *   }
+ * })
+ * ```
+ *
+ * #### Limiting repeating fields of map entries at runtime
+ *
+ * Repeating fields in map entries can be limited by appending `$value` to the field name in the runtime limit options:
+ *
+ * ```protobuf
+ * message SubMessage {
+ *  repeated uint32 repeatedField = 1;
+ * }
+ *
+ * message MyMessage {
+ *   map<string, SubMessage> messages = 1;
+ * }
+ * ```
+ *
+ * ```TypeScript
+ * const message = MyMessage.decode(buf, {
+ *   limits: {
+ *     messages: 5 // max 5x SubMessages in the map
+ *     messages$value: {
+ *       repeatedField: 5 // no SubMessage in the map can have more than 5 repeatedField entries
+ *     }
+ *   }
+ * })
+ * ```
+ *
  * ### Overriding 64 bit types
  *
  * By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s.
@@ -806,7 +880,47 @@ export interface ${messageDef.name} {
         // override setting type on js object
         const jsTypeOverride = findJsTypeOverride(fieldDef.type, fieldDef)
 
-        const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type](jsTypeOverride)}`
+        let fieldOpts = ''
+
+        if (fieldDef.message) {
+          let suffix = ''
+
+          if (fieldDef.repeated) {
+            suffix = '$'
+          }
+
+          fieldOpts = `, {
+                limits: opts.limits?.${fieldName}${suffix}
+              }`
+        }
+
+        if (fieldDef.map) {
+          fieldOpts = `, {
+                limits: {
+                  value: opts.limits?.${fieldName}$value
+                }
+              }`
+
+          // do not pass limit opts to map value types that are enums or
+          // primitives - only support messages
+          if (types[fieldDef.valueType] != null) {
+            // primmitive type
+            fieldOpts = ''
+          } else {
+            const valueType = findDef(fieldDef.valueType, messageDef, moduleDef)
+
+            if (isEnumDef(valueType)) {
+              // enum type
+              fieldOpts = ''
+            }
+          }
+        }
+
+        const parseValue = `${decoderGenerators[type] == null
+          ? `${codec}.decode(reader${type === 'message'
+            ? `, reader.uint32()${fieldOpts}`
+            : ''})`
+        : decoderGenerators[type](jsTypeOverride)}`
 
         if (fieldDef.map) {
           moduleDef.addImport('protons-runtime', 'CodeError')

diff --git a/packages/protons/test/fixtures/bitswap.ts b/packages/protons/test/fixtures/bitswap.ts
@@ -183,7 +183,9 @@ export namespace Message {
                   throw new CodeError('decode error - map field "entries" had too many elements', 'ERR_MAX_LENGTH')
                 }
 
-                obj.entries.push(Message.Wantlist.Entry.codec().decode(reader, reader.uint32()))
+                obj.entries.push(Message.Wantlist.Entry.codec().decode(reader, reader.uint32(), {
+                  limits: opts.limits?.entries$
+                }))
                 break
               }
               case 2: {
@@ -429,7 +431,9 @@ export namespace Message {
 
           switch (tag >>> 3) {
             case 1: {
-              obj.wantlist = Message.Wantlist.codec().decode(reader, reader.uint32())
+              obj.wantlist = Message.Wantlist.codec().decode(reader, reader.uint32(), {
+                limits: opts.limits?.wantlist
+              })
               break
             }
             case 2: {
@@ -445,15 +449,19 @@ export namespace Message {
                 throw new CodeError('decode error - map field "payload" had too many elements', 'ERR_MAX_LENGTH')
               }
 
-              obj.payload.push(Message.Block.codec().decode(reader, reader.uint32()))
+              obj.payload.push(Message.Block.codec().decode(reader, reader.uint32(), {
+                limits: opts.limits?.payload$
+              }))
               break
             }
             case 4: {
               if (opts.limits?.blockPresences != null && obj.blockPresences.length === opts.limits.blockPresences) {
                 throw new CodeError('decode error - map field "blockPresences" had too many elements', 'ERR_MAX_LENGTH')
               }
 
-              obj.blockPresences.push(Message.BlockPresence.codec().decode(reader, reader.uint32()))
+              obj.blockPresences.push(Message.BlockPresence.codec().decode(reader, reader.uint32(), {
+                limits: opts.limits?.blockPresences$
+              }))
               break
             }
             case 5: {

diff --git a/packages/protons/test/fixtures/circuit.ts b/packages/protons/test/fixtures/circuit.ts
@@ -203,11 +203,15 @@ export namespace CircuitRelay {
               break
             }
             case 2: {
-              obj.srcPeer = CircuitRelay.Peer.codec().decode(reader, reader.uint32())
+              obj.srcPeer = CircuitRelay.Peer.codec().decode(reader, reader.uint32(), {
+                limits: opts.limits?.srcPeer
+              })
               break
             }
             case 3: {
-              obj.dstPeer = CircuitRelay.Peer.codec().decode(reader, reader.uint32())
+              obj.dstPeer = CircuitRelay.Peer.codec().decode(reader, reader.uint32(), {
+                limits: opts.limits?.dstPeer
+              })
               break
             }
             case 4: {