From ace6f3960d0597619ff39e3b9f560e4534408071 Mon Sep 17 00:00:00 2001 From: Max Hniebergall Date: Wed, 12 Jul 2023 11:08:02 -0400 Subject: [PATCH 1/2] [NLP] Support the different mask tokens used by NLP models for Fill Mask --- specification/ml/_types/inference.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specification/ml/_types/inference.ts b/specification/ml/_types/inference.ts index 0959b46366..c75a3430de 100644 --- a/specification/ml/_types/inference.ts +++ b/specification/ml/_types/inference.ts @@ -265,6 +265,12 @@ export class NerInferenceOptions { /** Fill mask inference options */ export class FillMaskInferenceOptions { + /** The string/token which will be removed from incoming documents and replaced with the inference prediction(s). + * In a response, this field contains the mask token for the specified model/tokenizer. Each model and tokenizer + * has a predefined mask token which cannot be changed. Thus, it is recommended not to set this value in requests. + * However, if this field is present in a request, its value must match the predefined value for that model/tokenizer, + * otherwise the request will fail. */ + mask_token?: string /** Specifies the number of top class predictions to return. Defaults to 0. */ num_top_classes?: integer /** The tokenization options to update when inferring */ From 5f1b9a951e1b91ecaa0a8e46d3dfc7980a583190 Mon Sep 17 00:00:00 2001 From: Max Hniebergall Date: Wed, 12 Jul 2023 11:40:46 -0400 Subject: [PATCH 2/2] add missing output files --- output/schema/schema.json | 50 +++++++++++++++++++++++--------------- output/typescript/types.ts | 1 + 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/output/schema/schema.json b/output/schema/schema.json index abd919314c..76a6ba7c70 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -135103,6 +135103,18 @@ "namespace": "ml._types" }, "properties": [ + { + "description": "The string/token which will be removed from incoming documents and replaced with the inference prediction(s).\nIn a response, this field contains the mask token for the specified model/tokenizer. Each model and tokenizer\nhas a predefined mask token which cannot be changed. Thus, it is recommended not to set this value in requests.\nHowever, if this field is present in a request, its value must match the predefined value for that model/tokenizer,\notherwise the request will fail.", + "name": "mask_token", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, { "description": "Specifies the number of top class predictions to return. Defaults to 0.", "name": "num_top_classes", @@ -135140,7 +135152,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L266-L274" + "specLocation": "ml/_types/inference.ts#L266-L280" }, { "kind": "interface", @@ -135186,7 +135198,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L405-L412" + "specLocation": "ml/_types/inference.ts#L411-L418" }, { "kind": "interface", @@ -135906,7 +135918,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L290-L312", + "specLocation": "ml/_types/inference.ts#L296-L318", "variants": { "kind": "container" } @@ -136039,7 +136051,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L453-L500" + "specLocation": "ml/_types/inference.ts#L459-L506" }, { "kind": "interface", @@ -137806,7 +137818,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L398-L403" + "specLocation": "ml/_types/inference.ts#L404-L409" }, { "description": "BERT and MPNet tokenization configuration options", @@ -137992,7 +138004,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L350-L355" + "specLocation": "ml/_types/inference.ts#L356-L361" }, { "kind": "interface", @@ -138331,7 +138343,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L379-L384" + "specLocation": "ml/_types/inference.ts#L385-L390" }, { "kind": "interface", @@ -138373,7 +138385,7 @@ "name": "PredictedValue", "namespace": "ml._types" }, - "specLocation": "ml/_types/inference.ts#L451-L451", + "specLocation": "ml/_types/inference.ts#L457-L457", "type": { "items": [ { @@ -138465,7 +138477,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L276-L286" + "specLocation": "ml/_types/inference.ts#L282-L292" }, { "kind": "interface", @@ -138535,7 +138547,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L414-L425" + "specLocation": "ml/_types/inference.ts#L420-L431" }, { "kind": "interface", @@ -138883,7 +138895,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L357-L366" + "specLocation": "ml/_types/inference.ts#L363-L372" }, { "description": "Text embedding inference options", @@ -138963,7 +138975,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L386-L390" + "specLocation": "ml/_types/inference.ts#L392-L396" }, { "description": "Text expansion inference options", @@ -139031,7 +139043,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L392-L396" + "specLocation": "ml/_types/inference.ts#L398-L402" }, { "kind": "interface", @@ -139166,7 +139178,7 @@ "name": "TokenizationTruncate", "namespace": "ml._types" }, - "specLocation": "ml/_types/inference.ts#L344-L348" + "specLocation": "ml/_types/inference.ts#L350-L354" }, { "kind": "interface", @@ -139209,7 +139221,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L434-L438" + "specLocation": "ml/_types/inference.ts#L440-L444" }, { "kind": "interface", @@ -140413,7 +140425,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L427-L433" + "specLocation": "ml/_types/inference.ts#L433-L439" }, { "kind": "interface", @@ -140445,7 +140457,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L440-L443" + "specLocation": "ml/_types/inference.ts#L446-L449" }, { "kind": "interface", @@ -140491,7 +140503,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L445-L449" + "specLocation": "ml/_types/inference.ts#L451-L455" }, { "kind": "interface", @@ -141027,7 +141039,7 @@ } } ], - "specLocation": "ml/_types/inference.ts#L368-L377" + "specLocation": "ml/_types/inference.ts#L374-L383" }, { "attachedBehaviors": [ diff --git a/output/typescript/types.ts b/output/typescript/types.ts index 0d57b1b3f4..4a36e62dd4 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -12554,6 +12554,7 @@ export interface MlDiscoveryNode { export type MlExcludeFrequent = 'all' | 'none' | 'by' | 'over' export interface MlFillMaskInferenceOptions { + mask_token?: string num_top_classes?: integer tokenization?: MlTokenizationConfigContainer results_field?: string