From c9206b8685e321fd9a78f5ad1e7046b246885929 Mon Sep 17 00:00:00 2001 From: Paul Sebastian Date: Tue, 23 Jul 2024 06:53:52 -0700 Subject: [PATCH] add tests for fields and keywords Signed-off-by: Paul Sebastian --- .../public/antlr/dql/.generated/DQLParser.ts | 22 +- .../public/antlr/dql/code_completion.test.ts | 281 ++++++++++++++++++ .../data/public/antlr/dql/code_completion.ts | 17 +- .../antlr/dql/grammar/.antlr/DQLParser.java | 12 +- .../public/antlr/dql/grammar/DQLParser.g4 | 2 +- 5 files changed, 304 insertions(+), 30 deletions(-) create mode 100644 src/plugins/data/public/antlr/dql/code_completion.test.ts diff --git a/src/plugins/data/public/antlr/dql/.generated/DQLParser.ts b/src/plugins/data/public/antlr/dql/.generated/DQLParser.ts index 1cb741fc11ae..d666dc2089cb 100644 --- a/src/plugins/data/public/antlr/dql/.generated/DQLParser.ts +++ b/src/plugins/data/public/antlr/dql/.generated/DQLParser.ts @@ -368,7 +368,6 @@ export class DQLParser extends antlr.Parser { { this.state = 70; this.match(DQLParser.LPAREN); - { this.state = 72; this.errorHandler.sync(this); _la = this.tokenStream.LA(1); @@ -379,7 +378,6 @@ export class DQLParser extends antlr.Parser { } } - } this.state = 74; this.groupContent(); this.state = 82; @@ -397,7 +395,6 @@ export class DQLParser extends antlr.Parser { this.errorHandler.reportMatch(this); this.consume(); } - { this.state = 77; this.errorHandler.sync(this); _la = this.tokenStream.LA(1); @@ -408,7 +405,6 @@ export class DQLParser extends antlr.Parser { } } - } this.state = 79; this.groupContent(); } @@ -855,6 +851,15 @@ export class GroupExpressionContext extends antlr.ParserRuleContext { public RPAREN(): antlr.TerminalNode { return this.getToken(DQLParser.RPAREN, 0)!; } + public NOT(): antlr.TerminalNode[]; + public NOT(i: number): antlr.TerminalNode | null; + public NOT(i?: number): antlr.TerminalNode | null | antlr.TerminalNode[] { + if (i === undefined) { + return this.getTokens(DQLParser.NOT); + } else { + return this.getToken(DQLParser.NOT, i); + } + } public OR(): antlr.TerminalNode[]; public OR(i: number): antlr.TerminalNode | null; public OR(i?: number): antlr.TerminalNode | null | antlr.TerminalNode[] { @@ -873,15 +878,6 @@ export class GroupExpressionContext extends antlr.ParserRuleContext { return this.getToken(DQLParser.AND, i); } } - public NOT(): antlr.TerminalNode[]; - public NOT(i: number): antlr.TerminalNode | null; - public NOT(i?: number): antlr.TerminalNode | null | antlr.TerminalNode[] { - if (i === undefined) { - return this.getTokens(DQLParser.NOT); - } else { - return this.getToken(DQLParser.NOT, i); - } - } public override get ruleIndex(): number { return DQLParser.RULE_groupExpression; } diff --git a/src/plugins/data/public/antlr/dql/code_completion.test.ts b/src/plugins/data/public/antlr/dql/code_completion.test.ts new file mode 100644 index 000000000000..ff566dfd3347 --- /dev/null +++ b/src/plugins/data/public/antlr/dql/code_completion.test.ts @@ -0,0 +1,281 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { monaco } from '@osd/monaco'; +import { getSuggestions } from './code_completion'; +import { IIndexPattern } from '../..'; + +const testingIndex = ({ + title: 'opensearch_dashboards_sample_data_flights', + fields: [ + { + count: 0, + name: 'Carrier', + displayName: 'Carrier', + type: 'string', + esTypes: ['keyword'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 2, + name: 'DestCityName', + displayName: 'DestCityName', + type: 'string', + esTypes: ['keyword'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 0, + name: 'DestCountry', + displayName: 'DestCountry', + type: 'string', + esTypes: ['keyword'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 0, + name: 'DestWeather', + displayName: 'DestWeather', + type: 'string', + esTypes: ['keyword'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 0, + name: 'DistanceMiles', + displayName: 'DistanceMiles', + type: 'number', + esTypes: ['float'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 0, + name: 'FlightDelay', + displayName: 'FlightDelay', + type: 'boolean', + esTypes: ['boolean'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 0, + name: 'FlightNum', + displayName: 'FlightNum', + type: 'string', + esTypes: ['keyword'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 0, + name: 'OriginWeather', + displayName: 'OriginWeather', + type: 'string', + esTypes: ['keyword'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + subType: undefined, + }, + { + count: 0, + name: '_id', + displayName: '_id', + type: 'string', + esTypes: ['_id'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: false, + subType: undefined, + }, + { + count: 0, + name: '_index', + displayName: '_index', + type: 'string', + esTypes: ['_index'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: false, + subType: undefined, + }, + { + count: 0, + name: '_score', + displayName: '_score', + type: 'number', + scripted: false, + searchable: false, + aggregatable: false, + readFromDocValues: false, + subType: undefined, + }, + { + count: 0, + name: '_source', + displayName: '_source', + type: '_source', + esTypes: ['_source'], + scripted: false, + searchable: false, + aggregatable: false, + readFromDocValues: false, + subType: undefined, + }, + { + count: 0, + name: '_type', + displayName: '_type', + type: 'string', + esTypes: ['_type'], + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: false, + subType: undefined, + }, + ], +} as unknown) as IIndexPattern; + +const booleanOperatorSuggestions = [ + { text: 'or', type: 'keyword' }, + { text: 'and', type: 'keyword' }, +]; + +const notOperatorSuggestion = { text: 'not', type: 'keyword' }; + +const fieldNameSuggestions = [ + { text: 'Carrier', type: 'field' }, + { text: 'DestCityName', type: 'field' }, + { text: 'DestCountry', type: 'field' }, + { text: 'DestWeather', type: 'field' }, + { text: 'DistanceMiles', type: 'field' }, + { text: 'FlightDelay', type: 'field' }, + { text: 'FlightNum', type: 'field' }, + { text: 'OriginWeather', type: 'field' }, + { text: '_id', type: 'field' }, + { text: '_index', type: 'field' }, + { text: '_score', type: 'field' }, + { text: '_source', type: 'field' }, + { text: '_type', type: 'field' }, +]; + +const fieldNameWithNotSuggestions = fieldNameSuggestions.concat(notOperatorSuggestion); + +const getSuggestionsAtPos = async (query: string, endPos: number) => { + return await getSuggestions({ + query, + indexPatterns: [testingIndex], + position: new monaco.Position(1, endPos), + language: '', // not relevant + selectionEnd: 0, // not relevant + selectionStart: 0, // not relevant + }); +}; + +const getSuggestionAtEnd = async (query: string) => { + return await getSuggestionsAtPos(query, query.length + 1); +}; + +describe('Test Boolean Operators', () => { + it('should suggest AND and OR after expression', async () => { + expect(await getSuggestionAtEnd('field: value ')).toStrictEqual(booleanOperatorSuggestions); + }); + + it('should suggest NOT initially', async () => { + expect(await getSuggestionAtEnd('')).toContainEqual(notOperatorSuggestion); + }); + + it('should suggest NOT after expression', async () => { + expect(await getSuggestionAtEnd('field: value and ')).toContainEqual(notOperatorSuggestion); + }); + + it('should not suggest NOT twice', async () => { + expect(await getSuggestionAtEnd('not ')).not.toContainEqual(notOperatorSuggestion); + }); + + it('should suggest after multiple token search', async () => { + expect(await getSuggestionAtEnd('field: one two three ')).toStrictEqual( + booleanOperatorSuggestions + ); + }); + + it('should suggest after phrase value', async () => { + expect(await getSuggestionAtEnd('field: "value" ')).toStrictEqual(booleanOperatorSuggestions); + }); + + it('should suggest after number', async () => { + expect(await getSuggestionAtEnd('field: 123 ')).toStrictEqual(booleanOperatorSuggestions); + }); + + it('should not suggest after incomplete quote', async () => { + expect(await getSuggestionAtEnd('field: "value ')).not.toStrictEqual( + booleanOperatorSuggestions + ); + }); +}); + +describe('Test Boolean Operators within groups', () => { + it('should suggest AND and OR', async () => { + expect(await getSuggestionAtEnd('field: (value ')).toStrictEqual(booleanOperatorSuggestions); + }); + + it('should suggest NOT after expression', async () => { + expect(await getSuggestionAtEnd('field: (value and ')).toContainEqual(notOperatorSuggestion); + }); + + it('should suggest operator within nested group', async () => { + expect(await getSuggestionAtEnd('field: ("one" and ("two" ')).toStrictEqual( + booleanOperatorSuggestions + ); + }); +}); + +describe('Test field suggestions', () => { + it('basic field suggestion', async () => { + expect(await getSuggestionAtEnd('')).toStrictEqual(fieldNameWithNotSuggestions); + }); + + it('field suggestion after one term', async () => { + expect(await getSuggestionAtEnd('field: value and ')).toStrictEqual( + fieldNameWithNotSuggestions + ); + }); + + it('field suggestion within group', async () => { + expect(await getSuggestionAtEnd('field: value and (one: "two" or ')).toStrictEqual( + fieldNameWithNotSuggestions + ); + }); +}); diff --git a/src/plugins/data/public/antlr/dql/code_completion.ts b/src/plugins/data/public/antlr/dql/code_completion.ts index f0af21be54e7..5831969a3ce2 100644 --- a/src/plugins/data/public/antlr/dql/code_completion.ts +++ b/src/plugins/data/public/antlr/dql/code_completion.ts @@ -41,7 +41,6 @@ const findCursorIndex = ( const findFieldSuggestions = (indexPattern: IndexPattern) => { const fieldNames: string[] = indexPattern.fields - .getAll() .filter((idxField: IndexPatternField) => !idxField.subType) // filter removed .keyword fields .map((idxField: { displayName: string }) => { return idxField.displayName; @@ -72,11 +71,11 @@ const findValueSuggestions = async (index: IndexPattern, field: string, value: s // check to see if last field is within index and if it can suggest values, first check // if .keyword appended field exists because that has values const matchedField = - index.fields.getAll().find((idxField: IndexPatternField) => { + index.fields.find((idxField: IndexPatternField) => { // check to see if the field matches another field with .keyword appended if (idxField.displayName === `${field}.keyword`) return idxField; }) || - index.fields.getAll().find((idxField: IndexPatternField) => { + index.fields.find((idxField: IndexPatternField) => { // if the display name matches, return if (idxField.displayName === field) return idxField; }); @@ -162,11 +161,13 @@ export const getSuggestions = async ({ const { field: lastField = '', value: lastValue = '' } = visitor.visit(tree) ?? {}; if (!!lastField && candidates.tokens.has(DQLParser.PHRASE)) { const values = await findValueSuggestions(currentIndexPattern, lastField, lastValue ?? ''); - completions.push( - ...values.map((val: any) => { - return { text: val, type: 'value' }; - }) - ); + if (!!values) { + completions.push( + ...values?.map((val: any) => { + return { text: val, type: 'value' }; + }) + ); + } } // suggest other candidates, mainly keywords diff --git a/src/plugins/data/public/antlr/dql/grammar/.antlr/DQLParser.java b/src/plugins/data/public/antlr/dql/grammar/.antlr/DQLParser.java index 39fd0fb4a0f9..c55b54ae7a0f 100644 --- a/src/plugins/data/public/antlr/dql/grammar/.antlr/DQLParser.java +++ b/src/plugins/data/public/antlr/dql/grammar/.antlr/DQLParser.java @@ -503,6 +503,10 @@ public GroupContentContext groupContent(int i) { return getRuleContext(GroupContentContext.class,i); } public TerminalNode RPAREN() { return getToken(DQLParser.RPAREN, 0); } + public List NOT() { return getTokens(DQLParser.NOT); } + public TerminalNode NOT(int i) { + return getToken(DQLParser.NOT, i); + } public List OR() { return getTokens(DQLParser.OR); } public TerminalNode OR(int i) { return getToken(DQLParser.OR, i); @@ -511,10 +515,6 @@ public TerminalNode OR(int i) { public TerminalNode AND(int i) { return getToken(DQLParser.AND, i); } - public List NOT() { return getTokens(DQLParser.NOT); } - public TerminalNode NOT(int i) { - return getToken(DQLParser.NOT, i); - } public GroupExpressionContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @@ -530,7 +530,6 @@ public final GroupExpressionContext groupExpression() throws RecognitionExceptio { setState(70); match(LPAREN); - { setState(72); _errHandler.sync(this); _la = _input.LA(1); @@ -541,7 +540,6 @@ public final GroupExpressionContext groupExpression() throws RecognitionExceptio } } - } setState(74); groupContent(); setState(82); @@ -560,7 +558,6 @@ public final GroupExpressionContext groupExpression() throws RecognitionExceptio _errHandler.reportMatch(this); consume(); } - { setState(77); _errHandler.sync(this); _la = _input.LA(1); @@ -571,7 +568,6 @@ public final GroupExpressionContext groupExpression() throws RecognitionExceptio } } - } setState(79); groupContent(); } diff --git a/src/plugins/data/public/antlr/dql/grammar/DQLParser.g4 b/src/plugins/data/public/antlr/dql/grammar/DQLParser.g4 index 433ef903ae89..d10cee485848 100644 --- a/src/plugins/data/public/antlr/dql/grammar/DQLParser.g4 +++ b/src/plugins/data/public/antlr/dql/grammar/DQLParser.g4 @@ -41,7 +41,7 @@ tokenSearch ; groupExpression - : LPAREN (NOT?) groupContent ((OR | AND) (NOT?) groupContent)* RPAREN + : LPAREN NOT? groupContent ((OR | AND) NOT? groupContent)* RPAREN ; groupContent