From 259e0f43d354f0105719b1a4e913b6ddb18cc8a0 Mon Sep 17 00:00:00 2001 From: Matt Bishop Date: Wed, 29 Nov 2023 13:10:41 -0800 Subject: [PATCH] Fix strict mode handling of `[*]` token. Fix comparison handling to support sequences on both sides of the operator. Added more tests for named vars. Update dependencies. Increment version. --- package.json | 17 +++--- src/iterators.ts | 10 +++- src/json-path-statement.ts | 7 +-- "src/\306\222-base.ts" | 107 +++++++++++++++++++++++++++++-------- test/codegen-tests.ts | 46 ++++++++++++---- test/statement-tests.ts | 74 +++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 45 deletions(-) diff --git a/package.json b/package.json index 347c05e..5d08776 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sql-jsonpath-js", - "version": "0.1.0", + "version": "0.2.0", "description": "JavaScript implementation of the SQL/JSONPath API from SQL2016.", "type": "module", "types": "./dist/index.d.ts", @@ -28,18 +28,19 @@ "url": "https://github.com/mattbishop/sql-jsonpath-js" }, "dependencies": { - "chevrotain": "^11.0.0", + "chevrotain": "^11.0.3", + "indexed-iterable": "^1.0.2", "iterare": "^1.2.1", "luxon": "^3.3.0" }, "devDependencies": { - "@types/chai": "4.3.5", - "@types/luxon": "3.3.0", - "@types/mocha": "10.0.1", - "@types/node": "18.16.19", - "chai": "4.3.7", + "@types/chai": "4.3.11", + "@types/luxon": "3.3.5", + "@types/mocha": "10.0.6", + "@types/node": "20.10.1", + "chai": "4.3.10", "mocha": "10.2.0", "ts-node": "10.9.1", - "typescript": "5.1.6" + "typescript": "5.3.2" } } diff --git a/src/iterators.ts b/src/iterators.ts index 637ac4e..1e9ee74 100644 --- a/src/iterators.ts +++ b/src/iterators.ts @@ -1,3 +1,5 @@ +import {iterate} from "iterare" + export function isAsyncIterable>(input: T | unknown): input is T { return !!input && typeof input === "object" @@ -5,13 +7,15 @@ export function isAsyncIterable>(input: T | unk && Symbol.asyncIterator in input } +export const EMPTY_ITERATOR = iterate([]) + /** * @internal */ export class SingletonIterator implements Iterator { - private done: boolean = false + private done = false constructor(private readonly singleton: T) { } @@ -30,7 +34,7 @@ export class SingletonIterator implements Iterator { */ export class DefaultOnEmptyIterator implements Iterator { - private started: boolean = false + private started = false constructor(private readonly value: T, private iterator: Iterator) { } @@ -40,8 +44,10 @@ export class DefaultOnEmptyIterator implements Iterator { this.started = true const first = this.iterator.next() if (first.done) { + // iterator is empty, vend the default value this.iterator = new SingletonIterator(this.value) } else { + // iterator has at least one value, so vend it back. return first } } diff --git a/src/json-path-statement.ts b/src/json-path-statement.ts index 54e04ff..c41594b 100644 --- a/src/json-path-statement.ts +++ b/src/json-path-statement.ts @@ -4,7 +4,7 @@ import {IteratorWithOperators} from "iterare/lib/iterate.js" import {isIterable} from "iterare/lib/utils.js" import {CodegenContext, newCodegenVisitor} from "./codegen-visitor.js" import {ƒBase} from "./ƒ-base.js" -import {DefaultOnEmptyIterator, DefaultOnErrorIterator, SingletonIterator} from "./iterators.js" +import {DefaultOnEmptyIterator, DefaultOnErrorIterator, EMPTY_ITERATOR, SingletonIterator} from "./iterators.js" import {Input, NamedVariables, SqlJsonPathStatement, StatementConfig} from "./json-path.js" import {JsonPathParser} from "./parser.js" import {allTokens} from "./tokens.js" @@ -47,7 +47,7 @@ export type SJPFn = ($: unknown, $named?: NamedVariables) => IteratorWithOperato * @internal */ export function createFunction({source, lax}: CodegenContext): SJPFn { - const fn = Function("ƒ", "$", "$$", source) + const fn = new Function("ƒ", "$", "$$", source) const ƒ = new ƒBase(lax) return ($, $named = {}) => { @@ -55,6 +55,7 @@ export function createFunction({source, lax}: CodegenContext): SJPFn { if ($named.hasOwnProperty(name)) { return $named[name] } + // thrown for both LAX and STRICT modes throw new Error(`no variable named '$${name}'`) } const result = fn(ƒ, $, $$) @@ -101,7 +102,7 @@ export function createStatement(text: string): SqlJsonPathStatement { values(input: Input, config: StatementConfig = {}): IterableIterator { const {variables} = config const iterator = find(wrapInput(input), variables) - .filter((v) => v !== ƒBase.EMPTY) + .filter((v) => v !== EMPTY_ITERATOR) return defaultsIterator(iterator, config) } } diff --git "a/src/\306\222-base.ts" "b/src/\306\222-base.ts" index 431f476..60de8f0 100644 --- "a/src/\306\222-base.ts" +++ "b/src/\306\222-base.ts" @@ -1,8 +1,10 @@ +import {CachedIterable} from "indexed-iterable" import {iterate} from "iterare" import {IteratorWithOperators} from "iterare/lib/iterate.js" +import {isIterable} from "iterare/lib/utils.js" import {DateTime, FixedOffsetZone} from "luxon" import {KeyValue} from "./json-path" -import {SingletonIterator} from "./iterators.js" +import {EMPTY_ITERATOR, SingletonIterator} from "./iterators.js" enum Pred { @@ -29,8 +31,6 @@ type StrictConfig = { */ export class ƒBase { - static EMPTY = iterate([]) - constructor(private readonly lax: boolean) { } @@ -65,11 +65,43 @@ export class ƒBase { return ƒBase._type(input) === "object" } + /** + * Turn any input into a Seq. Does not consider lax or strict mode. + * @param input the input to Seq. + */ + private static _toSeq(input: unknown): Seq { + return ƒBase._isSeq(input) + ? input.flatten() + : iterate(Array.isArray(input) + ? input + : new SingletonIterator(input)) + } - private _wrap(input: unknown, strict?: StrictConfig): Array { - if (!this.lax && strict && !strict.test(input)) { + + /** + * Examine input with strict test, if any. Throws error if in strict mode and + * the strictness test does not pass. + * @param input the input to test. + * @param strict the strictness config. + * @private + */ + private _checkStrict(input: unknown, strict?: StrictConfig) { + if (this.lax || ƒBase._isSeq(input)) { + return + } + if (strict && !strict.test(input)) { throw new Error(`In 'strict' mode! ${strict.error} Found: ${JSON.stringify(input)}`) } + } + + /** + * Turn any input, like an iterator, into an array. Only used in lax mode. + * @param input The input to wrap. + * @param strict strict config, if any. + * @private + */ + private _wrap(input: unknown, strict?: StrictConfig): Array { + this._checkStrict(input, strict) if (ƒBase._isSeq(input)) { return input.map((v) => Array.isArray(v) ? v : [v]) .toArray() @@ -80,21 +112,21 @@ export class ƒBase { } /** - * Turn an array into an iterator. Only used in lax mode. + * Turn any input, like an array, into an iterator. Only used in lax mode. * @param input The input to unwrap. * @param strict strict config, if any. * @private */ private _unwrap(input: unknown, strict?: StrictConfig): Seq { - if (!this.lax && strict && !strict.test(input)) { - throw new Error(`In 'strict' mode! ${strict.error} Found: ${JSON.stringify(input)}`) - } - if (ƒBase._isSeq(input)) { - return input.flatten() + if (this.lax) { + return ƒBase._toSeq(input) } - return iterate(Array.isArray(input) + + this._checkStrict(input, strict) + + return ƒBase._isSeq(input) ? input - : new SingletonIterator(input)) + : iterate(new SingletonIterator(input)) } private static _autoFlatMap>(input: unknown, mapƒ: Mapƒ): I { @@ -119,7 +151,7 @@ export class ƒBase { private static _objectValues(input: unknown): Seq { return ƒBase._isObject(input) ? iterate(Object.values(input)) - : ƒBase.EMPTY + : EMPTY_ITERATOR } private static _mustBeNumber(input: SingleOrIterator, method: string): number { @@ -238,7 +270,9 @@ export class ƒBase { private _boxStar(input: unknown): Seq { - return this._unwrap(input, { test: Array.isArray, error: "[*] can only be applied to an array in strict mode." }) + // [*] is not the same as unwrap. it always turns the array into a seq + this._checkStrict(input, { test: Array.isArray, error: "[*] can only be applied to an array in strict mode." }) + return ƒBase._toSeq(input) } boxStar(input: unknown): Seq { @@ -251,7 +285,7 @@ export class ƒBase { return obj[member] } if (this.lax) { - return ƒBase.EMPTY + return EMPTY_ITERATOR } throw new Error(`Object does not contain key ${member}, in strict mode.`) } @@ -259,7 +293,7 @@ export class ƒBase { private _member(input: unknown, member: string): Seq { return this._unwrap(input, { test: ƒBase._isObject, error: ".member can only be applied to an object." }) .map((i) => this._getMember(i, member)) - .filter((i) => i !== ƒBase.EMPTY) + .filter((i) => i !== EMPTY_ITERATOR) } member(input: unknown, member: string): Seq { @@ -275,7 +309,7 @@ export class ƒBase { : value } if (this.lax) { - return ƒBase.EMPTY + return EMPTY_ITERATOR } throw new Error (`In 'strict' mode. Array subscript [${pos}] is out of bounds.`) } @@ -369,9 +403,38 @@ export class ƒBase { } compare(compOp: string, left: unknown, right: any): SingleOrIterator { - return ƒBase._autoMap(left, (l) => ƒBase._compare(compOp, l, right)) - } + if (!this.lax) { + if (Array.isArray(left)) { + throw new Error("In 'strict' mode! left side of comparison cannot be an array.") + } + if (Array.isArray(right)) { + throw new Error("In 'strict' mode! right side of comparison cannot be an array.") + } + } + + let rightCompare + if (isIterable(right)) { + const resettableRight = new CachedIterable(right) + rightCompare = (l: any) => { + let retVal = Pred.FALSE + for (const r of resettableRight) { + retVal = ƒBase._compare(compOp, l, r) + if (retVal == Pred.TRUE) { + break + } + } + return retVal + } + } else { + rightCompare = (l: any) => ƒBase._compare(compOp, l, right) + } + left = Array.isArray(left) + ? iterate(left) + : left + + return ƒBase._autoMap(left, rightCompare) + } not(input: any): Pred { return input === Pred.TRUE @@ -408,12 +471,12 @@ export class ƒBase { if (ƒBase._isSeq(result)) { const next = result.next() value = next.done - ? ƒBase.EMPTY + ? EMPTY_ITERATOR : next.value } else { value = result } - return ƒBase._toPred(value !== ƒBase.EMPTY) + return ƒBase._toPred(value !== EMPTY_ITERATOR) } catch (e) { return Pred.UNKNOWN } diff --git a/test/codegen-tests.ts b/test/codegen-tests.ts index 5aeb6b7..bdbfc86 100644 --- a/test/codegen-tests.ts +++ b/test/codegen-tests.ts @@ -358,13 +358,21 @@ describe("Codegen tests", () => { expect(() => fn({t: "shirt"})).to.throw }) - it("[*] iterator values", () => { + it("lax [*][*] iterator values", () => { const ctx = generateFunctionSource('$[*][*]') expect (ctx.source).to.equal('return ƒ.boxStar(ƒ.boxStar($))') const fn = createFunctionForTest(ctx) const arrayDates = fn([[77, 88], [14, 16], [true, false], [["a", "b"]]]) expect(arrayDates).to.deep.equal([77, 88, 14, 16, true, false, ["a", "b"]]) }) + + it("strict [*][*] iterator values", () => { + const ctx = generateFunctionSource('strict $[*][*]') + expect (ctx.source).to.equal('return ƒ.boxStar(ƒ.boxStar($))') + const fn = createFunctionForTest(ctx) + const arrayDates = fn([[77, 88], [14, 16], [true, false], [["a", "b"]]]) + expect(arrayDates).to.deep.equal([77, 88, 14, 16, true, false, ["a", "b"]]) + }) }) }) @@ -590,7 +598,7 @@ describe("Codegen tests", () => { }) describe("filter", () => { - describe("compare", () => { + describe("lax compare", () => { it("can filter comparison predicates", () => { const ctx = generateFunctionSource('$ ? (@ == 1)') expect(ctx.source).to.equal('return ƒ.filter($,v=>ƒ.compare("==",v,1))') @@ -614,19 +622,37 @@ describe("Codegen tests", () => { const actual = fn([[1], [21, 7], [5, 1]]) expect(actual).to.deep.equal([[1], [5, 1]]) }) + + it("can filter value accessor predicates", () => { + const ctx = generateFunctionSource('$ ? (@.sleepy == true)') + expect(ctx.source).to.equal('return ƒ.filter($,v=>ƒ.compare("==",ƒ.member(v,"sleepy"),true))') + const fn = createFunctionForTest(ctx) + const actual = fn([{sleepy: true}, {sleepy: false}, {sleepy: "yes"}, {not: 1}]) + expect(actual).to.deep.equal([{sleepy: true}]) + }) }) - it("can filter value accessor predicates", () => { - const ctx = generateFunctionSource('strict $ ? (@.sleepy == true)') - expect(ctx.source).to.equal('return ƒ.filter($,v=>ƒ.compare("==",ƒ.member(v,"sleepy"),true))') - const fn = createFunctionForTest(ctx) - const actual = fn([{sleepy: true}, {sleepy: false}, {sleepy: "yes"}, {not: 1}]) - expect(actual).to.deep.equal([{sleepy: true}]) + describe("strict filter", () => { + it("filter does not unwrap arrays in strict mode, and does not throw errors", () => { + const ctx = generateFunctionSource('strict $ ? (@.sleepy == true)') + expect(ctx.source).to.equal('return ƒ.filter($,v=>ƒ.compare("==",ƒ.member(v,"sleepy"),true))') + const fn = createFunctionForTest(ctx) + const actual = fn([{sleepy: true}, {sleepy: false}, {sleepy: "yes"}, {not: 1}]) + expect(actual).to.deep.equal([]) + }) + + it("can filter predicate", () => { + const ctx = generateFunctionSource('strict $ ? (@ == 1)') + expect(ctx.source).to.equal('return ƒ.filter($,v=>ƒ.compare("==",v,1))') + const fn = createFunctionForTest(ctx) + const actual = fn(1) + expect(actual).to.deep.equal([1]) + }) }) describe("exists", () => { it("can filter predicates on members", () => { - const ctx = generateFunctionSource('strict $ ? (exists(@.z))') + const ctx = generateFunctionSource('$ ? (exists(@.z))') expect(ctx.source).to.equal('return ƒ.filter($,v=>ƒ.exists(()=>(ƒ.member(v,"z"))))') const fn = createFunctionForTest(ctx) const actual = fn([{z: true}, {y: false}, {a: "yes"}]) @@ -642,7 +668,7 @@ describe("Codegen tests", () => { }) it("can filter predicates and extract members", () => { - const ctx = generateFunctionSource('strict $ ? (exists(@.z)).z') + const ctx = generateFunctionSource('$ ? (exists(@.z)).z') expect(ctx.source).to.equal('return ƒ.member(ƒ.filter($,v=>ƒ.exists(()=>(ƒ.member(v,"z")))),"z")') const fn = createFunctionForTest(ctx) const actual = fn([{z: 121.2}, {y: -99.828}, {a: "yes"}]) diff --git a/test/statement-tests.ts b/test/statement-tests.ts index ebe0712..8c72791 100644 --- a/test/statement-tests.ts +++ b/test/statement-tests.ts @@ -80,6 +80,80 @@ describe("Statement tests", () => { expect(actual.next().done).to.be.true }) + it("unwraps sequence of arrays", () => { + const stmt = compile('$.things[*][*]') + const actual = stmt.values({things:[["matt", true], 100, ["mary", "abby"], [{a: 4}]]}) + expect(one(actual)).to.equal("matt") + expect(one(actual)).to.be.true + expect(one(actual)).to.equal(100) + expect(one(actual)).to.equal("mary") + expect(one(actual)).to.equal("abby") + expect(one(actual)).to.deep.equal({a: 4}) + expect(actual.next().done).to.be.true + }) + + it("strictly unwraps sequence of arrays", () => { + const stmt = compile('strict $.things[*][*]') + const actual = stmt.values({things:[["matt", true], [1, 2], [{a: 4}]]}) + expect(one(actual)).to.equal("matt") + expect(one(actual)).to.be.true + expect(one(actual)).to.equal(1) + expect(one(actual)).to.equal(2) + expect(one(actual)).to.deep.equal({a: 4}) + expect(actual.next().done).to.be.true + }) + + it("searches array with an named array value on right side of comparison", () => { + const stmt = compile('$.players ? (@ == $names[*])') + const variables = {names:["mary", "angie"]} + const actual = stmt.values({players:["matt", "angie", "mark", "mary", "abby"]}, {variables}) + expect(one(actual)).to.equal("angie") + expect(one(actual)).to.equal("mary") + expect(actual.next().done).to.be.true + }) + + it("strict searches array with an named array value on right side of comparison", () => { + const stmt = compile('strict $ ? (@[*] == "mary")') + const actual = stmt.values([["mary", "angie"]]) + const first = one(actual) + expect(first).to.deep.equal(["mary", "angie"]) + expect(actual.next().done).to.be.true + }) + + it("searches array with an named array value on right side of comparison", () => { + const stmt = compile('$.players ? (@ == $names)') + const variables = {names:["mary", "angie"]} + const actual = stmt.values({players:["matt", "angie", "mark", "mary", "abby"]}, {variables}) + expect(one(actual)).to.equal("angie") + expect(one(actual)).to.equal("mary") + expect(actual.next().done).to.be.true + }) + + it("strict does not unwrap named array, but also does not throw an error.", () => { + const stmt = compile('strict $.players ? ($names == @)') + const variables = {names:["mary", "angie"]} + const actual = stmt.values({players:["matt", "angie", "mark", "mary", "abby"]}, {variables}) + expect(actual.next().done).to.be.true + }) + + it("searches array with an unwrapped named array value on left side of comparison", () => { + const stmt = compile('$.players ? ($names == @)') + const variables = {names:["mary", "angie"]} + const actual = stmt.values({players:["matt", "angie", "mark", "mary", "abby"]}, {variables}) + expect(one(actual)).to.equal("angie") + expect(one(actual)).to.equal("mary") + expect(actual.next().done).to.be.true + }) + + it("searches array with a specific element in a named array value", () => { + const stmt = compile('$.players ? ($names.first[1] == @)') + // return ƒ.filter(ƒ.member($,"players"),v=>ƒ.compare("==",v,ƒ.array(ƒ.member($$("names"),"first"),[1]))) + const variables = {names:{first:["mary", "angie"]}} + const actual = stmt.values({players:["matt", "angie", "mark", "mary", "abby"]}, {variables}) + expect(one(actual)).to.equal("angie") + expect(actual.next().done).to.be.true + }) + it("can do arithmetic", () => { const stmt = compile('$ ? (@ % 2 == 0)') const actual = stmt.values([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])