diff --git a/packages/common/src/content/_fetch.ts b/packages/common/src/content/_fetch.ts index 34f4343a284..4fcc09893ee 100644 --- a/packages/common/src/content/_fetch.ts +++ b/packages/common/src/content/_fetch.ts @@ -1,8 +1,7 @@ import { IItem } from "@esri/arcgis-rest-portal"; -import { BBox } from ".."; import { ItemOrServerEnrichment } from "../items/_enrichments"; import { hubApiRequest } from "../request"; -import { IHubRequestOptions, IHubGeography } from "../types"; +import { BBox, IHubRequestOptions, IHubGeography } from "../types"; import { isMapOrFeatureServerUrl } from "../urls"; import { cloneObject } from "../util"; import { includes } from "../utils"; diff --git a/packages/common/src/content/compose.ts b/packages/common/src/content/compose.ts index e1ae5957e2d..f69d1cc48db 100644 --- a/packages/common/src/content/compose.ts +++ b/packages/common/src/content/compose.ts @@ -498,6 +498,51 @@ export interface IComposeContentOptions extends IHubContentEnrichments { searchDescription?: string; } +/** + * get the layer object for + * - an item that refers to a specific layer of a service + * - a multi-layer services (if a layer id was passed in) + * - a single layer feature service + * @param item + * @param layers the layers and tables returned from the service + * @param layerId a specific id + * @returns layer definition + * @private + */ +export const getItemLayer = ( + item: IItem, + layers: ILayerDefinition[], + layerId?: number +) => { + // if item refers to a layer we always want to use that layer id + // otherwise use the layer id that was passed in (if any) + const _layerIdFromUrl = getLayerIdFromUrl(item.url); + const _layerId = _layerIdFromUrl ? parseInt(_layerIdFromUrl, 10) : layerId; + return ( + layers && + (!isNil(_layerId) + ? // find the explicitly set layer id + layers.find((_layer) => _layer.id === _layerId) + : // for feature servers with a single layer always show the layer + isFeatureService(item.type) && getOnlyQueryLayer(layers)) + ); +}; + +// TODO: we should re-define ILayerDefinition +// in IServerEnrichments.ts to include isView +interface ILayerViewDefinition extends ILayerDefinition { + isView?: boolean; +} + +/** + * determine if a layer is a layer view + * @param layer + * @returns + * @private + */ +export const isLayerView = (layer: ILayerDefinition) => + (layer as ILayerViewDefinition).isView; + /** * Compose a new content object out of an item, enrichments, and context * @param item @@ -528,19 +573,7 @@ export const composeContent = ( } = options || {}; // set common variables that we will use in the derived properties below - // if item refers to a layer we always want to use that layer id - // otherwise use the layer id that was passed in (if any) - const _layerIdFromUrl = getLayerIdFromUrl(item.url); - const layerId = _layerIdFromUrl - ? parseInt(_layerIdFromUrl, 10) - : options?.layerId; - const layer = - layers && - (!isNil(layerId) - ? // find the explicitly set layer id - layers.find((_layer) => _layer.id === layerId) - : // for feature servers with a single layer always show the layer - isFeatureService(item.type) && getOnlyQueryLayer(layers)); + const layer = getItemLayer(item, layers, options?.layerId); // NOTE: we only set hubId for public items in online const hubId = canUseHubApiForItem(item, requestOptions) ? layer @@ -750,6 +783,17 @@ export const composeContent = ( null ); }, + get viewDefinition() { + // if this is a layer view and we have the item data + // find the definition that corresponds to the current layer + const dataLayer = + layer && + isLayerView(layer) && + data && + Array.isArray(data.layers) && + data.layers.find((_layer) => _layer.id === layer.id); + return dataLayer ? dataLayer.layerDefinition : undefined; + }, get orgId() { // NOTE: it's undocumented, but the portal API will return orgId for items... sometimes return org ? org.id : item.orgId || ownerUser?.orgId; diff --git a/packages/common/src/content/fetch.ts b/packages/common/src/content/fetch.ts index 1f9ede16eaa..cd21dde0067 100644 --- a/packages/common/src/content/fetch.ts +++ b/packages/common/src/content/fetch.ts @@ -1,8 +1,10 @@ +import { queryFeatures } from "@esri/arcgis-rest-feature-layer"; import { getItem } from "@esri/arcgis-rest-portal"; -import { composeContent } from "./compose"; +import { IHubContent } from "../core"; import { ItemOrServerEnrichment, fetchItemEnrichments, + IItemAndEnrichments, } from "../items/_enrichments"; import { IHubRequestOptions } from "../types"; import { isNil } from "../util"; @@ -14,6 +16,7 @@ import { getContentEnrichments, } from "./_fetch"; import { canUseHubApiForItem } from "./_internal"; +import { composeContent, getItemLayer, isLayerView } from "./compose"; interface IFetchItemAndEnrichmentsOptions extends IHubRequestOptions { enrichments?: ItemOrServerEnrichment[]; @@ -24,13 +27,34 @@ export interface IFetchContentOptions extends IFetchItemAndEnrichmentsOptions { siteOrgKey?: string; } +const maybeFetchLayerEnrichments = async ( + itemAndEnrichments: IItemAndEnrichments, + options?: IFetchContentOptions +) => { + // determine if this is a client-side feature layer view + const { item, data, layers } = itemAndEnrichments; + const layer = + layers && getItemLayer(item, layers, options && options.layerId); + // TODO: add recordCount here too? + const layerEnrichments = + layer && isLayerView(layer) && !data + ? // NOTE: I'm not sure what conditions causes a layer view + // to store (at least part of) it's view definition in item data + // it seems that most do not, but until we have a reliable signal + // we just fetch the item data for all layer views + await fetchItemEnrichments(item, ["data"], options) + : undefined; + // TODO: merge errors + return { ...itemAndEnrichments, ...layerEnrichments }; +}; + const fetchItemAndEnrichments = async ( itemId: string, - options?: IFetchItemAndEnrichmentsOptions + options?: IFetchContentOptions ) => { - // TODO: add error handling + // fetch the item const item = await getItem(itemId, options); - // TODO: allow override enrichments + // fetch the enrichments const enrichmentsToFetch = options?.enrichments || getContentEnrichments(item); const enrichments = await fetchItemEnrichments( @@ -38,7 +62,7 @@ const fetchItemAndEnrichments = async ( enrichmentsToFetch, options ); - return { ...enrichments, item }; + return maybeFetchLayerEnrichments({ ...enrichments, item }, options); }; const fetchContentById = async ( @@ -52,8 +76,7 @@ const fetchContentById = async ( options ); // did the caller request a specific layer - const specifiedLayerId = - options && !isNil(options.layerId) && options.layerId; + const specifiedLayerId = options && options.layerId; // if this is a public item and we're not in enterprise // fetch the slug and remaining enrichments from the Hub API const { slug, layerId, boundary, extent, searchDescription, statistics } = @@ -65,7 +88,7 @@ const fetchContentById = async ( requestOptions: options, ...itemEnrichments, slug, - layerId: specifiedLayerId || layerId, + layerId: isNil(specifiedLayerId) ? layerId : specifiedLayerId, boundary, extent, searchDescription, @@ -119,6 +142,17 @@ const fetchContentBySlug = async ( }); }; +const fetchContentRecordCount = async (content: IHubContent) => { + const { url, viewDefinition } = content; + const where = viewDefinition?.definitionExpression; + const response: any = await queryFeatures({ + url, + where, + returnCountOnly: true, + }); + return response.count; +}; + /** * Fetch enriched content from the Portal and Hub APIs. * @param identifier content slug or id @@ -131,14 +165,19 @@ const fetchContentBySlug = async ( * const content = await fetchContent('my-org::item-name') * ``` */ -export const fetchContent = ( +export const fetchContent = async ( identifier: string, options?: IFetchContentOptions ) => { - return isSlug(identifier) - ? fetchContentBySlug( + const content = isSlug(identifier) + ? await fetchContentBySlug( addContextToSlug(identifier, options?.siteOrgKey), options ) - : fetchContentById(identifier, options); + : await fetchContentById(identifier, options); + // fetch record count for layers, tables, or proxied CSVs + const { isProxied, layer } = content; + content.recordCount = + isProxied || !!layer ? await fetchContentRecordCount(content) : undefined; + return content; }; diff --git a/packages/common/src/core/types/IHubContent.ts b/packages/common/src/core/types/IHubContent.ts index ab34438b8db..bd2473ba8f6 100644 --- a/packages/common/src/core/types/IHubContent.ts +++ b/packages/common/src/core/types/IHubContent.ts @@ -114,6 +114,11 @@ export interface IHubContent /** links to additional resources specified in the formal item metadata */ additionalResources?: IHubAdditionalResource[]; + /** definition for content that refers to a client-side layer view */ + viewDefinition?: { definitionExpression?: string }; + + // TODO: metrics, urls, publisher, etc? + /////////// // TODO: remove these deprecated props at the next breaking version ////////// @@ -126,6 +131,4 @@ export interface IHubContent /* DEPRECATED: use org.id instead */ orgId?: string; - - // TODO: metrics, urls, publisher, etc? } diff --git a/packages/common/test/content/fetch.test.ts b/packages/common/test/content/fetch.test.ts index 713201d0141..1164c78e7b6 100644 --- a/packages/common/test/content/fetch.test.ts +++ b/packages/common/test/content/fetch.test.ts @@ -1,5 +1,6 @@ import { IPolygon } from "@esri/arcgis-rest-types"; import * as portalModule from "@esri/arcgis-rest-portal"; +import * as featureLayerModule from "@esri/arcgis-rest-feature-layer"; import { IHubRequestOptions, fetchContent, @@ -12,6 +13,26 @@ import * as _fetchModule from "../../src/content/_fetch"; import * as documentItem from "../mocks/items/document.json"; import * as multiLayerFeatureServiceItem from "../mocks/items/multi-layer-feature-service.json"; +// mock the item enrichments that would be returned for a multi-layer service +const getMultiLayerItemEnrichments = () => { + const layer = { id: 0, name: "layer0 " }; + const table = { id: 1, name: "table1 " }; + const item = multiLayerFeatureServiceItem as unknown as portalModule.IItem; + return { + item, + groupIds: ["foo", "bar"], + metadata: null as any, + ownerUser: { + username: item.owner, + }, + server: { + currentVersion: 10.71, + serviceDescription: "For demo purposes only.", + }, + layers: [layer, table], + }; +}; + describe("fetchContent", () => { let portal: string; let hubApiUrl: string; @@ -54,6 +75,7 @@ describe("fetchContent", () => { const getItemSpy = spyOn(portalModule, "getItem").and.returnValue( Promise.resolve(documentItem) ); + const queryFeaturesSpy = spyOn(featureLayerModule, "queryFeatures"); // call fetch content const options = { ...requestOpts, @@ -67,6 +89,7 @@ describe("fetchContent", () => { expect(fetchHubEnrichmentsSpy).toHaveBeenCalledWith(slug, options); expect(getItemSpy).toHaveBeenCalledTimes(1); expect(getItemSpy).toHaveBeenCalledWith(itemId, options); + expect(queryFeaturesSpy).not.toHaveBeenCalled(); // inspect the results expect(result.item).toEqual(documentItem); expect(result.boundary).toEqual({ @@ -76,6 +99,7 @@ describe("fetchContent", () => { expect(result.statistics).toBeUndefined(); }); describe("with a specific layer", () => { + let itemEnrichments: _enrichmentsModule.IItemAndEnrichments; beforeEach(() => { // initialize the above above variables for the multi-layer feature service item // see: https://hubqa.arcgis.com/api/v3/datasets/eda6e08848924887b00a67ca319ec1bf @@ -111,10 +135,11 @@ describe("fetchContent", () => { // "setBy": null }, }; + itemEnrichments = getMultiLayerItemEnrichments(); }); it("should re-fetch hub enrichments for feature services", async () => { // expected layer - const layerId = 2; + const layerId = 1; // the hub API enrichments that we expect for the above layer // see: https://hubqa.arcgis.com/api/v3/datasets/eda6e08848924887b00a67ca319ec1bf_2?fields[datasets]=slug,boundary,statistics const layerHubEnrichments = { @@ -127,6 +152,10 @@ describe("fetchContent", () => { statistics: {}, }; // initialize the spies + const fetchItemEnrichmentsSpy = spyOn( + _enrichmentsModule, + "fetchItemEnrichments" + ).and.returnValue(Promise.resolve(itemEnrichments)); const fetchHubEnrichmentsSpy = spyOn( _fetchModule, "fetchHubEnrichmentsBySlug" @@ -138,13 +167,16 @@ describe("fetchContent", () => { _fetchModule, "fetchHubEnrichmentsById" ).and.returnValue(Promise.resolve(layerHubEnrichments)); + const count = 100; + const queryFeaturesSpy = spyOn( + featureLayerModule, + "queryFeatures" + ).and.returnValue(Promise.resolve({ count })); // call fetch content const options = { ...requestOpts, siteOrgKey, layerId, - // this keeps us from having to mock the call to fetch item enrichments - enrichments: [], } as IFetchContentOptions; const result = await fetchContent(shortSlug, options); // inspect the calls @@ -152,11 +184,18 @@ describe("fetchContent", () => { expect(fetchHubEnrichmentsSpy).toHaveBeenCalledWith(slug, options); expect(getItemSpy).toHaveBeenCalledTimes(1); expect(getItemSpy).toHaveBeenCalledWith(itemId, options); + expect(fetchItemEnrichmentsSpy).toHaveBeenCalledTimes(1); + expect(fetchItemEnrichmentsSpy).toHaveBeenCalledWith( + multiLayerFeatureServiceItem, + ["groupIds", "metadata", "ownerUser", "org", "server", "layers"], + options + ); expect(fetchLayerHubEnrichmentsSpy).toHaveBeenCalledTimes(1); expect(fetchLayerHubEnrichmentsSpy).toHaveBeenCalledWith( `${itemId}_${layerId}`, options ); + expect(queryFeaturesSpy).toHaveBeenCalledTimes(1); // inspect the results expect(result.item).toEqual( multiLayerFeatureServiceItem as unknown as portalModule.IItem @@ -166,6 +205,105 @@ describe("fetchContent", () => { expect(result.boundary).toBe(layerHubEnrichments.boundary); expect(result.boundary).not.toBe(hubEnrichments.boundary); expect(result.statistics).toEqual(layerHubEnrichments.statistics); + expect(result.recordCount).toBe(count); + }); + it("should fetch view definition for client-side layer views", async () => { + const layerId = 2; + // NOTE: testing edge case here by pretending that + // this layer view ends up filtering out all records + const definitionExpression = "1=2"; + const count = 0; + // mock the data response for a client-side layer view + const data = { + layers: [ + { + id: layerId, + layerDefinition: { + definitionExpression, + }, + }, + ], + }; + // expected layer + itemEnrichments.layers.push({ + id: layerId, + isView: true, + name: "layerView", + } as any); + // the hub API enrichments that we expect for the above layer + // see: https://hubqa.arcgis.com/api/v3/datasets/eda6e08848924887b00a67ca319ec1bf_2?fields[datasets]=slug,boundary,statistics + const layerHubEnrichments = { + itemId, + layerId, + slug: "dc::municipal-fire-stations", + // NOTE: in this case the layer's boundary is the same as the parent's + // for now I'm just cloning it so that we can test reference equality below + boundary: cloneObject(hubEnrichments.boundary), + statistics: {}, + }; + // initialize the spies + const fetchItemEnrichmentsSpy = spyOn( + _enrichmentsModule, + "fetchItemEnrichments" + ).and.callFake( + ( + item: any, + enrichments: _enrichmentsModule.ItemOrServerEnrichment[] + ) => { + return Promise.resolve( + enrichments.length === 1 && enrichments[0] === "data" + ? { data } + : itemEnrichments + ); + } + ); + const fetchHubEnrichmentsSpy = spyOn( + _fetchModule, + "fetchHubEnrichmentsBySlug" + ).and.returnValue(Promise.resolve(layerHubEnrichments)); + const getItemSpy = spyOn(portalModule, "getItem").and.returnValue( + Promise.resolve(multiLayerFeatureServiceItem) + ); + const queryFeaturesSpy = spyOn( + featureLayerModule, + "queryFeatures" + ).and.returnValue(Promise.resolve({ count })); + // call fetch content + // NOTE: not passing siteOrgKey and instead using fully qualified slug for layer + slug = layerHubEnrichments.slug; + const options = { + ...requestOpts, + layerId, + } as IFetchContentOptions; + const result = await fetchContent(slug, options); + // inspect the calls + expect(fetchHubEnrichmentsSpy).toHaveBeenCalledTimes(1); + expect(fetchHubEnrichmentsSpy).toHaveBeenCalledWith(slug, options); + expect(getItemSpy).toHaveBeenCalledTimes(1); + expect(getItemSpy).toHaveBeenCalledWith(itemId, options); + expect(fetchItemEnrichmentsSpy).toHaveBeenCalledTimes(2); + expect(fetchItemEnrichmentsSpy.calls.argsFor(0)[1]).toEqual([ + "groupIds", + "metadata", + "ownerUser", + "org", + "server", + "layers", + ]); + expect(fetchItemEnrichmentsSpy.calls.argsFor(1)[1]).toEqual(["data"]); + expect(queryFeaturesSpy).toHaveBeenCalledTimes(1); + const queryFeaturesArg = queryFeaturesSpy.calls.argsFor(0)[0] as any; + expect(queryFeaturesArg.url).toEqual(result.url); + expect(queryFeaturesArg.returnCountOnly).toBeTruthy(); + expect(queryFeaturesArg.where).toBe(definitionExpression); + // inspect the results + expect(result.item).toEqual( + multiLayerFeatureServiceItem as unknown as portalModule.IItem + ); + expect(result.viewDefinition.definitionExpression).toBe( + definitionExpression + ); + expect(result.recordCount).toBe(count); }); }); describe("with defaults", () => { @@ -192,6 +330,7 @@ describe("fetchContent", () => { _enrichmentsModule, "fetchItemEnrichments" ).and.returnValue(Promise.resolve(itemEnrichments)); + const queryFeaturesSpy = spyOn(featureLayerModule, "queryFeatures"); // call fetch content const options = { ...requestOpts, @@ -209,6 +348,7 @@ describe("fetchContent", () => { ["groupIds", "metadata", "ownerUser", "org"], options ); + expect(queryFeaturesSpy).not.toHaveBeenCalled(); // inspect the results expect(result.item).toEqual(documentItem); expect(result.boundary).toEqual({ @@ -230,6 +370,7 @@ describe("fetchContent", () => { _enrichmentsModule, "fetchItemEnrichments" ).and.returnValue(Promise.resolve(itemEnrichments)); + const queryFeaturesSpy = spyOn(featureLayerModule, "queryFeatures"); // call fetch content // NOTE: we have to pass the fully qualified slug if not passing options const result = await fetchContent(slug); @@ -244,6 +385,7 @@ describe("fetchContent", () => { ["groupIds", "metadata", "ownerUser", "org"], undefined ); + expect(queryFeaturesSpy).not.toHaveBeenCalled(); // inspect the results expect(result.item).toEqual(documentItem); expect(result.boundary).toEqual({ @@ -264,6 +406,7 @@ describe("fetchContent", () => { const getItemSpy = spyOn(portalModule, "getItem").and.returnValue( Promise.resolve(documentItem) ); + const queryFeaturesSpy = spyOn(featureLayerModule, "queryFeatures"); // call fetch content const options = { ...requestOpts, @@ -276,6 +419,7 @@ describe("fetchContent", () => { expect(fetchHubEnrichmentsSpy).toHaveBeenCalledWith(itemId, options); expect(getItemSpy).toHaveBeenCalledTimes(1); expect(getItemSpy).toHaveBeenCalledWith(itemId, options); + expect(queryFeaturesSpy).not.toHaveBeenCalled(); // inspect the results expect(result.item).toEqual(documentItem); expect(result.boundary).toEqual({ @@ -300,6 +444,7 @@ describe("fetchContent", () => { const getItemSpy = spyOn(portalModule, "getItem").and.returnValue( Promise.resolve(documentItem) ); + const queryFeaturesSpy = spyOn(featureLayerModule, "queryFeatures"); // call fetch content const options = { ...requestOpts, @@ -311,6 +456,7 @@ describe("fetchContent", () => { expect(fetchHubEnrichmentsSpy).not.toHaveBeenCalled(); expect(getItemSpy).toHaveBeenCalledTimes(1); expect(getItemSpy).toHaveBeenCalledWith(itemId, options); + expect(queryFeaturesSpy).not.toHaveBeenCalled(); // inspect the results expect(result.item).toEqual(documentItem); expect(result.boundary).toEqual({ @@ -327,21 +473,8 @@ describe("fetchContent", () => { // NOTE: other tests pass enrichments: [] to avoid stubbing fetchItemEnrichments // this test covers the more common use case where we fetch the default enrichments // for a feature service item. - const layer = { id: 0, name: "layer0 " }; - const table = { id: 1, name: "table1 " }; - const itemEnrichments = { - groupIds: ["foo", "bar"], - metadata: null, - ownerUser: { - username: multiLayerFeatureServiceItem.owner, - }, - server: { - currentVersion: 10.71, - serviceDescription: "For demo purposes only.", - }, - layers: [layer, table], - } as IItemEnrichments; - const layerId = 1; + const itemEnrichments = getMultiLayerItemEnrichments(); + const layerId = 0; // initialize the spies const getItemSpy = spyOn(portalModule, "getItem").and.returnValue( Promise.resolve(multiLayerFeatureServiceItem) @@ -350,6 +483,11 @@ describe("fetchContent", () => { _enrichmentsModule, "fetchItemEnrichments" ).and.returnValue(Promise.resolve(itemEnrichments)); + const count = 1000; + const queryFeaturesSpy = spyOn( + featureLayerModule, + "queryFeatures" + ).and.returnValue(Promise.resolve({ count })); // call fetch content const options = { ...requestOpts, @@ -365,13 +503,19 @@ describe("fetchContent", () => { ["groupIds", "metadata", "ownerUser", "org", "server", "layers"], options ); + expect(queryFeaturesSpy).toHaveBeenCalledTimes(1); + const queryFeaturesArg = queryFeaturesSpy.calls.argsFor(0)[0] as any; + expect(queryFeaturesArg.url).toEqual(result.url); + expect(queryFeaturesArg.returnCountOnly).toBeTruthy(); + expect(queryFeaturesArg.where).toBeUndefined(); // inspect the results expect(result.item).toEqual( multiLayerFeatureServiceItem as portalModule.IItem ); // expect(result.boundary).toEqual({ geometry: undefined, provenance: undefined }) expect(result.statistics).toBeUndefined(); - expect(result.layer).toEqual(table); + expect(result.layer.id).toBe(layerId); + expect(result.recordCount).toEqual(count); }); }); });