diff --git a/packages/downloads/src/hub/hub-poll-download-metadata.ts b/packages/downloads/src/hub/hub-poll-download-metadata.ts index d2daf53fdb3..7ad612b4970 100644 --- a/packages/downloads/src/hub/hub-poll-download-metadata.ts +++ b/packages/downloads/src/hub/hub-poll-download-metadata.ts @@ -12,6 +12,7 @@ export interface IHubDownloadMetadataPollParameters downloadId: string; eventEmitter: EventEmitter; pollingInterval: number; + existingFileDate?: string; } class HubPoller implements IPoller { @@ -23,11 +24,19 @@ class HubPoller implements IPoller { } activatePoll(params: IHubDownloadMetadataPollParameters) { - const { downloadId, eventEmitter, pollingInterval } = params; + const { + downloadId, + eventEmitter, + pollingInterval, + existingFileDate = new Date(0).toISOString() + } = params; + + const existingFileTimestamp = new Date(existingFileDate).getTime(); + this.pollTimer = setInterval(() => { hubRequestDownloadMetadata(params) .then(metadata => { - if (isUpToDate(metadata)) { + if (isReady(metadata, existingFileTimestamp)) { eventEmitter.emit(`${downloadId}ExportComplete`, { detail: { metadata } }); @@ -65,8 +74,13 @@ export function hubPollDownloadMetadata( return poller; } -function isUpToDate(metadata: any) { - return metadata && metadata.status === "ready"; +function isReady(metadata: any, preExportFileTimestamp: number) { + const { status, lastModified } = metadata; + const currentFileDate = new Date(lastModified).getTime(); + return ( + status === "ready" || + (status === "ready_unknown" && currentFileDate > preExportFileTimestamp) + ); } function exportDatasetFailed(metadata: any) { diff --git a/packages/downloads/src/poll-download-metadata.ts b/packages/downloads/src/poll-download-metadata.ts index 90b2d004702..6b2776cbfad 100644 --- a/packages/downloads/src/poll-download-metadata.ts +++ b/packages/downloads/src/poll-download-metadata.ts @@ -30,6 +30,8 @@ export interface IPollDownloadMetadataRequestParams { jobId?: string; /* Time-stamp for export start. Required for Portal downloads only. */ exportCreated?: number; + /* ISO string of existing export file date. Used to track export progress when lastEditDate is unknown. */ + existingFileDate?: string; } /** @@ -54,7 +56,8 @@ export function pollDownloadMetadata( spatialRefId, geometry, where, - host + host, + existingFileDate } = params; if (target === "portal") { @@ -82,6 +85,7 @@ export function pollDownloadMetadata( pollingInterval, spatialRefId, geometry, - where + where, + existingFileDate }); } diff --git a/packages/downloads/test/hub/hub-poll-download-metadata.test.ts b/packages/downloads/test/hub/hub-poll-download-metadata.test.ts index 136020484ec..e291b5b35a4 100644 --- a/packages/downloads/test/hub/hub-poll-download-metadata.test.ts +++ b/packages/downloads/test/hub/hub-poll-download-metadata.test.ts @@ -6,64 +6,93 @@ function delay(milliseconds: number) { return new Promise(resolve => setTimeout(resolve, milliseconds)); } -const fixtures = { - exportFailed: { - data: [ - { - id: "dd4580c810204019a7b8eb3e0b329dd6_0", - type: "downloads", - attributes: { - spatialRefId: "4326", - format: "CSV", - status: "error_creating", - featureSet: "full", - errors: [{ message: "Some error" }], - source: { - type: "Feature Service", - url: - "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json", - supportsExtract: true, - lastEditDate: "2020-06-18T01:17:31.492Z", - spatialRefId: "4326" - } - }, - links: { - content: - "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv" +const exportFailed = { + data: [ + { + id: "dd4580c810204019a7b8eb3e0b329dd6_0", + type: "downloads", + attributes: { + spatialRefId: "4326", + format: "CSV", + status: "error_creating", + featureSet: "full", + errors: [{ message: "Some error" }], + source: { + type: "Feature Service", + url: + "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json", + supportsExtract: true, + lastEditDate: "2020-06-18T01:17:31.492Z", + spatialRefId: "4326" } + }, + links: { + content: + "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv" } - ] - }, - exportSucceeded: { - data: [ - { - id: "dd4580c810204019a7b8eb3e0b329dd6_0", - type: "downloads", - attributes: { - spatialRefId: "4326", - format: "CSV", - contentLength: 1391454, - lastModified: "2020-06-17T13:04:28.000Z", - contentLastModified: "2020-06-17T01:16:01.933Z", - cacheTime: 13121, - status: "ready", - featureSet: "full", - source: { - type: "Feature Service", - url: - "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json", - supportsExtract: true, - lastEditDate: "2020-06-18T01:15:31.492Z", - spatialRefId: "4326" - } - }, - links: { - content: - "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv" + } + ] +}; + +const exportReady = { + data: [ + { + id: "dd4580c810204019a7b8eb3e0b329dd6_0", + type: "downloads", + attributes: { + spatialRefId: "4326", + format: "CSV", + contentLength: 1391454, + lastModified: "2020-06-17T13:04:28.000Z", + contentLastModified: "2020-06-17T01:16:01.933Z", + cacheTime: 13121, + status: "ready", + featureSet: "full", + source: { + type: "Feature Service", + url: + "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json", + supportsExtract: true, + lastEditDate: "2020-06-18T01:15:31.492Z", + spatialRefId: "4326" } + }, + links: { + content: + "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv" } - ] - } + } + ] +}; + +const exportReadyUnknown = { + data: [ + { + id: "dd4580c810204019a7b8eb3e0b329dd6_0", + type: "downloads", + attributes: { + spatialRefId: "4326", + format: "CSV", + contentLength: 1391454, + lastModified: "2020-06-17T13:04:28.000Z", + contentLastModified: "2020-06-17T13:04:28.000Z", + cacheTime: 13121, + status: "ready_unknown", + featureSet: "full", + source: { + type: "Feature Service", + url: + "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json", + supportsExtract: true, + spatialRefId: "4326" + } + }, + links: { + content: + "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv" + } + } + ] }; describe("hubPollDownloadMetadata", () => { @@ -113,7 +142,7 @@ describe("hubPollDownloadMetadata", () => { "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv", { status: 200, - body: fixtures.exportFailed + body: exportFailed } ); const mockEventEmitter = new EventEmitter(); @@ -155,13 +184,13 @@ describe("hubPollDownloadMetadata", () => { } }); - it("handle successful export", async done => { + it("handle ready export", async done => { try { fetchMock.mock( "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv", { status: 200, - body: fixtures.exportSucceeded + body: exportReady } ); const mockEventEmitter = new EventEmitter(); @@ -203,6 +232,86 @@ describe("hubPollDownloadMetadata", () => { } }); + it("handle ready_unknown export (export complete)", async done => { + try { + fetchMock.mock( + "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv", + { + status: 200, + body: exportReadyUnknown + } + ); + const mockEventEmitter = new EventEmitter(); + spyOn(mockEventEmitter, "emit"); + const poller = hubPollDownloadMetadata({ + host: "http://hub.com", + datasetId: "abcdef0123456789abcdef0123456789_0", + downloadId: "test-id", + spatialRefId: "4326", + format: "CSV", + eventEmitter: mockEventEmitter, + pollingInterval: 10, + existingFileDate: "2020-06-15T13:04:28.000Z" + }); + await delay(50); + expect(mockEventEmitter.emit as any).toHaveBeenCalledTimes(1); + const [ + topic, + customEvent + ] = (mockEventEmitter.emit as any).calls.first().args; + expect(topic).toEqual("test-idExportComplete"); + expect(customEvent.detail.metadata).toEqual({ + downloadId: + "abcdef0123456789abcdef0123456789_0:CSV:4326:undefined:undefined", + contentLastModified: "2020-06-17T13:04:28.000Z", + lastEditDate: undefined, + lastModified: "2020-06-17T13:04:28.000Z", + status: "ready_unknown", + errors: [], + downloadUrl: + "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv", + contentLength: 1391454, + cacheTime: 13121 + }); + expect(poller.pollTimer).toEqual(null); + } catch (err) { + expect(err).toEqual(undefined); + } finally { + done(); + } + }); + + it("handle ready_unknown export (export queued)", async done => { + try { + fetchMock.mock( + "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv", + { + status: 200, + body: exportReadyUnknown + } + ); + const mockEventEmitter = new EventEmitter(); + spyOn(mockEventEmitter, "emit"); + const poller = hubPollDownloadMetadata({ + host: "http://hub.com", + datasetId: "abcdef0123456789abcdef0123456789_0", + downloadId: "test-id", + spatialRefId: "4326", + format: "CSV", + eventEmitter: mockEventEmitter, + pollingInterval: 10, + existingFileDate: "2020-06-17T13:04:28.000Z" + }); + await delay(50); + expect(mockEventEmitter.emit as any).toHaveBeenCalledTimes(0); + expect(poller.pollTimer !== null).toEqual(true); + } catch (err) { + expect(err).toEqual(undefined); + } finally { + done(); + } + }, 16000); + it("handle multiple polls", async done => { try { fetchMock.mock( diff --git a/packages/downloads/test/poll-download-metadata.test.ts b/packages/downloads/test/poll-download-metadata.test.ts index 2c099d9228c..54c41c57ea0 100644 --- a/packages/downloads/test/poll-download-metadata.test.ts +++ b/packages/downloads/test/poll-download-metadata.test.ts @@ -1,43 +1,48 @@ - -import { UserSession } from '@esri/arcgis-rest-auth'; +import { UserSession } from "@esri/arcgis-rest-auth"; import { pollDownloadMetadata } from "../src/poll-download-metadata"; -import * as hubPoller from '../src/hub/hub-poll-download-metadata'; -import * as portalPoller from '../src/portal/portal-poll-export-job-status' -import * as EventEmitter from 'eventemitter3'; +import * as hubPoller from "../src/hub/hub-poll-download-metadata"; +import * as portalPoller from "../src/portal/portal-poll-export-job-status"; +import * as EventEmitter from "eventemitter3"; describe("pollDownloadMetadata", () => { - it('handle hub polling', async done => { + it("handle hub polling", async done => { try { const mockEventEmitter = new EventEmitter(); - spyOn(mockEventEmitter, 'emit'); + spyOn(mockEventEmitter, "emit"); - spyOn(hubPoller, 'hubPollDownloadMetadata').and.returnValue( + spyOn(hubPoller, "hubPollDownloadMetadata").and.returnValue( new Promise((resolve, reject) => { - resolve(); - })) + resolve(); + }) + ); pollDownloadMetadata({ - host: 'http://hub.com/', - datasetId: 'abcdef0123456789abcdef0123456789_0', - spatialRefId: '4326', - format: 'CSV', - downloadId: 'download-id', + host: "http://hub.com/", + datasetId: "abcdef0123456789abcdef0123456789_0", + spatialRefId: "4326", + format: "CSV", + downloadId: "download-id", eventEmitter: mockEventEmitter, pollingInterval: 10 }); - expect(hubPoller.hubPollDownloadMetadata as any).toHaveBeenCalledTimes(1) - expect((hubPoller.hubPollDownloadMetadata as any).calls.first().args).toEqual([{ - host: 'http://hub.com/', - downloadId: 'download-id', - datasetId: 'abcdef0123456789abcdef0123456789_0', - format: 'CSV', - eventEmitter: mockEventEmitter, - pollingInterval: 10, - spatialRefId: '4326', - geometry: undefined, - where: undefined - }]) + expect(hubPoller.hubPollDownloadMetadata as any).toHaveBeenCalledTimes(1); + expect( + (hubPoller.hubPollDownloadMetadata as any).calls.first().args + ).toEqual([ + { + host: "http://hub.com/", + downloadId: "download-id", + datasetId: "abcdef0123456789abcdef0123456789_0", + format: "CSV", + eventEmitter: mockEventEmitter, + pollingInterval: 10, + spatialRefId: "4326", + geometry: undefined, + where: undefined, + existingFileDate: undefined + } + ]); } catch (err) { expect(err).toBeUndefined(); } finally { @@ -45,56 +50,64 @@ describe("pollDownloadMetadata", () => { } }); - it('handle portal download', async done => { + it("handle portal download", async done => { const authentication = new UserSession({ - username: 'portal-user', - portal: 'http://portal.com/sharing/rest', - token: '123', - }); - authentication.getToken = () => new Promise((resolve) => { - resolve('123') + username: "portal-user", + portal: "http://portal.com/sharing/rest", + token: "123" }); + authentication.getToken = () => + new Promise(resolve => { + resolve("123"); + }); try { const mockEventEmitter = new EventEmitter(); - spyOn(mockEventEmitter, 'emit'); + spyOn(mockEventEmitter, "emit"); - spyOn(portalPoller, 'portalPollExportJobStatus').and.returnValue( + spyOn(portalPoller, "portalPollExportJobStatus").and.returnValue( new Promise((resolve, reject) => { - resolve(); - })) + resolve(); + }) + ); pollDownloadMetadata({ - target: 'portal', - datasetId: 'abcdef0123456789abcdef0123456789_0', - spatialRefId: '4326', - format: 'CSV', - downloadId: 'download-id', - jobId: 'job-id', + target: "portal", + datasetId: "abcdef0123456789abcdef0123456789_0", + spatialRefId: "4326", + format: "CSV", + downloadId: "download-id", + jobId: "job-id", exportCreated: 1000, eventEmitter: mockEventEmitter, pollingInterval: 10, authentication }); - expect(portalPoller.portalPollExportJobStatus as any).toHaveBeenCalledTimes(1) - expect((portalPoller.portalPollExportJobStatus as any).calls.first().args).toEqual([{ - downloadId: 'download-id', - datasetId: 'abcdef0123456789abcdef0123456789_0', - jobId: 'job-id', - format: 'CSV', - spatialRefId: '4326', - eventEmitter: mockEventEmitter, - pollingInterval: 10, - authentication, - exportCreated: 1000, - geometry: undefined, - where: undefined - }]) + expect( + portalPoller.portalPollExportJobStatus as any + ).toHaveBeenCalledTimes(1); + expect( + (portalPoller.portalPollExportJobStatus as any).calls.first().args + ).toEqual([ + { + downloadId: "download-id", + datasetId: "abcdef0123456789abcdef0123456789_0", + jobId: "job-id", + format: "CSV", + spatialRefId: "4326", + eventEmitter: mockEventEmitter, + pollingInterval: 10, + authentication, + exportCreated: 1000, + geometry: undefined, + where: undefined + } + ]); } catch (err) { expect(err).toBeUndefined(); } finally { done(); } }); -}); \ No newline at end of file +});