fix(@esri/hub-downloads): handle polling for exports of datasets that…

… do not use lastEditDate. affects: @esri/hub-downloads
Esri · Jan 7, 2021 · c135b2c · c135b2c
1 parent 163ad99
commit c135b2c
Show file tree

Hide file tree

Showing 4 changed files with 262 additions and 122 deletions.
diff --git a/packages/downloads/src/hub/hub-poll-download-metadata.ts b/packages/downloads/src/hub/hub-poll-download-metadata.ts
@@ -12,6 +12,7 @@ export interface IHubDownloadMetadataPollParameters
   downloadId: string;
   eventEmitter: EventEmitter;
   pollingInterval: number;
+  existingFileDate?: string;
 }
 
 class HubPoller implements IPoller {
@@ -23,11 +24,19 @@ class HubPoller implements IPoller {
   }
 
   activatePoll(params: IHubDownloadMetadataPollParameters) {
-    const { downloadId, eventEmitter, pollingInterval } = params;
+    const {
+      downloadId,
+      eventEmitter,
+      pollingInterval,
+      existingFileDate = new Date(0).toISOString()
+    } = params;
+
+    const existingFileTimestamp = new Date(existingFileDate).getTime();
+
     this.pollTimer = setInterval(() => {
       hubRequestDownloadMetadata(params)
         .then(metadata => {
-          if (isUpToDate(metadata)) {
+          if (isReady(metadata, existingFileTimestamp)) {
             eventEmitter.emit(`${downloadId}ExportComplete`, {
               detail: { metadata }
             });
@@ -65,8 +74,13 @@ export function hubPollDownloadMetadata(
   return poller;
 }
 
-function isUpToDate(metadata: any) {
-  return metadata && metadata.status === "ready";
+function isReady(metadata: any, preExportFileTimestamp: number) {
+  const { status, lastModified } = metadata;
+  const currentFileDate = new Date(lastModified).getTime();
+  return (
+    status === "ready" ||
+    (status === "ready_unknown" && currentFileDate > preExportFileTimestamp)
+  );
 }
 
 function exportDatasetFailed(metadata: any) {

diff --git a/packages/downloads/src/poll-download-metadata.ts b/packages/downloads/src/poll-download-metadata.ts
@@ -30,6 +30,8 @@ export interface IPollDownloadMetadataRequestParams {
   jobId?: string;
   /* Time-stamp for export start. Required for Portal downloads only. */
   exportCreated?: number;
+  /* ISO string of existing export file date. Used to track export progress when lastEditDate is unknown. */
+  existingFileDate?: string;
 }
 
 /**
@@ -54,7 +56,8 @@ export function pollDownloadMetadata(
     spatialRefId,
     geometry,
     where,
-    host
+    host,
+    existingFileDate
   } = params;
 
   if (target === "portal") {
@@ -82,6 +85,7 @@ export function pollDownloadMetadata(
     pollingInterval,
     spatialRefId,
     geometry,
-    where
+    where,
+    existingFileDate
   });
 }
diff --git a/packages/downloads/test/hub/hub-poll-download-metadata.test.ts b/packages/downloads/test/hub/hub-poll-download-metadata.test.ts
@@ -6,64 +6,93 @@ function delay(milliseconds: number) {
   return new Promise(resolve => setTimeout(resolve, milliseconds));
 }
 
-const fixtures = {
-  exportFailed: {
-    data: [
-      {
-        id: "dd4580c810204019a7b8eb3e0b329dd6_0",
-        type: "downloads",
-        attributes: {
-          spatialRefId: "4326",
-          format: "CSV",
-          status: "error_creating",
-          featureSet: "full",
-          errors: [{ message: "Some error" }],
-          source: {
-            type: "Feature Service",
-            url:
-              "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json",
-            supportsExtract: true,
-            lastEditDate: "2020-06-18T01:17:31.492Z",
-            spatialRefId: "4326"
-          }
-        },
-        links: {
-          content:
-            "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv"
+const exportFailed = {
+  data: [
+    {
+      id: "dd4580c810204019a7b8eb3e0b329dd6_0",
+      type: "downloads",
+      attributes: {
+        spatialRefId: "4326",
+        format: "CSV",
+        status: "error_creating",
+        featureSet: "full",
+        errors: [{ message: "Some error" }],
+        source: {
+          type: "Feature Service",
+          url:
+            "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json",
+          supportsExtract: true,
+          lastEditDate: "2020-06-18T01:17:31.492Z",
+          spatialRefId: "4326"
         }
+      },
+      links: {
+        content:
+          "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv"
       }
-    ]
-  },
-  exportSucceeded: {
-    data: [
-      {
-        id: "dd4580c810204019a7b8eb3e0b329dd6_0",
-        type: "downloads",
-        attributes: {
-          spatialRefId: "4326",
-          format: "CSV",
-          contentLength: 1391454,
-          lastModified: "2020-06-17T13:04:28.000Z",
-          contentLastModified: "2020-06-17T01:16:01.933Z",
-          cacheTime: 13121,
-          status: "ready",
-          featureSet: "full",
-          source: {
-            type: "Feature Service",
-            url:
-              "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json",
-            supportsExtract: true,
-            lastEditDate: "2020-06-18T01:15:31.492Z",
-            spatialRefId: "4326"
-          }
-        },
-        links: {
-          content:
-            "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv"
+    }
+  ]
+};
+
+const exportReady = {
+  data: [
+    {
+      id: "dd4580c810204019a7b8eb3e0b329dd6_0",
+      type: "downloads",
+      attributes: {
+        spatialRefId: "4326",
+        format: "CSV",
+        contentLength: 1391454,
+        lastModified: "2020-06-17T13:04:28.000Z",
+        contentLastModified: "2020-06-17T01:16:01.933Z",
+        cacheTime: 13121,
+        status: "ready",
+        featureSet: "full",
+        source: {
+          type: "Feature Service",
+          url:
+            "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json",
+          supportsExtract: true,
+          lastEditDate: "2020-06-18T01:15:31.492Z",
+          spatialRefId: "4326"
         }
+      },
+      links: {
+        content:
+          "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv"
       }
-    ]
-  }
+    }
+  ]
+};
+
+const exportReadyUnknown = {
+  data: [
+    {
+      id: "dd4580c810204019a7b8eb3e0b329dd6_0",
+      type: "downloads",
+      attributes: {
+        spatialRefId: "4326",
+        format: "CSV",
+        contentLength: 1391454,
+        lastModified: "2020-06-17T13:04:28.000Z",
+        contentLastModified: "2020-06-17T13:04:28.000Z",
+        cacheTime: 13121,
+        status: "ready_unknown",
+        featureSet: "full",
+        source: {
+          type: "Feature Service",
+          url:
+            "https://services7.arcgis.com/mOBPykOjAyBO2ZKk/arcgis/rest/services/RKI_COVID19/FeatureServer/0?f=json",
+          supportsExtract: true,
+          spatialRefId: "4326"
+        }
+      },
+      links: {
+        content:
+          "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv"
+      }
+    }
+  ]
 };
 
 describe("hubPollDownloadMetadata", () => {
@@ -113,7 +142,7 @@ describe("hubPollDownloadMetadata", () => {
         "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv",
         {
           status: 200,
-          body: fixtures.exportFailed
+          body: exportFailed
         }
       );
       const mockEventEmitter = new EventEmitter();
@@ -155,13 +184,13 @@ describe("hubPollDownloadMetadata", () => {
     }
   });
 
-  it("handle successful export", async done => {
+  it("handle ready export", async done => {
     try {
       fetchMock.mock(
         "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv",
         {
           status: 200,
-          body: fixtures.exportSucceeded
+          body: exportReady
         }
       );
       const mockEventEmitter = new EventEmitter();
@@ -203,6 +232,86 @@ describe("hubPollDownloadMetadata", () => {
     }
   });
 
+  it("handle ready_unknown export (export complete)", async done => {
+    try {
+      fetchMock.mock(
+        "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv",
+        {
+          status: 200,
+          body: exportReadyUnknown
+        }
+      );
+      const mockEventEmitter = new EventEmitter();
+      spyOn(mockEventEmitter, "emit");
+      const poller = hubPollDownloadMetadata({
+        host: "http://hub.com",
+        datasetId: "abcdef0123456789abcdef0123456789_0",
+        downloadId: "test-id",
+        spatialRefId: "4326",
+        format: "CSV",
+        eventEmitter: mockEventEmitter,
+        pollingInterval: 10,
+        existingFileDate: "2020-06-15T13:04:28.000Z"
+      });
+      await delay(50);
+      expect(mockEventEmitter.emit as any).toHaveBeenCalledTimes(1);
+      const [
+        topic,
+        customEvent
+      ] = (mockEventEmitter.emit as any).calls.first().args;
+      expect(topic).toEqual("test-idExportComplete");
+      expect(customEvent.detail.metadata).toEqual({
+        downloadId:
+          "abcdef0123456789abcdef0123456789_0:CSV:4326:undefined:undefined",
+        contentLastModified: "2020-06-17T13:04:28.000Z",
+        lastEditDate: undefined,
+        lastModified: "2020-06-17T13:04:28.000Z",
+        status: "ready_unknown",
+        errors: [],
+        downloadUrl:
+          "https://dev-hub-indexer.s3.amazonaws.com/files/dd4580c810204019a7b8eb3e0b329dd6/0/full/4326/dd4580c810204019a7b8eb3e0b329dd6_0_full_4326.csv",
+        contentLength: 1391454,
+        cacheTime: 13121
+      });
+      expect(poller.pollTimer).toEqual(null);
+    } catch (err) {
+      expect(err).toEqual(undefined);
+    } finally {
+      done();
+    }
+  });
+
+  it("handle ready_unknown export (export queued)", async done => {
+    try {
+      fetchMock.mock(
+        "http://hub.com/api/v3/datasets/abcdef0123456789abcdef0123456789_0/downloads?spatialRefId=4326&formats=csv",
+        {
+          status: 200,
+          body: exportReadyUnknown
+        }
+      );
+      const mockEventEmitter = new EventEmitter();
+      spyOn(mockEventEmitter, "emit");
+      const poller = hubPollDownloadMetadata({
+        host: "http://hub.com",
+        datasetId: "abcdef0123456789abcdef0123456789_0",
+        downloadId: "test-id",
+        spatialRefId: "4326",
+        format: "CSV",
+        eventEmitter: mockEventEmitter,
+        pollingInterval: 10,
+        existingFileDate: "2020-06-17T13:04:28.000Z"
+      });
+      await delay(50);
+      expect(mockEventEmitter.emit as any).toHaveBeenCalledTimes(0);
+      expect(poller.pollTimer !== null).toEqual(true);
+    } catch (err) {
+      expect(err).toEqual(undefined);
+    } finally {
+      done();
+    }
+  }, 16000);
+
   it("handle multiple polls", async done => {
     try {
       fetchMock.mock(