From bf360603430951fcfeb6007dbdd62a011bd19e35 Mon Sep 17 00:00:00 2001 From: Konstantin Myakshin Date: Tue, 11 Jun 2024 00:04:52 +0200 Subject: [PATCH] feat: Added retry capability for uploading files Signed-off-by: Konstantin Myakshin --- __tests__/utils/upload.spec.ts | 20 ++++++++++++++++++++ lib/uploader.ts | 8 ++++---- lib/utils/upload.ts | 15 ++++++++++++++- package-lock.json | 23 +++++++++++++++++++++++ package.json | 1 + 5 files changed, 62 insertions(+), 5 deletions(-) diff --git a/__tests__/utils/upload.spec.ts b/__tests__/utils/upload.spec.ts index 5fc6aeae..f3cbf9df 100644 --- a/__tests__/utils/upload.spec.ts +++ b/__tests__/utils/upload.spec.ts @@ -78,6 +78,10 @@ describe('Initialize chunks upload temporary workspace', () => { expect(axiosMock.request).toHaveBeenCalledWith({ method: 'MKCOL', url, + 'axios-retry': { + retries: 5, + retryDelay: expect.any(Function), + }, }) }) @@ -105,6 +109,10 @@ describe('Initialize chunks upload temporary workspace', () => { headers: { Destination: 'https://cloud.domain.com/remote.php/dav/files/test/image.jpg', }, + 'axios-retry': { + retries: 5, + retryDelay: expect.any(Function), + }, }) }) }) @@ -130,6 +138,10 @@ describe('Upload data', () => { headers: { 'Content-Type': 'application/octet-stream', }, + 'axios-retry': { + retries: 5, + retryDelay: expect.any(Function), + }, }) }) test('Upload async data stream', async () => { @@ -155,6 +167,10 @@ describe('Upload data', () => { headers: { 'Content-Type': 'application/octet-stream', }, + 'axios-retry': { + retries: 5, + retryDelay: expect.any(Function), + }, }) }) @@ -179,6 +195,10 @@ describe('Upload data', () => { Destination: url, 'Content-Type': 'application/octet-stream', }, + 'axios-retry': { + retries: 5, + retryDelay: expect.any(Function), + }, }) }) diff --git a/lib/uploader.ts b/lib/uploader.ts index 9bef99cd..b040f536 100644 --- a/lib/uploader.ts +++ b/lib/uploader.ts @@ -350,8 +350,9 @@ export class Uploader { * @param {string} destination the destination path relative to the root folder. e.g. /foo/bar.txt * @param {File|FileSystemFileEntry} fileHandle the file to upload * @param {string} root the root folder to upload to + * @param retries number of retries */ - upload(destination: string, fileHandle: File|FileSystemFileEntry, root?: string): PCancelable { + upload(destination: string, fileHandle: File|FileSystemFileEntry, root?: string, retries: number = 5): PCancelable { root = root || this.root const destinationPath = `${root.replace(/\/$/, '')}/${destination.replace(/^\//, '')}` @@ -387,7 +388,7 @@ export class Uploader { logger.debug('Initializing chunked upload', { file, upload }) // Let's initialize a chunk upload - const tempUrl = await initChunkWorkspace(encodedDestinationFile) + const tempUrl = await initChunkWorkspace(encodedDestinationFile, retries) const chunksQueue: Array> = [] // Generate chunks array @@ -411,6 +412,7 @@ export class Uploader { 'OC-Total-Length': file.size, 'Content-Type': 'application/octet-stream', }, + retries, ) // Update upload progress on chunk completion .then(() => { upload.uploaded = upload.uploaded + maxChunkSize }) @@ -424,8 +426,6 @@ export class Uploader { if (!isCancel(error)) { logger.error(`Chunk ${chunk + 1} ${bufferStart} - ${bufferEnd} uploading failed`, { error, upload }) - // TODO: support retrying ? - // https://github.com/nextcloud-libraries/nextcloud-upload/issues/5 upload.cancel() upload.status = UploadStatus.FAILED } diff --git a/lib/utils/upload.ts b/lib/utils/upload.ts index 007ee286..5bf9c845 100644 --- a/lib/utils/upload.ts +++ b/lib/utils/upload.ts @@ -2,6 +2,8 @@ import type { AxiosProgressEvent, AxiosResponse } from 'axios' import { generateRemoteUrl } from '@nextcloud/router' import { getCurrentUser } from '@nextcloud/auth' import axios from '@nextcloud/axios' +import axiosRetry from 'axios-retry'; +axiosRetry(axios, { retries: 0 }); type UploadData = Blob | (() => Promise) @@ -13,6 +15,7 @@ type UploadData = Blob | (() => Promise) * @param onUploadProgress the progress callback * @param destinationFile the final destination file (often used for chunked uploads) * @param headers additional headers + * @param retries number of retries */ export const uploadData = async function( url: string, @@ -21,6 +24,7 @@ export const uploadData = async function( onUploadProgress:(event: AxiosProgressEvent) => void = () => {}, destinationFile: string | undefined = undefined, headers: Record = {}, + retries: number = 5, ): Promise { let data: Blob @@ -49,6 +53,10 @@ export const uploadData = async function( signal, onUploadProgress, headers, + 'axios-retry': { + retries, + retryDelay: (retryCount, error) => axiosRetry.exponentialDelay(retryCount, error, 1000), + }, }) } @@ -70,8 +78,9 @@ export const getChunk = function(file: File, start: number, length: number): Pro /** * Create a temporary upload workspace to upload the chunks to * @param destinationFile The file name after finishing the chunked upload + * @param retries number of retries */ -export const initChunkWorkspace = async function(destinationFile: string | undefined = undefined): Promise { +export const initChunkWorkspace = async function(destinationFile: string | undefined = undefined, retries: number = 5): Promise { const chunksWorkspace = generateRemoteUrl(`dav/uploads/${getCurrentUser()?.uid}`) const hash = [...Array(16)].map(() => Math.floor(Math.random() * 16).toString(16)).join('') const tempWorkspace = `web-file-upload-${hash}` @@ -82,6 +91,10 @@ export const initChunkWorkspace = async function(destinationFile: string | undef method: 'MKCOL', url, headers, + 'axios-retry': { + retries, + retryDelay: (retryCount, error) => axiosRetry.exponentialDelay(retryCount, error, 1000), + }, }) return url diff --git a/package-lock.json b/package-lock.json index b4a774bb..589a612f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@nextcloud/paths": "^2.1.0", "@nextcloud/router": "^3.0.0", "axios": "^1.7.2", + "axios-retry": "^4.4.0", "crypto-browserify": "^3.12.0", "p-cancelable": "^4.0.1", "p-queue": "^8.0.0", @@ -3561,6 +3562,17 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/axios-retry": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-4.4.0.tgz", + "integrity": "sha512-yewTKjzl6jSgc+2M7FCJ3LxRGgL1iiXHcj+E6h6xie6H1mTHr7yqaUroWIvVXG1UKSPwGDXxV05YxtGvrD6Paw==", + "dependencies": { + "is-retry-allowed": "^2.2.0" + }, + "peerDependencies": { + "axios": "0.x || 1.x" + } + }, "node_modules/bail": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", @@ -7556,6 +7568,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-retry-allowed": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-shared-array-buffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", diff --git a/package.json b/package.json index 7ec8e155..4655a8b3 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "@nextcloud/paths": "^2.1.0", "@nextcloud/router": "^3.0.0", "axios": "^1.7.2", + "axios-retry": "^4.4.0", "crypto-browserify": "^3.12.0", "p-cancelable": "^4.0.1", "p-queue": "^8.0.0",