Skip to content

Commit

Permalink
feat: add function to allow user to set destination in transfer manag…
Browse files Browse the repository at this point in the history
…er (#2497)

* feat: add function to allow user to set destination in transfer manager

* lint
  • Loading branch information
ddelgrosso1 authored Jul 15, 2024
1 parent ded96e3 commit dc1e488
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 3 deletions.
12 changes: 9 additions & 3 deletions src/transfer-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ const GCCL_GCS_CMD_FEATURE = {

export interface UploadManyFilesOptions {
concurrencyLimit?: number;
customDestinationBuilder?(
path: string,
options: UploadManyFilesOptions
): string;
skipIfExists?: boolean;
prefix?: string;
passthroughOptions?: Omit<UploadOptions, 'destination'>;
Expand Down Expand Up @@ -411,6 +415,8 @@ export class TransferManager {
* @typedef {object} UploadManyFilesOptions
* @property {number} [concurrencyLimit] The number of concurrently executing promises
* to use when uploading the files.
* @property {Function} [customDestinationBuilder] A fuction that will take the current path of a local file
* and return a string representing a custom path to be used to upload the file to GCS.
* @property {boolean} [skipIfExists] Do not upload the file if it already exists in
* the bucket. This will set the precondition ifGenerationMatch = 0.
* @property {string} [prefix] A prefix to append to all of the uploaded files.
Expand Down Expand Up @@ -490,9 +496,9 @@ export class TransferManager {
[GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.UPLOAD_MANY,
};

passThroughOptionsCopy.destination = filePath
.split(path.sep)
.join(path.posix.sep);
passThroughOptionsCopy.destination = options.customDestinationBuilder
? options.customDestinationBuilder(filePath, options)
: filePath.split(path.sep).join(path.posix.sep);
if (options.prefix) {
passThroughOptionsCopy.destination = path.posix.join(
...options.prefix.split(path.sep),
Expand Down
19 changes: 19 additions & 0 deletions test/transfer-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
MultiPartUploadError,
MultiPartUploadHelper,
UploadOptions,
UploadManyFilesOptions,

Check warning on line 29 in test/transfer-manager.ts

View workflow job for this annotation

GitHub Actions / lint

'UploadManyFilesOptions' is defined but never used
TransferManager,
Storage,
DownloadResponse,
Expand Down Expand Up @@ -173,6 +174,24 @@ describe('Transfer Manager', () => {

await transferManager.uploadManyFiles([filePath]);
});

it('allows the user to apply a custom destination transformation when supplied a custom function', async () => {
const paths = ['a', 'b', 'foo/bar', 'bar.txt'];
const expected = ['foo/a', 'b/bar', 'foo/foo/bar', 'bar.txt/bar'];
sandbox.stub(bucket, 'upload').callsFake((path, options) => {
const uploadOpts = options as UploadOptions;
assert(expected.includes(uploadOpts.destination as string));
});

let callCount = 0;
const transformationFunc = (path: string) => {
assert.strictEqual(path, paths[callCount]);
return expected[callCount++];
};
await transferManager.uploadManyFiles(paths, {
customDestinationBuilder: transformationFunc,
});
});
});

describe('downloadManyFiles', () => {
Expand Down

0 comments on commit dc1e488

Please sign in to comment.