diff --git a/readme.md b/readme.md index 5efb6ed9e..59fb8c154 100644 --- a/readme.md +++ b/readme.md @@ -791,6 +791,15 @@ The maximum amount of request that should be triggered. [Retries on failure](#re For example, it can be helpful during development to avoid an infinite number of requests. +###### pagination.stackAllItems + +Type: `boolean`\ +Default: `true` + +Defines how the parameter `allItems` in [pagination.paginate](#pagination.paginate), [pagination.filter](#pagination.filter) and [pagination.shouldContinue](#pagination.shouldContinue) is managed. When set to `false`, the parameter `allItems` is always an empty array. + +This option can be helpful to save on memory usage when working with a large dataset. + ##### localAddress Type: `string` diff --git a/source/as-promise/types.ts b/source/as-promise/types.ts index 0ccc54250..07dd991e0 100644 --- a/source/as-promise/types.ts +++ b/source/as-promise/types.ts @@ -78,6 +78,7 @@ export interface PaginationOptions { shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean; countLimit?: number; requestLimit?: number; + stackAllItems?: boolean; }; } diff --git a/source/create.ts b/source/create.ts index ebeb7fd33..1eeb81427 100644 --- a/source/create.ts +++ b/source/create.ts @@ -205,6 +205,7 @@ const create = (defaults: InstanceDefaults): Got => { } const all: T[] = []; + let {countLimit} = pagination; let numberOfRequests = 0; while (numberOfRequests < pagination.requestLimit) { @@ -224,10 +225,13 @@ const create = (defaults: InstanceDefaults): Got => { yield item; - all.push(item as T); + if (pagination.stackAllItems) { + all.push(item as T); + } + current.push(item as T); - if (all.length === pagination.countLimit) { + if (--countLimit <= 0) { return; } } diff --git a/source/index.ts b/source/index.ts index 2df013727..eeb5efc75 100644 --- a/source/index.ts +++ b/source/index.ts @@ -109,7 +109,8 @@ const defaults: InstanceDefaults = { filter: () => true, shouldContinue: () => true, countLimit: Infinity, - requestLimit: 10000 + requestLimit: 10000, + stackAllItems: true } }, handlers: [defaultHandler], diff --git a/test/pagination.ts b/test/pagination.ts index 0e378a472..65dc7aab2 100644 --- a/test/pagination.ts +++ b/test/pagination.ts @@ -350,3 +350,59 @@ test('`requestLimit` works', withServer, async (t, server, got) => { t.deepEqual(results, [1]); }); + +test('`stackAllItems` set to true', withServer, async (t, server, got) => { + attachHandler(server, 3); + + let itemCount = 0; + const result = await got.paginate.all({ + pagination: { + stackAllItems: true, + filter: (_item, allItems, _currentItems) => { + t.is(allItems.length, itemCount); + + return true; + }, + shouldContinue: (_item, allItems, _currentItems) => { + t.is(allItems.length, itemCount); + + return true; + }, + paginate: (response, allItems, currentItems) => { + itemCount += 1; + t.is(allItems.length, itemCount); + + return got.defaults.options.pagination!.paginate(response, allItems, currentItems); + } + } + }); + + t.deepEqual(result, [1, 2, 3]); +}); + +test('`stackAllItems` set to false', withServer, async (t, server, got) => { + attachHandler(server, 3); + + const result = await got.paginate.all({ + pagination: { + stackAllItems: false, + filter: (_item, allItems, _currentItems) => { + t.is(allItems.length, 0); + + return true; + }, + shouldContinue: (_item, allItems, _currentItems) => { + t.is(allItems.length, 0); + + return true; + }, + paginate: (response, allItems, currentItems) => { + t.is(allItems.length, 0); + + return got.defaults.options.pagination!.paginate(response, allItems, currentItems); + } + } + }); + + t.deepEqual(result, [1, 2, 3]); +});