diff --git a/src/core/datastore/cachestore.js b/src/core/datastore/cachestore.js index 27c8b6da8..465c9042a 100644 --- a/src/core/datastore/cachestore.js +++ b/src/core/datastore/cachestore.js @@ -24,6 +24,11 @@ export class CacheStore extends NetworkStore { */ this.ttl = options.ttl || undefined; this.syncManager = syncManagerProvider.getSyncManager(); + + /** + * @type {boolean} + */ + this.useDeltaSet = options.useDeltaSet === true; } /** @@ -110,7 +115,7 @@ export class CacheStore extends NetworkStore { * @return {Promise.} Promise */ pull(query, options = {}) { - options = assign({ useDeltaFetch: this.useDeltaFetch }, options); + options = assign({ useDeltaSet: this.useDeltaSet }, options); return this.syncManager.getSyncItemCountByEntityQuery(this.collection, query) .then((count) => { if (count > 0) { @@ -134,7 +139,7 @@ export class CacheStore extends NetworkStore { * @return {Promise.<{push: [], pull: number}>} Promise */ sync(query, options) { - options = assign({ useDeltaFetch: this.useDeltaFetch }, options); + options = assign({ useDeltaSet: this.useDeltaSet }, options); const result = {}; return this.push(query, options) .then((pushResult) => { diff --git a/src/core/datastore/cachestore.spec.js b/src/core/datastore/cachestore.spec.js index 6083427ea..34841b3ea 100644 --- a/src/core/datastore/cachestore.spec.js +++ b/src/core/datastore/cachestore.spec.js @@ -6,7 +6,7 @@ import { SyncOperation } from './sync'; import { init } from '../kinvey'; import { Query } from '../query'; import { Aggregation } from '../aggregation'; -import { KinveyError, NotFoundError, ServerError } from '../errors'; +import { KinveyError, NotFoundError, ServerError, BadRequestError } from '../errors'; import { randomString } from '../utils'; import { NetworkRack } from '../request'; import { NodeHttpMiddleware } from '../../node/http'; @@ -188,6 +188,324 @@ describe('CacheStore', () => { }); }); + describe('Delta Set', () => { + it('should find the entities', (done) => { + const entity1 = { _id: randomString() }; + const entity2 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true }); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull() + .then(() => { + const changedEntity2 = Object.assign({}, entity2, { title: 'test' }); + nock(client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query({ since: lastRequestDate.toISOString() }) + .reply(200, { changed: [changedEntity2], deleted: [{ _id: entity1._id }] }, { + 'X-Kinvey-Request-Start': new Date().toISOString() + }); + + store.find() + .subscribe(onNextSpy, done, () => { + try { + expect(onNextSpy.calls.length).toEqual(2); + expect(onNextSpy.calls[0].arguments).toEqual([[entity1, entity2]]); + expect(onNextSpy.calls[1].arguments).toEqual([[changedEntity2]]); + done(); + } catch (error) { + done(error); + } + }); + }) + .catch(done); + }); + + it('should find the entities that match the query', (done) => { + const entity1 = { _id: randomString(), title: 'Test' }; + const entity2 = { _id: randomString(), title: 'Test' }; + const store = new CacheStore(collection, null, { useDeltaSet: true }); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + const query = new Query().equalTo('title', 'Test'); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .query(query.toQueryString()) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull(query) + .then(() => { + const changedEntity2 = Object.assign({}, entity2, { author: 'Kinvey' }); + nock(client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query(Object.assign({ since: lastRequestDate.toISOString() }, query.toQueryString())) + .reply(200, { changed: [changedEntity2], deleted: [{ _id: entity1._id }] }, { + 'X-Kinvey-Request-Start': new Date().toISOString() + }); + + store.find(query) + .subscribe(onNextSpy, done, () => { + try { + expect(onNextSpy.calls.length).toEqual(2); + expect(onNextSpy.calls[0].arguments).toEqual([[entity1, entity2]]); + expect(onNextSpy.calls[1].arguments).toEqual([[changedEntity2]]); + done(); + } catch (error) { + done(error); + } + }); + }) + .catch(done); + }); + + it('should not send a delta set request if skip is used', (done) => { + const entity1 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true }); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + const query = new Query(); + query.skip = 1; + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .query(query.toQueryString()) + .reply(200, [entity1], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull(query) + .then(() => { + nock(client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .query(query.toQueryString()) + .reply(200, [entity1], { + 'X-Kinvey-Request-Start': new Date().toISOString() + }); + + store.find(query) + .subscribe(onNextSpy, done, () => { + try { + expect(onNextSpy.calls.length).toEqual(2); + done(); + } catch (error) { + done(error); + } + }); + }) + .catch(done); + }); + + it('should not send a delta set request if limit is used', (done) => { + const entity1 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true }); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + const query = new Query(); + query.limit = 1; + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .query(query.toQueryString()) + .reply(200, [entity1], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull(query) + .then(() => { + nock(client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .query(query.toQueryString()) + .reply(200, [entity1], { + 'X-Kinvey-Request-Start': new Date().toISOString() + }); + + store.find(query) + .subscribe(onNextSpy, done, () => { + try { + expect(onNextSpy.calls.length).toEqual(2); + done(); + } catch (error) { + done(error); + } + }); + }) + .catch(done); + }); + + it('should work with a tagged datastore', (done) => { + const entity1 = { _id: randomString() }; + const entity2 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true, tag: randomString() }); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull() + .then(() => { + const changedEntity2 = Object.assign({}, entity2, { title: 'test' }); + nock(client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query({ since: lastRequestDate.toISOString() }) + .reply(200, { changed: [changedEntity2], deleted: [{ _id: entity1._id }] }, { + 'X-Kinvey-Request-Start': new Date().toISOString() + }); + + store.find() + .subscribe(onNextSpy, done, () => { + try { + expect(onNextSpy.calls.length).toEqual(2); + expect(onNextSpy.calls[0].arguments).toEqual([[entity1, entity2]]); + expect(onNextSpy.calls[1].arguments).toEqual([[changedEntity2]]); + done(); + } catch (error) { + done(error); + } + }); + }) + .catch(done); + }); + + it('should send regular GET request with outdated lastRequest', function (done){ + const entity1 = { _id: randomString() }; + const entity2 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true}); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + lastRequestDate.setDate(new Date().getDate()-31); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull() + .then(()=>{ + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query({ since: lastRequestDate.toISOString() }) + .reply(400, {debug: "The 'since' timestamp must be within the past 1 days.", + description: "The value specified for one of the request parameters is out of range", + error: "ParameterValueOutOfRange"}); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + store.find() + .subscribe(onNextSpy, done, ()=>{ + try{ + expect(onNextSpy.calls.length).toEqual(2); + expect(onNextSpy.calls[0].arguments).toEqual([[entity1, entity2]]); + expect(onNextSpy.calls[1].arguments).toEqual([[entity1, entity2]]); + done(); + } catch (error) { + done(error); + } + }) + }) + .catch(done); + }); + + it('should send regular GET request when configuration is missing on the backend', function (done){ + const entity1 = { _id: randomString() }; + const entity2 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true}); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + const firstNock = nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull() + .then(()=>{ + firstNock.done(); + const secondNock = nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query({ since: lastRequestDate.toISOString() }) + .reply(403, { + "error": "MissingConfiguration", + "description": "This feature is not properly configured for this app backend. Please configure it through the console first, or contact support for more information.", + "debug": "This collection has not been configured for Delta Set access." + }); + + const thirdNock = nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + store.find() + .subscribe(onNextSpy, done, ()=>{ + try{ + secondNock.done(); + thirdNock.done(); + expect(onNextSpy.calls.length).toEqual(2); + expect(onNextSpy.calls[0].arguments).toEqual([[entity1, entity2]]); + expect(onNextSpy.calls[1].arguments).toEqual([[entity1, entity2]]); + done(); + } catch (error) { + done(error); + } + }) + }) + .catch(done); + }); + + it('should return error if more than 10000 items are changed', function (done){ + const entity1 = { _id: randomString() }; + const entity2 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true}); + const onNextSpy = expect.createSpy(); + const lastRequestDate = new Date(); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + store.pull() + .then(()=>{ + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query({ since: lastRequestDate.toISOString() }) + .reply(400, { + error: "BadRequest", + description: "Unable to understand request", + debug: "ResultSetSizeExceeded" + }); + store.find() + .subscribe(null, (error)=>{ + try{ + expect(error).toBeA(BadRequestError); + expect(error.debug).toEqual('ResultSetSizeExceeded') + done(); + } catch (e) { + done(e); + } + }) + }) + .catch(done); + }); + }); + it('should remove entities that no longer exist on the backend from the cache', (done) => { const entity1 = { _id: randomString() }; const entity2 = { _id: randomString() }; @@ -1116,6 +1434,64 @@ describe('CacheStore', () => { expect(entities).toEqual([entity1, entity2]); }); }); + + it('should perform a delta set request', () => { + const entity1 = { _id: randomString() }; + const entity2 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true }); + const lastRequestDate = new Date(); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + return store.pull() + .then(() => { + const changedEntity2 = Object.assign({}, entity2, { title: 'test' }); + nock(client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query({ since: lastRequestDate.toISOString() }) + .reply(200, { changed: [changedEntity2], deleted: [{ _id: entity1._id }] }, { + 'X-Kinvey-Request-Start': new Date().toISOString() + }); + + store.pull() + .then((count) => { + expect(count).toEqual(1); + }); + }); + }); + + it('should perform a delta set request with a tagged datastore', () => { + const entity1 = { _id: randomString() }; + const entity2 = { _id: randomString() }; + const store = new CacheStore(collection, null, { useDeltaSet: true, tag: randomString() }); + const lastRequestDate = new Date(); + + nock(store.client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}`) + .reply(200, [entity1, entity2], { + 'X-Kinvey-Request-Start': lastRequestDate.toISOString() + }); + + return store.pull() + .then(() => { + const changedEntity2 = Object.assign({}, entity2, { title: 'test' }); + nock(client.apiHostname) + .get(`/appdata/${store.client.appKey}/${collection}/_deltaset`) + .query({ since: lastRequestDate.toISOString() }) + .reply(200, { changed: [changedEntity2], deleted: [{ _id: entity1._id }] }, { + 'X-Kinvey-Request-Start': new Date().toISOString() + }); + + store.pull() + .then((count) => { + expect(count).toEqual(1); + }); + }); + }); }); describe('sync()', () => { diff --git a/src/core/datastore/deltaset.js b/src/core/datastore/deltaset.js new file mode 100644 index 000000000..b4bd690f9 --- /dev/null +++ b/src/core/datastore/deltaset.js @@ -0,0 +1,50 @@ +import { format } from 'url'; +import { KinveyRequest, RequestMethod, AuthType } from '../request'; +import { Client } from '../client'; +import { + InvalidCachedQuery, + ParameterValueOutOfRangeError, + ResultSetSizeExceededError, + MissingConfigurationError +} from '../errors'; +import { buildCollectionUrl } from './utils'; +import { getCachedQuery } from './querycache'; + +export function deltaSet(collectionName, query, options) { + return getCachedQuery(collectionName, query) + .then((cachedQuery) => { + if (!cachedQuery || !cachedQuery.lastRequest) { + throw new InvalidCachedQuery(); + } + + const client = Client.sharedInstance(); + const request = new KinveyRequest({ + authType: AuthType.Default, + method: RequestMethod.GET, + url: format({ + protocol: client.apiProtocol, + host: client.apiHost, + pathname: buildCollectionUrl(collectionName, null, '_deltaset'), + query: { since: cachedQuery.lastRequest } + }), + query, + timeout: options.timeout, + followRedirect: options.followRedirect, + cache: options.cache, + properties: options.properties, + skipBL: options.skipBL, + trace: options.trace, + client + }); + return request.execute() + .catch((error) => { + if (error instanceof ParameterValueOutOfRangeError + || error instanceof ResultSetSizeExceededError + || error instanceof MissingConfigurationError) { + throw new InvalidCachedQuery(); + } + + throw error; + }); + }); +} diff --git a/src/core/datastore/networkstore.js b/src/core/datastore/networkstore.js index b59a3bcc5..5ffe4ca1f 100644 --- a/src/core/datastore/networkstore.js +++ b/src/core/datastore/networkstore.js @@ -37,11 +37,6 @@ export class NetworkStore { * @type {Client} */ this.client = options.client; - - /** - * @type {boolean} - */ - this.useDeltaFetch = options.useDeltaFetch === true; } /** @@ -93,7 +88,7 @@ export class NetworkStore { * @param {Properties} [options.properties] Custom properties to send with * the request. * @param {Number} [options.timeout] Timeout for the request. - * @param {Boolean} [options.useDeltaFetch] Turn on or off the use of delta fetch. + * @param {Boolean} [options.useDeltaSet] Turn on or off the use of delta fetch. * @return {Observable} Observable. */ find(query, options = {}) { @@ -102,7 +97,7 @@ export class NetworkStore { return wrapInObservable(errPromise); } - options = assign({ useDeltaFetch: this.useDeltaFetch }, options); + options = assign({ useDeltaSet: this.useDeltaSet }, options); const operation = this._buildOperationObject(OperationType.Read, query); const opPromise = this._executeOperation(operation, options); return this._ensureObservable(opPromise); @@ -116,7 +111,7 @@ export class NetworkStore { * @param {Properties} [options.properties] Custom properties to send with * the request. * @param {Number} [options.timeout] Timeout for the request. - * @param {Boolean} [options.useDeltaFetch] Turn on or off the use of delta fetch. + * @param {Boolean} [options.useDeltaSet] Turn on or off the use of delta fetch. * @return {Observable} Observable. */ findById(id, options = {}) { diff --git a/src/core/datastore/persisters/sql-key-value-store-persister.js b/src/core/datastore/persisters/sql-key-value-store-persister.js index c461cce15..bab1f73e9 100644 --- a/src/core/datastore/persisters/sql-key-value-store-persister.js +++ b/src/core/datastore/persisters/sql-key-value-store-persister.js @@ -18,7 +18,13 @@ export class SqlKeyValueStorePersister extends KeyValueStorePersister { const query = 'SELECT name AS value FROM #{collection} WHERE type = ?'; return this._sqlModule.openTransaction(sqliteCollectionsMaster, query, ['table'], false) .then((response) => { - return response.filter(table => (/^[a-zA-Z0-9-]{1,128}/).test(table)); + return response.filter((table) => { + if (table === '_QueryCache' || table === '__testSupport__') { + return true; + } + + return /^[a-zA-Z0-9-]{1,128}/.test(table) + }); }); } diff --git a/src/core/datastore/processors/cache-offline-data-processor.js b/src/core/datastore/processors/cache-offline-data-processor.js index 7070473e9..a4b313dbb 100644 --- a/src/core/datastore/processors/cache-offline-data-processor.js +++ b/src/core/datastore/processors/cache-offline-data-processor.js @@ -2,12 +2,14 @@ import { Promise } from 'es6-promise'; import clone from 'lodash/clone'; import { Query } from '../../query'; -import { NotFoundError } from '../../errors'; +import { NotFoundError, InvalidCachedQuery } from '../../errors'; import { OfflineDataProcessor } from './offline-data-processor'; import { ensureArray } from '../../utils'; import { wrapInObservable } from '../../observable'; import { isLocalEntity, isNotEmpty, isEmpty, getEntitiesPendingPushError } from '../utils'; +import { deltaSet } from '../deltaset'; +import { getCachedQuery, updateCachedQuery, deleteCachedQuery } from '../querycache'; // imported for type info // import { NetworkRepository } from '../repositories'; @@ -76,8 +78,10 @@ export class CacheOfflineDataProcessor extends OfflineDataProcessor { }); } - _processRead(collection, query, options) { + _processRead(collection, query, options = {}) { let offlineEntities; + let { useDeltaSet } = options; + return wrapInObservable((observer) => { return super._processRead(collection, query, options) .then((entities) => { @@ -85,10 +89,65 @@ export class CacheOfflineDataProcessor extends OfflineDataProcessor { observer.next(offlineEntities); return this._ensureCountBeforeRead(collection, 'fetch the entities', query); }) - .then(() => this._networkRepository.read(collection, query, options)) - .then((networkEntities) => { - observer.next(networkEntities); - return this._replaceOfflineEntities(collection, offlineEntities, networkEntities); + .then(() => { + if (useDeltaSet) { + return deltaSet(collection, query, options) + .catch((error) => { + if (error instanceof InvalidCachedQuery) { + useDeltaSet = false; + return getCachedQuery(collection, query) + .then((cachedQuery) => deleteCachedQuery(cachedQuery)) + .catch((error) => { + if (error instanceof NotFoundError) { + return null; + } + + throw error; + }) + .then(() => this._networkRepository.read(collection, query, Object.assign(options, { dataOnly: false }))); + } + + throw error; + }); + } + + return this._networkRepository.read(collection, query, Object.assign(options, { dataOnly: false })); + }) + .then((response) => { + return getCachedQuery(collection, query) + .then((cachedQuery) => { + if (cachedQuery && response.headers) { + cachedQuery.lastRequest = response.headers.requestStart; + return updateCachedQuery(cachedQuery); + } + + return null; + }) + .then(() => response.data ? response.data : response); + }) + .then((data) => { + if (useDeltaSet) { + const promises = []; + + if (data.deleted.length > 0) { + const deleteQuery = new Query(); + deleteQuery.containsAll('_id', data.deleted.map((entity) => entity._id)); + promises.push(this._deleteEntitiesOffline(collection, deleteQuery, data.deleted)); + } + + if (data.changed.length > 0) { + promises.push(this._replaceOfflineEntities(collection, data.changed, data.changed)); + } + + return Promise.all(promises); + } + + return this._replaceOfflineEntities(collection, offlineEntities, data); + }) + .then(() => super._processRead(collection, query, options)) + .then((entities) => { + observer.next(entities); + return entities; }); }); } diff --git a/src/core/datastore/processors/offline-data-processor.js b/src/core/datastore/processors/offline-data-processor.js index 4b8af3dff..1ae16e080 100644 --- a/src/core/datastore/processors/offline-data-processor.js +++ b/src/core/datastore/processors/offline-data-processor.js @@ -6,6 +6,7 @@ import { repositoryProvider } from '../repositories'; import { DataProcessor } from './data-processor'; import { generateEntityId, isEmpty } from '../utils'; import { ensureArray, isDefined } from '../../utils'; +import { clearQueryCache } from '../querycache'; // imported for typings // import { SyncManager } from '../sync'; @@ -64,6 +65,7 @@ export class OfflineDataProcessor extends DataProcessor { _processClear(collection, query, options) { return this._syncManager.clearSync(collection, query) + .then(() => clearQueryCache(collection)) .then(() => this._getRepository()) .then(repo => repo.delete(collection, query, options)); } diff --git a/src/core/datastore/querycache.js b/src/core/datastore/querycache.js new file mode 100644 index 000000000..1f0e7f9e4 --- /dev/null +++ b/src/core/datastore/querycache.js @@ -0,0 +1,93 @@ +import { Promise } from 'es6-promise'; +import isNumber from 'lodash/isNumber'; +import isEmpty from 'lodash/isEmpty'; +import { Query } from '../query'; +import { repositoryProvider } from './repositories'; +import { generateEntityId } from './utils'; + +export const queryCacheCollectionName = '_QueryCache'; + +function serializeQuery(query) { + if (query && ((isNumber(query.skip) && query.skip > 0) || isNumber(query.limit))) { + return null; + } + + const queryObject = query ? query.toQueryString() : {}; + return queryObject && !isEmpty(queryObject) ? JSON.stringify(queryObject) : ''; +} + +export function getCachedQuery(collectionName, query) { + const serializedQuery = serializeQuery(query); + + if (!serializedQuery && serializedQuery !== '') { + return Promise.resolve(null); + } + + return repositoryProvider.getOfflineRepository() + .then((offlineRepo) => { + const queryCacheQuery = new Query() + .equalTo('collectionName', collectionName) + .and() + .equalTo('query', serializedQuery); + return offlineRepo.read(queryCacheCollectionName, queryCacheQuery) + .then((cachedQueries = []) => { + if (cachedQueries.length > 0) { + return cachedQueries[0]; + } + + return { + _id: generateEntityId(), + collectionName: collectionName, + query: serializedQuery + }; + }); + }); +} + +export function createCachedQuery(collectionName, query) { + const serializedQuery = serializeQuery(query); + + if (!serializedQuery) { + return Promise.resolve(null); + } + + return repositoryProvider.getOfflineRepository() + .then((offlineRepo) => { + const cachedQuery = { + _id: generateEntityId(), + collectionName: collectionName, + query: serializedQuery + }; + return offlineRepo.create(queryCacheCollectionName, cachedQuery); + }); +} + +export function updateCachedQuery(cachedQuery) { + if (!cachedQuery) { + return Promise.resolve(null); + } + + return repositoryProvider.getOfflineRepository() + .then((offlineRepo) => { + return offlineRepo.update(queryCacheCollectionName, cachedQuery); + }); +} + +export function deleteCachedQuery(cachedQuery) { + if (!cachedQuery) { + return Promise.resolve(null); + } + + return repositoryProvider.getOfflineRepository() + .then((offlineRepo) => { + return offlineRepo.deleteById(queryCacheCollectionName, cachedQuery._id); + }); +} + +export function clearQueryCache(collectionName) { + return repositoryProvider.getOfflineRepository() + .then((offlineRepo) => { + const query = new Query().equalTo('collectionName', collectionName); + return offlineRepo.delete(queryCacheCollectionName, query); + }); +} diff --git a/src/core/datastore/repositories/network-repository.js b/src/core/datastore/repositories/network-repository.js index 278df9b45..8b551d3f1 100644 --- a/src/core/datastore/repositories/network-repository.js +++ b/src/core/datastore/repositories/network-repository.js @@ -1,8 +1,7 @@ import { Promise } from 'es6-promise'; -import { KinveyRequest, RequestMethod, DeltaFetchRequest } from '../../request'; +import { KinveyRequest, RequestMethod } from '../../request'; import { Aggregation } from '../../aggregation'; - import { Repository } from './repository'; import { ensureArray } from '../../utils'; import { buildCollectionUrl } from './utils'; @@ -25,7 +24,7 @@ import { buildCollectionUrl } from './utils'; export class NetworkRepository extends Repository { read(collection, query, options = {}) { const requestConfig = this._buildRequestConfig(collection, RequestMethod.GET, null, query, null, null, options); - return this._makeHttpRequest(requestConfig, options.useDeltaFetch); + return this._makeHttpRequest(requestConfig, options.dataOnly); } readById(collection, entityId, options) { @@ -55,8 +54,14 @@ export class NetworkRepository extends Repository { count(collection, query, options) { const requestConfig = this._buildRequestConfig(collection, RequestMethod.GET, null, query, null, '_count', null, options); - return this._makeHttpRequest(requestConfig) - .then(response => response.count); + return this._makeHttpRequest(requestConfig, options.dataOnly) + .then((response) => { + if (options.dataOnly === false) { + return response; + } + + return response.count; + }); } group(collection, aggregationQuery, options) { @@ -76,12 +81,8 @@ export class NetworkRepository extends Repository { .then(res => (isSingle ? res && res[0] : res)); } - _makeHttpRequest(requestConfig, deltaFetch) { - if (deltaFetch) { - const request = new DeltaFetchRequest(requestConfig); - return request.execute().then(r => r.data); - } - return KinveyRequest.execute(requestConfig); + _makeHttpRequest(requestConfig, dataOnly) { + return KinveyRequest.execute(requestConfig, null, dataOnly); } /** diff --git a/src/core/datastore/sync/sync-manager.js b/src/core/datastore/sync/sync-manager.js index b5771e5a9..a2412914f 100644 --- a/src/core/datastore/sync/sync-manager.js +++ b/src/core/datastore/sync/sync-manager.js @@ -2,23 +2,16 @@ import { Promise } from 'es6-promise'; import clone from 'lodash/clone'; import { Log } from '../../log'; -import { KinveyError, NotFoundError, SyncError } from '../../errors'; - +import { KinveyError, NotFoundError, SyncError, InvalidCachedQuery } from '../../errors'; import { getPlatformConfig } from '../../platform-configs'; import { SyncOperation } from './sync-operation'; import { maxEntityLimit, defaultPullSortField } from './utils'; import { isEmpty } from '../utils'; import { repositoryProvider } from '../repositories'; import { Query } from '../../query'; -import { - ensureArray, - isNonemptyString, - forEachAsync, - splitQueryIntoPages -} from '../../utils'; - -// imported for typings -// import { SyncStateManager } from './sync-state-manager'; +import { ensureArray, isNonemptyString, forEachAsync, splitQueryIntoPages } from '../../utils'; +import { deltaSet } from '../deltaset'; +import { getCachedQuery, updateCachedQuery, deleteCachedQuery } from '../querycache'; const { maxConcurrentPullRequests: maxConcurrentPulls, @@ -67,18 +60,82 @@ export class SyncManager { }); } - pull(collection, query, options) { - if (!isNonemptyString(collection)) { - return Promise.reject(new KinveyError('Invalid or missing collection name')); - } + pull(collection, query, options = {}) { + return Promise.resolve() + .then(() => { + if (!isNonemptyString(collection)) { + throw new KinveyError('Invalid or missing collection name'); + } + }) + .then(() => { + if (options.useDeltaSet) { + return deltaSet(collection, query, options) + .then((response) => { + return getCachedQuery(collection, query) + .then((cachedQuery) => { + if (cachedQuery) { + cachedQuery.lastRequest = response.headers.requestStart; + return updateCachedQuery(cachedQuery); + } + + return null; + }) + .then(() => response.data); + }) + .then((data) => { + if (data.deleted.length > 0) { + const deleteQuery = new Query(); + deleteQuery.containsAll('_id', data.deleted.map((entity) => entity._id)); + return this._deleteOfflineEntities(collection, deleteQuery) + .then(() => data); + } + + return data; + }) + .then((data) => { + if (data.changed.length > 0) { + return this._getOfflineRepo() + .then((offlineRepo) => offlineRepo.update(collection, data.changed)) + .then(() => data.changed.length); + } + + return 0; + }); + } else if (options.autoPagination) { + return this._paginatedPull(collection, query, options); + } - if (options && (options.autoPagination && !options.useDeltaFetch)) { - return this._paginatedPull(collection, query, options); - } + return this._fetchItemsFromServer(collection, query, options) + .then((response) => { + return getCachedQuery(collection, query) + .then((cachedQuery) => { + if (cachedQuery && response.headers) { + cachedQuery.lastRequest = response.headers.requestStart; + return updateCachedQuery(cachedQuery); + } + + return null; + }) + .then(() => response.data ? response.data : response); + }) + .then((data) => this._replaceOfflineEntities(collection, query, data).then((data) => data.length)); + }) + .catch((error) => { + if (error instanceof InvalidCachedQuery) { + return getCachedQuery(collection, query) + .then((cachedQuery) => deleteCachedQuery(cachedQuery)) + .catch((error) => { + if (error instanceof NotFoundError) { + return null; + } + + throw error; + }) + .then(() => this.pull(collection, query, Object.assign(options, { useDeltaSet: false }))); + } - return this._fetchItemsFromServer(collection, query, options) - .then(entities => this._replaceOfflineEntities(collection, query, entities)) - .then(replacedEntities => replacedEntities.length); + throw error; + }); } getSyncItemCount(collection) { @@ -150,6 +207,11 @@ export class SyncManager { _replaceOfflineEntities(collection, deleteOfflineQuery, networkEntities = []) { // TODO: this can potentially be deleteOfflineQuery.and().notIn(networkEntitiesIds) // but inmemory filtering with this filter seems to take too long + if (deleteOfflineQuery && (deleteOfflineQuery.hasSkip() || deleteOfflineQuery.hasLimit())) { + return this._getOfflineRepo() + .then((repo) => repo.update(collection, networkEntities)); + } + return this._deleteOfflineEntities(collection, deleteOfflineQuery) .then(() => this._getOfflineRepo()) .then(repo => repo.update(collection, networkEntities)); @@ -290,7 +352,7 @@ export class SyncManager { } _fetchItemsFromServer(collection, query, options) { - return this._networkRepo.read(collection, query, options); + return this._networkRepo.read(collection, query, Object.assign(options, { dataOnly: false })); } _getOfflineRepo() { @@ -365,8 +427,8 @@ export class SyncManager { _getInternalPullQuery(userQuery, totalCount) { userQuery = userQuery || {}; - const { filter, sort, fields, skip } = userQuery; - const query = new Query({ filter, sort, fields, skip }); + const { filter, sort, fields } = userQuery; + const query = new Query({ filter, sort, fields }); query.limit = totalCount; if (!sort || isEmpty(sort)) { @@ -396,27 +458,40 @@ export class SyncManager { _getExpectedEntityCount(collection, userQuery) { const countQuery = new Query({ filter: userQuery.filter }); - return this._networkRepo.count(collection, countQuery) - .then(totalCount => { - return Math.min(totalCount - userQuery.skip, userQuery.limit || Infinity); + return this._networkRepo.count(collection, countQuery, { dataOnly: false }) + .then((response) => { + return { + lastRequest: response.headers ? response.headers.requestStart : undefined, + count: response.data ? response.data.count : response + }; }); } _paginatedPull(collection, userQuery, options = {}) { let pullQuery; - let expectedCount; userQuery = userQuery || new Query(); return this._getExpectedEntityCount(collection, userQuery) - .then((count) => { - expectedCount = count; - pullQuery = this._getInternalPullQuery(userQuery, expectedCount); - return this._deleteOfflineEntities(collection, pullQuery); - }) - .then(() => { - const pageSizeSetting = options.autoPagination && options.autoPagination.pageSize; - const pageSize = pageSizeSetting || maxEntityLimit; - const paginatedQueries = splitQueryIntoPages(pullQuery, pageSize, expectedCount); - return this._executePaginationQueries(collection, paginatedQueries, options); + .then(({ lastRequest, count }) => { + pullQuery = this._getInternalPullQuery(userQuery, count); + return this._deleteOfflineEntities(collection, pullQuery) + .then(() => { + const pageSizeSetting = options.autoPagination && options.autoPagination.pageSize; + const pageSize = pageSizeSetting || maxEntityLimit; + const paginatedQueries = splitQueryIntoPages(pullQuery, pageSize, count); + return this._executePaginationQueries(collection, paginatedQueries, options); + }) + .then((result) => { + return getCachedQuery(collection, userQuery) + .then((cachedQuery) => { + if (cachedQuery) { + cachedQuery.lastRequest = lastRequest; + return updateCachedQuery(cachedQuery); + } + + return null; + }) + .then(() => result); + }); }); } } diff --git a/src/core/datastore/utils/utils.js b/src/core/datastore/utils/utils.js index 2356cc6c2..b04a6a538 100644 --- a/src/core/datastore/utils/utils.js +++ b/src/core/datastore/utils/utils.js @@ -5,6 +5,8 @@ import { isNonemptyString } from '../../utils'; export const dataStoreTagSeparator = '.'; +export { buildCollectionUrl } from '../repositories/utils'; + export function getEntitiesPendingPushError(entityCount, prefix) { let countMsg = `are ${entityCount} entities, matching the provided query`; diff --git a/src/core/datastore/working-with-data-tests/cache-offline-data-processor.spec.js b/src/core/datastore/working-with-data-tests/cache-offline-data-processor.spec.js index f07576481..b99b522e4 100644 --- a/src/core/datastore/working-with-data-tests/cache-offline-data-processor.spec.js +++ b/src/core/datastore/working-with-data-tests/cache-offline-data-processor.spec.js @@ -100,7 +100,7 @@ describe('CacheOfflineDataProcessor', () => { it('should call OfflineRepo.read()', () => { return dataProcessor.process(operation, options).toPromise() .then(() => { - validateSpyCalls(offlineRepoMock.read, 1, [collection, operation.query, options]); + validateSpyCalls(offlineRepoMock.read, 2, [collection, operation.query, options], [collection, operation.query, options]); }); }); diff --git a/src/core/datastore/working-with-data-tests/sync-manager.spec.js b/src/core/datastore/working-with-data-tests/sync-manager.spec.js index 28f80b726..6d23f549c 100644 --- a/src/core/datastore/working-with-data-tests/sync-manager.spec.js +++ b/src/core/datastore/working-with-data-tests/sync-manager.spec.js @@ -354,29 +354,17 @@ describe('SyncManager delegating to repos and SyncStateManager', () => { networkRepoMock.count = createPromiseSpy(backendEntityCount); }); - it('should do a regular deltaset request if useDeltaFetch is true', () => { - options.useDeltaFetch = true; - return syncManager.pull(collection, null, options) - .then(() => { - validateSpyCalls(utilsMock.splitQueryIntoPages, 0); - validateSpyCalls(networkRepoMock.count, 0); - validateSpyCalls(networkRepoMock.read, 1, [collection, null, options]); - validateSpyCalls(offlineRepoMock.delete, 1, [collection, null]); - validateSpyCalls(offlineRepoMock.update, 1, [collection, []]); - }); - }); - it('should do a paginated request, when autoPagination is true', () => { return syncManager.pull(collection, null, options) .then(() => { - validateSpyCalls(networkRepoMock.count, 1, [collection, new Query()]); + validateSpyCalls(networkRepoMock.count, 1, [collection, new Query(), { dataOnly: false }]); }); }); it('should do a paginated request, when autoPagination is an object', () => { return syncManager.pull(collection, null, { autoPagination: { pageSize: 14 } }) .then(() => { - validateSpyCalls(networkRepoMock.count, 1, [collection, new Query()]); + validateSpyCalls(networkRepoMock.count, 1, [collection, new Query(), { dataOnly: false }]); }); }); @@ -386,7 +374,7 @@ describe('SyncManager delegating to repos and SyncStateManager', () => { query.ascending(randomString()); // check that sort is ignored return syncManager.pull(collection, query, options) .then(() => { - validateSpyCalls(networkRepoMock.count, 1, [collection, new Query()]); + validateSpyCalls(networkRepoMock.count, 1, [collection, new Query(), { dataOnly: false }]); }); }); @@ -464,68 +452,29 @@ describe('SyncManager delegating to repos and SyncStateManager', () => { }); }); - it('should reflect user query skip', () => { + it('should ignore user query skip', () => { const query = new Query(); query.skip = 123; - return syncManager.pull(collection, query, options) - .then(() => { - const expectedQuery = new Query(); - expectedQuery.skip = query.skip; - expectedQuery.limit = backendEntityCount - query.skip; - expectedQuery.ascending(defaultSortField); - validateSpyCalls(utilsMock.splitQueryIntoPages, 1, [expectedQuery, pageSize, backendEntityCount - query.skip]); - }); - }); - - it('should reflect user query limit', () => { - const query = new Query(); - query.limit = 4321; - return syncManager.pull(collection, query, options) - .then(() => { - const expectedQuery = new Query(); - expectedQuery.skip = 0; - expectedQuery.limit = query.limit; - expectedQuery.ascending(defaultSortField); - validateSpyCalls(utilsMock.splitQueryIntoPages, 1, [expectedQuery, pageSize, query.limit]); - }); - }); - - it('should have a limit value equal to the total entity count, if total count is less than userQuery.limit', () => { - const query = new Query(); - query.limit = 1e10; return syncManager.pull(collection, query, options) .then(() => { const expectedQuery = new Query(); expectedQuery.skip = 0; expectedQuery.limit = backendEntityCount; expectedQuery.ascending(defaultSortField); - validateSpyCalls(utilsMock.splitQueryIntoPages, 1, [expectedQuery, pageSize, backendEntityCount - query.skip]); + validateSpyCalls(utilsMock.splitQueryIntoPages, 1, [expectedQuery, pageSize, backendEntityCount]); }); }); - it('should have a limit value equal to the userQuery.limit, if total count is greater than userQuery.limit', () => { + it('should ignore user query limit', () => { const query = new Query(); - query.limit = 3; + query.limit = 4321; return syncManager.pull(collection, query, options) .then(() => { const expectedQuery = new Query(); expectedQuery.skip = 0; - expectedQuery.limit = query.limit; - expectedQuery.ascending(defaultSortField); - validateSpyCalls(utilsMock.splitQueryIntoPages, 1, [expectedQuery, pageSize, query.limit]); - }); - }); - - it('should calculate the total entity count to pull, considering the userQuery.skip value', () => { - const query = new Query(); - query.skip = 12; - return syncManager.pull(collection, query, options) - .then(() => { - const expectedQuery = new Query(); - expectedQuery.skip = query.skip; - expectedQuery.limit = backendEntityCount - query.skip; + expectedQuery.limit = backendEntityCount; expectedQuery.ascending(defaultSortField); - validateSpyCalls(utilsMock.splitQueryIntoPages, 1, [expectedQuery, pageSize, expectedQuery.limit]); + validateSpyCalls(utilsMock.splitQueryIntoPages, 1, [expectedQuery, pageSize, backendEntityCount]); }); }); }); diff --git a/src/core/errors/index.js b/src/core/errors/index.js index fd0a696ca..aeb7563cc 100644 --- a/src/core/errors/index.js +++ b/src/core/errors/index.js @@ -11,13 +11,15 @@ export * from './featureUnavailable'; export * from './incompleteRequestBody'; export * from './indirectCollectionAccessDisallowed'; export * from './insufficientCredentials'; +export * from './invalidCachedQuery'; export * from './invalidCredentials'; export * from './invalidIdentifier'; export * from './invalidQuerySyntax'; export * from './jsonParse'; +export * from './kinvey'; export * from './kinveyInternalErrorRetry'; export * from './kinveyInternalErrorStop'; -export * from './kinvey'; +export * from './missingConfiguration'; export * from './missingQuery'; export * from './missingRequestHeader'; export * from './missingRequestParameter'; @@ -29,6 +31,7 @@ export * from './notFound'; export * from './parameterValueOutOfRange'; export * from './popup'; export * from './query'; +export * from './resultSetSizeExceeded'; export * from './server'; export * from './staleRequest'; export * from './sync'; diff --git a/src/core/errors/invalidCachedQuery.js b/src/core/errors/invalidCachedQuery.js new file mode 100644 index 000000000..5e9b95322 --- /dev/null +++ b/src/core/errors/invalidCachedQuery.js @@ -0,0 +1,12 @@ +import { BaseError } from './base'; + +export function InvalidCachedQuery(message, debug, code, kinveyRequestId) { + this.name = 'InvalidCachedQuery'; + this.message = message || 'Invalid cached query.'; + this.debug = debug || undefined; + this.code = code || undefined; + this.kinveyRequestId = kinveyRequestId || undefined; + this.stack = (new Error()).stack; +} +InvalidCachedQuery.prototype = Object.create(BaseError.prototype); +InvalidCachedQuery.prototype.constructor = InvalidCachedQuery; diff --git a/src/core/errors/missingConfiguration.js b/src/core/errors/missingConfiguration.js new file mode 100644 index 000000000..78e68da6d --- /dev/null +++ b/src/core/errors/missingConfiguration.js @@ -0,0 +1,12 @@ +import { BaseError } from './base'; + +export function MissingConfigurationError(message, debug, code, kinveyRequestId) { + this.name = 'MissingConfigurationError'; + this.message = message || 'Missing configuration error.'; + this.debug = debug || undefined; + this.code = code || undefined; + this.kinveyRequestId = kinveyRequestId || undefined; + this.stack = (new Error()).stack; +} +MissingConfigurationError.prototype = Object.create(BaseError.prototype); +MissingConfigurationError.prototype.constructor = MissingConfigurationError; diff --git a/src/core/errors/resultSetSizeExceeded.js b/src/core/errors/resultSetSizeExceeded.js new file mode 100644 index 000000000..f183294a0 --- /dev/null +++ b/src/core/errors/resultSetSizeExceeded.js @@ -0,0 +1,12 @@ +import { BaseError } from './base'; + +export function ResultSetSizeExceededError(message, debug, code, kinveyRequestId) { + this.name = 'ResultSetSizeExceededError'; + this.message = message || 'Result set size exceeded.'; + this.debug = debug || undefined; + this.code = code || undefined; + this.kinveyRequestId = kinveyRequestId || undefined; + this.stack = (new Error()).stack; +} +ResultSetSizeExceededError.prototype = Object.create(BaseError.prototype); +ResultSetSizeExceededError.prototype.constructor = ResultSetSizeExceededError; diff --git a/src/core/query.js b/src/core/query.js index 9bd302caa..191a8d081 100644 --- a/src/core/query.js +++ b/src/core/query.js @@ -216,6 +216,14 @@ export class Query { }, true); } + hasSkip() { + return isNumber(this.skip) && this.skip > 0; + } + + hasLimit() { + return isNumber(this.limit); + } + /** * Adds an equal to filter to the query. Requires `field` to equal `value`. * Any existing filters on `field` will be discarded. diff --git a/src/core/request/deltafetch.js b/src/core/request/deltafetch.js deleted file mode 100644 index d95d0f1ea..000000000 --- a/src/core/request/deltafetch.js +++ /dev/null @@ -1,193 +0,0 @@ -import Promise from 'es6-promise'; -import keyBy from 'lodash/keyBy'; -import reduce from 'lodash/reduce'; -import result from 'lodash/result'; -import values from 'lodash/values'; -import forEach from 'lodash/forEach'; -import isArray from 'lodash/isArray'; -import isString from 'lodash/isString'; -import { KinveyError, NotFoundError } from '../errors'; -import { isDefined } from '../utils'; -import { Query } from '../query'; -import { RequestMethod } from './request'; -import { KinveyRequest } from './network'; -import { Response, StatusCode } from './response'; -import { repositoryProvider } from '../datastore/repositories'; - -const maxIdsPerRequest = 200; - -export class DeltaFetchRequest extends KinveyRequest { - get method() { - return super.method; - } - - set method(method) { - // Cast the method to a string - if (!isString(method)) { - method = String(method); - } - - // Make sure the the method is upper case - method = method.toUpperCase(); - - // Verify that the method is allowed - switch (method) { - case RequestMethod.GET: - super.method = method; - break; - case RequestMethod.POST: - case RequestMethod.PATCH: - case RequestMethod.PUT: - case RequestMethod.DELETE: - default: - throw new KinveyError('Invalid request Method. Only RequestMethod.GET is allowed.'); - } - } - - execute() { - return repositoryProvider.getOfflineRepository() - .then(repo => repo.read(this._getCollectionFromUrl(), this.query)) - .catch((error) => { - if (!(error instanceof NotFoundError)) { - throw error; - } - - return []; - }) - .then((cacheData) => { - if (isArray(cacheData) && cacheData.length > 0) { - const cacheDocuments = keyBy(cacheData, '_id'); - const query = new Query(result(this.query, 'toJSON', this.query)); - query.fields = ['_id', '_kmd.lmt']; - const request = new KinveyRequest({ - method: RequestMethod.GET, - url: this.url, - headers: this.headers, - authType: this.authType, - query: query, - timeout: this.timeout, - client: this.client, - properties: this.properties, - skipBL: this.skipBL, - trace: this.trace, - followRedirect: this.followRedirect, - cache: this.cache - }); - - return request.execute() - .then(response => response.data) - .then((networkData) => { - const networkDocuments = keyBy(networkData, '_id'); - const deltaSet = networkDocuments; - const cacheDocumentIds = Object.keys(cacheDocuments); - - forEach(cacheDocumentIds, (id) => { - const cacheDocument = cacheDocuments[id]; - const networkDocument = networkDocuments[id]; - - if (networkDocument) { - if (isDefined(networkDocument._kmd) && isDefined(cacheDocument._kmd) - && networkDocument._kmd.lmt === cacheDocument._kmd.lmt) { - delete deltaSet[id]; - } else { - delete cacheDocuments[id]; - } - } else { - delete cacheDocuments[id]; - } - }); - - const deltaSetIds = Object.keys(deltaSet); - const promises = []; - let i = 0; - - while (i < deltaSetIds.length) { - const query = new Query(result(this.query, 'toJSON', this.query)); - const ids = deltaSetIds.slice(i, deltaSetIds.length > maxIdsPerRequest + i ? - maxIdsPerRequest : deltaSetIds.length); - query.contains('_id', ids); - - const request = new KinveyRequest({ - method: RequestMethod.GET, - url: this.url, - headers: this.headers, - authType: this.authType, - query: query, - timeout: this.timeout, - client: this.client, - properties: this.properties, - skipBL: this.skipBL, - trace: this.trace, - followRedirect: this.followRedirect, - cache: this.cache - }); - - const promise = request.execute(); - promises.push(promise); - i += maxIdsPerRequest; - } - - return Promise.all(promises); - }) - .then((responses) => { - const response = reduce(responses, (result, response) => { - if (response.isSuccess()) { - const headers = result.headers; - headers.addAll(response.headers); - result.headers = headers; - result.data = result.data.concat(response.data); - } - - return result; - }, new Response({ - statusCode: StatusCode.Ok, - data: [] - })); - - response.data = response.data.concat(values(cacheDocuments)); - - if (this.query) { - const query = new Query(result(this.query, 'toJSON', this.query)); - query.skip = 0; - query.limit = 0; - response.data = query.process(response.data); - } - - return response; - }); - } - - const request = new KinveyRequest({ - method: RequestMethod.GET, - url: this.url, - headers: this.headers, - authType: this.authType, - query: this.query, - timeout: this.timeout, - client: this.client, - properties: this.properties, - skipBL: this.skipBL, - trace: this.trace, - followRedirect: this.followRedirect, - cache: this.cache - }); - return request.execute(); - }); - } - - _getCollectionFromUrl() { - const appkeyStr = `appdata/${this.client.appKey}/`; - const ind = this.url.indexOf(appkeyStr); - let indOfQueryString = this.url.indexOf('?'); // shouldn't have anything past the collection, like an ID - - if (ind === -1) { - throw new KinveyError('An unexpected error occured. Could not find collection'); - } - - if (indOfQueryString === -1) { - indOfQueryString = Infinity; - } - - return this.url.substring(ind + appkeyStr.length, indOfQueryString); - } -} diff --git a/src/core/request/deltafetch.spec.js b/src/core/request/deltafetch.spec.js deleted file mode 100644 index 0ae0c37ba..000000000 --- a/src/core/request/deltafetch.spec.js +++ /dev/null @@ -1,316 +0,0 @@ -import nock from 'nock'; -import expect from 'expect'; -import { AuthType } from './network'; -import { DeltaFetchRequest } from './deltafetch'; -import { RequestMethod } from './request'; -import { NetworkRack } from './rack'; -import { KinveyError } from '../errors'; -import { SyncStore } from '../datastore'; -import { randomString } from '../utils'; -import { Query } from '../query'; -import { init } from '../kinvey'; -import { User } from '../user'; -import { NodeHttpMiddleware } from '../../node/http'; - -const collection = 'books'; - -describe('DeltaFetchRequest', () => { - let client; - - before(() => { - NetworkRack.useHttpMiddleware(new NodeHttpMiddleware({})); - }); - - before(() => { - client = init({ - appKey: randomString(), - appSecret: randomString() - }); - }); - - before(() => { - const username = randomString(); - const password = randomString(); - const reply = { - _id: randomString(), - _kmd: { - lmt: new Date().toISOString(), - ect: new Date().toISOString(), - authtoken: randomString() - }, - username: username, - _acl: { - creator: randomString() - } - }; - - nock(client.apiHostname) - .post(`/user/${client.appKey}/login`, { username: username, password: password }) - .reply(200, reply); - - return User.login(username, password); - }); - - describe('method', () => { - it('should not be able to be set to POST', () => { - expect(() => { - const request = new DeltaFetchRequest(); - request.method = RequestMethod.POST; - }).toThrow(KinveyError, /Invalid request Method. Only RequestMethod.GET is allowed./); - }); - - it('should not be able to be set to PATCH', () => { - expect(() => { - const request = new DeltaFetchRequest(); - request.method = RequestMethod.PATCH; - }).toThrow(KinveyError, /Invalid request Method. Only RequestMethod.GET is allowed./); - }); - - it('should not be able to be set to PUT', () => { - expect(() => { - const request = new DeltaFetchRequest(); - request.method = RequestMethod.PUT; - }).toThrow(KinveyError, /Invalid request Method. Only RequestMethod.GET is allowed./); - }); - - it('should not be able to be set to DELETE', () => { - expect(() => { - const request = new DeltaFetchRequest(); - request.method = RequestMethod.DELETE; - }).toThrow(KinveyError, /Invalid request Method. Only RequestMethod.GET is allowed./); - }); - - it('should be able to be set to GET', () => { - const request = new DeltaFetchRequest(); - request.method = RequestMethod.GET; - expect(request.method).toEqual(RequestMethod.GET); - }); - }); - - describe('execute()', () => { - const entity1 = { - _id: randomString(), - _acl: { - creator: randomString() - }, - _kmd: { - lmt: new Date().toISOString(), - ect: new Date().toISOString() - }, - title: 'entity1' - }; - const entity2 = { - _id: randomString(), - _acl: { - creator: randomString() - }, - _kmd: { - lmt: new Date().toISOString(), - ect: new Date().toISOString() - }, - title: 'entity2' - }; - - beforeEach(() => { - // API response - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .reply(200, [entity1, entity2], { - 'Content-Type': 'application/json' - }); - - // Pull data into cache - const store = new SyncStore(collection); - return store.pull() - .then((entities) => { - expect(entities).toEqual(2); - }); - }); - - afterEach(() => { - const store = new SyncStore(collection); - return store.clear() - .then(() => { - return store.find().toPromise(); - }) - .then((entities) => { - expect(entities).toEqual([]); - }); - }); - - it('should not fetch any entities from the network when delta set is empty', () => { - const request = new DeltaFetchRequest({ - method: RequestMethod.GET, - authType: AuthType.Default, - url: `${client.apiHostname}/appdata/${client.appKey}/${collection}`, - client: client - }); - - // API response - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .query({ - fields: '_id,_kmd.lmt' - }) - .reply(200, [], { - 'Content-Type': 'application/json' - }); - - return request.execute() - .then((response) => { - const data = response.data; - expect(data).toBeA(Array); - expect(data).toEqual([]); - }); - }); - - it('should not fetch any entities from the network when delta set is equal to the cache', () => { - const request = new DeltaFetchRequest({ - method: RequestMethod.GET, - authType: AuthType.Default, - url: `${client.apiHostname}/appdata/${client.appKey}/${collection}`, - client: client - }); - - // API response - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .query({ - fields: '_id,_kmd.lmt' - }) - .reply(200, [{ - _id: entity1._id, - _kmd: entity1._kmd - }, { - _id: entity2._id, - _kmd: entity2._kmd - }], { - 'Content-Type': 'application/json' - }); - - return request.execute() - .then((response) => { - const data = response.data; - expect(data).toBeA(Array); - expect(data).toEqual([entity1, entity2]); - }); - }); - - it('should fetch only the updated entities from the network', () => { - const request = new DeltaFetchRequest({ - method: RequestMethod.GET, - authType: AuthType.Default, - url: `${client.apiHostname}/appdata/${client.appKey}/${collection}`, - client: client - }); - - // API response - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .query({ - fields: '_id,_kmd.lmt' - }) - .reply(200, [{ - _id: entity1._id, - _kmd: entity1._kmd - }, { - _id: entity2._id, - _kmd: { - lmt: new Date().toISOString(), - ect: entity2._kmd.ect - } - }], { - 'Content-Type': 'application/json' - }); - - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .query({ - query: `{"_id":{"$in":["${entity2._id}"]}}` - }) - .reply(200, [entity2], { - 'Content-Type': 'application/json' - }); - - return request.execute() - .then((response) => { - const data = response.data; - expect(data).toBeA(Array); - expect(data).toEqual([entity2, entity1]); - }); - }); - - it('should fetch only the updated entities from the network matching the query', () => { - const query = new Query(); - query.equalTo('_id', entity1._id); - const request = new DeltaFetchRequest({ - method: RequestMethod.GET, - authType: AuthType.Default, - url: `${client.apiHostname}/appdata/${client.appKey}/${collection}`, - query: query, - client: client - }); - - // API response - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .query({ - fields: '_id,_kmd.lmt', - query: `{"_id":"${entity1._id}"}` - }) - .reply(200, [{ - _id: entity1._id, - _kmd: { - lmt: new Date().toISOString(), - ect: entity1._kmd.ect - } - }], { - 'Content-Type': 'application/json' - }); - - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .query({ - query: `{"_id":{"$in":["${entity1._id}"]}}` - }) - .reply(200, [entity1], { - 'Content-Type': 'application/json' - }); - - return request.execute() - .then((response) => { - const data = response.data; - expect(data).toBeA(Array); - expect(data).toEqual([entity1]); - }); - }); - - it('should fetch the data from the network if there is not any data in the cache', () => { - const store = new SyncStore(collection); - const request = new DeltaFetchRequest({ - method: RequestMethod.GET, - authType: AuthType.Default, - url: `${client.apiHostname}/appdata/${client.appKey}/${collection}`, - client: client - }); - - // API response - nock(client.apiHostname, { encodedQueryParams: true }) - .get(`/appdata/${client.appKey}/${collection}`) - .reply(200, [], { - 'Content-Type': 'application/json' - }); - - - return store.clear() - .then(() => { - return request.execute(); - }) - .then((response) => { - const data = response.data; - expect(data).toBeA(Array); - expect(data).toEqual([]); - }); - }); - }); -}); diff --git a/src/core/request/headers.js b/src/core/request/headers.js index 84a037c24..2706aa6bd 100644 --- a/src/core/request/headers.js +++ b/src/core/request/headers.js @@ -3,12 +3,18 @@ import isString from 'lodash/isString'; import isPlainObject from 'lodash/isPlainObject'; import { isDefined } from '../utils'; +export const kinveyRequestStartHeader = 'X-Kinvey-Request-Start'; + export class Headers { constructor(headers = {}) { this.headers = {}; this.addAll(headers); } + get requestStart() { + return this.get(kinveyRequestStartHeader); + } + get(name) { if (name) { if (isString(name) === false) { diff --git a/src/core/request/index.js b/src/core/request/index.js index a467802ca..0fd28c73e 100644 --- a/src/core/request/index.js +++ b/src/core/request/index.js @@ -1,4 +1,3 @@ -export * from './deltafetch'; export * from './headers'; export * from './network'; export * from './request'; diff --git a/src/core/request/response.js b/src/core/request/response.js index 14a49fa63..025b038ad 100644 --- a/src/core/request/response.js +++ b/src/core/request/response.js @@ -22,8 +22,10 @@ import { MissingQueryError, MissingRequestHeaderError, MissingRequestParameterError, + MissingConfigurationError, NotFoundError, ParameterValueOutOfRangeError, + ResultSetSizeExceededError, ServerError, StaleRequestError, UserAlreadyExistsError, @@ -182,6 +184,8 @@ export class KinveyResponse extends Response { error = new MissingRequestHeaderError(message, debug, code, kinveyRequestId); } else if (name === 'MissingRequestParameter') { error = new MissingRequestParameterError(message, debug, code, kinveyRequestId); + } else if (name === 'MissingConfiguration') { + error = new MissingConfigurationError(message, debug, code, kinveyRequestId); } else if (name === 'EntityNotFound' || name === 'CollectionNotFound' || name === 'AppNotFound' @@ -191,6 +195,8 @@ export class KinveyResponse extends Response { error = new NotFoundError(message, debug, code, kinveyRequestId); } else if (name === 'ParameterValueOutOfRange') { error = new ParameterValueOutOfRangeError(message, debug, code, kinveyRequestId); + } else if (name === 'ResultSetSizeExceeded') { + error = new ResultSetSizeExceededError(message, debug, code, kinveyRequestId); } else if (name === 'ServerError') { error = new ServerError(message, debug, code, kinveyRequestId); } else if (name === 'StaleRequest') { diff --git a/src/kinvey.d.ts b/src/kinvey.d.ts index 9e1fc53ad..92bf0e633 100644 --- a/src/kinvey.d.ts +++ b/src/kinvey.d.ts @@ -23,7 +23,7 @@ export namespace Kinvey { interface RequestOptions { properties?: Properties; timeout?: number; - useDeltaFetch?: boolean; + useDeltaSet?: boolean; version?: string; micId?: string; } @@ -265,7 +265,7 @@ export namespace Kinvey { static collection(collection: string, type?: DataStoreType, options?: { client?: Client; ttl?: number; - useDeltaFetch?: boolean; + useDeltaSet?: boolean; }): CacheStore; static clearCache(options?: RequestOptions): Promise<{}>; } @@ -275,7 +275,6 @@ export namespace Kinvey { protected constructor(); client: Client; pathname: string; - useDeltaFetch: boolean; find(query?: Query, options?: RequestOptions): Observable; findById(id: string, options?: RequestOptions): Observable; @@ -299,6 +298,8 @@ export namespace Kinvey { // CacheStore class class CacheStore extends NetworkStore { + useDeltaSet: boolean; + clear(query?: Query, options?: RequestOptions): Promise<{ count: number }>; @@ -341,7 +342,6 @@ export namespace Kinvey { // Files class export class Files { - static useDeltaFetch: boolean; static find(query?: Query, options?: RequestOptions): Promise; static findById(id: string, options?: RequestOptions): Promise; static download(name: string, options?: RequestOptions): Promise; @@ -543,7 +543,7 @@ export function ping(): Promise; interface RequestOptions { properties?: Properties; timeout?: number; - useDeltaFetch?: boolean; + useDeltaSet?: boolean; version?: string; micId?: string; } @@ -780,7 +780,7 @@ interface SyncEntity extends Entity { interface PullRequestOptions { timeout?: number, - useDeltaFetch?: boolean, + useDeltaSet?: boolean, autoPagination?: boolean | { pageSize?: number } } @@ -789,7 +789,7 @@ export abstract class DataStore { static collection(collection: string, type?: DataStoreType, options?: { client?: Client; ttl?: number; - useDeltaFetch?: boolean; + useDeltaSet?: boolean; }): CacheStore; static clearCache(options?: RequestOptions): Promise<{}>; } @@ -799,7 +799,6 @@ declare class NetworkStore { protected constructor(); client: Client; pathname: string; - useDeltaFetch: boolean; find(query?: Query, options?: RequestOptions): Observable; findById(id: string, options?: RequestOptions): Observable; @@ -817,6 +816,8 @@ declare class NetworkStore { // CacheStore class declare class CacheStore extends NetworkStore { + useDeltaSet: boolean; + clear(query?: Query, options?: RequestOptions): Promise<{ count: number }>; @@ -859,7 +860,6 @@ export interface File extends Entity { // Files class export class Files { - static useDeltaFetch: boolean; static find(query?: Query, options?: RequestOptions): Promise; static findById(id: string, options?: RequestOptions): Promise; static download(name: string, options?: RequestOptions): Promise; diff --git a/src/nativescript/filestore/filestore.d.ts b/src/nativescript/filestore/filestore.d.ts index aa9414ef3..a034a899d 100644 --- a/src/nativescript/filestore/filestore.d.ts +++ b/src/nativescript/filestore/filestore.d.ts @@ -1,7 +1,6 @@ import { Kinvey } from '../kinvey'; export class FileStore { - useDeltaFetch: boolean; find(query?: Kinvey.Query, options?: Kinvey.RequestOptions): Promise; findById(id: string, options?: Kinvey.RequestOptions): Promise; download(name: string, options?: Kinvey.RequestOptions): Promise; diff --git a/test/integration/configs/tests-config.js b/test/integration/configs/tests-config.js index bb556896e..818ea226d 100644 --- a/test/integration/configs/tests-config.js +++ b/test/integration/configs/tests-config.js @@ -1,6 +1,7 @@ const testsConfig = { collectionName: 'Books', + deltaCollectionName: 'BooksDelta', fbEmail: process.env.FACEBOOK_EMAIL, fbPassword: process.env.FACEBOOK_PASSWORD, authServiceId: 'decad9197f0f4680a46d902327c5c131' diff --git a/test/integration/tests/delta-set.js b/test/integration/tests/delta-set.js new file mode 100644 index 000000000..92d5ba80b --- /dev/null +++ b/test/integration/tests/delta-set.js @@ -0,0 +1,1167 @@ + +function testFunc() { + const dataStoreTypes = [Kinvey.DataStoreType.Cache, Kinvey.DataStoreType.Sync]; + const deltaCollectionName = externalConfig.deltaCollectionName; + const collectionWithoutDelta = externalConfig.collectionName; + const deltaNetworkStore = Kinvey.DataStore.collection(deltaCollectionName, Kinvey.DataStoreType.Network); + const syncStore = Kinvey.DataStore.collection(deltaCollectionName, Kinvey.DataStoreType.Sync); + const cacheStore = Kinvey.DataStore.collection(deltaCollectionName, Kinvey.DataStoreType.Cache); + const deltaSyncStore = Kinvey.DataStore.collection(deltaCollectionName, Kinvey.DataStoreType.Sync, { useDeltaSet: true }); + const deltaCacheStore = Kinvey.DataStore.collection(deltaCollectionName, Kinvey.DataStoreType.Cache, { useDeltaSet: true }); + const tagStore = 'kinveyTest'; + + const validatePullOperation = (result, expectedItems, expectedPulledItemsCount, tagStore, collectionName) => { + const collectioNameForStore = collectionName?collectionName:deltaCollectionName; + const taggedDataStore = tagStore?Kinvey.DataStore.collection(deltaCollectionName, Kinvey.DataStoreType.Sync, { tag: tagStore }):null; + const syncStoreToFind = Kinvey.DataStore.collection(collectioNameForStore, Kinvey.DataStoreType.Sync) + expect(result).to.equal(expectedPulledItemsCount || expectedItems.length); + const storeToFind = tagStore ? taggedDataStore:syncStoreToFind; + return storeToFind.find().toPromise() + .then((result) => { + expectedItems.forEach((entity) => { + const cachedEntity = _.find(result, e => e._id === entity._id); + expect(utilities.deleteEntityMetadata(cachedEntity)).to.deep.equal(entity); + }); + }); + } + + const validateNewPullOperation = (result, expectedPulledItems, expectedDeletedItems, tagStore) => { + expect(result).to.equal(expectedPulledItems.length); + const storeToFind = tagStore ? Kinvey.DataStore.collection(deltaCollectionName, Kinvey.DataStoreType.Sync, { tag: tagStore }) : syncStore; + return storeToFind.find().toPromise() + .then((result) => { + expectedPulledItems.forEach((entity) => { + const cachedEntity = _.find(result, e => e._id === entity._id); + expect(utilities.deleteEntityMetadata(cachedEntity)).to.deep.equal(entity); + }); + + expectedDeletedItems.forEach((entity) => { + const deletedEntity = _.find(result, e => e._id === entity._id); + expect(deletedEntity).to.equal(undefined); + }); + }); + } + + dataStoreTypes.forEach((currentDataStoreType) => { + describe(`${currentDataStoreType} Deltaset tests`, () => { + const conditionalDescribe = currentDataStoreType === Kinvey.DataStoreType.Sync ? describe.skip : describe; + describe('pull', () => { + const entity1 = utilities.getEntity(utilities.randomString()); + const entity2 = utilities.getEntity(utilities.randomString()); + const entity3 = utilities.getEntity(utilities.randomString()); + const createdUserIds = []; + const deltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true }); + const taggedDeltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true, tag: tagStore }); + + before((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => Kinvey.User.signup()) + .then((user) => { + createdUserIds.push(user.data._id); + done(); + }) + .catch(done); + }); + + beforeEach((done) => { + utilities.cleanUpCollectionData(deltaCollectionName) + .then(() => deltaNetworkStore.save(entity1)) + .then(() => deltaNetworkStore.save(entity2)) + .then(() => done()) + .catch(done); + }); + + after((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items without changes', (done) => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [], [])) + .then(() => done()) + .catch((error) => done(error)); + }); + + it('should return correct number of items with disabled deltaset', (done) => { + const disabledDeltaSetStore = currentDataStoreType === Kinvey.DataStoreType.Cache ? cacheStore : syncStore; + disabledDeltaSetStore.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => disabledDeltaSetStore.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with created item', (done) => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [entity3], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with created item with 3rd request', (done) => { + const entity4 = utilities.getEntity(utilities.randomString()); + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [entity3], [])) + .then(() => deltaNetworkStore.save(entity4)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [entity4], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with auto-pagination', (done) => { + deltaStoreToTest.pull(new Kinvey.Query(), { autoPagination: true }) + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.pull(new Kinvey.Query(), { autoPagination: true })) + .then((result) => validateNewPullOperation(result, [entity3], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with auto-pagination and skip and limit', (done) => { + const query = new Kinvey.Query(); + query.skip = 1; + query.limit = 2; + deltaStoreToTest.pull(query, { autoPagination: true }) + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.pull(query, { autoPagination: true })) + .then((result) => validateNewPullOperation(result, [entity1, entity2, entity3], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with tagged dataStore', (done) => { + const onNextSpy = sinon.spy(); + syncStore.save(entity1) + .then(() => taggedDeltaStoreToTest.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2], 2, tagStore)) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => taggedDeltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [entity3], [], tagStore)) + .then(() => deltaNetworkStore.removeById(entity1._id)) + .then(() => taggedDeltaStoreToTest.pull()) + .then((result) => { validateNewPullOperation(result, [], [entity1], tagStore) }) + .then(() => { + syncStore.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(Kinvey.DataStoreType.Sync, onNextSpy, [entity1]); + done(); + } catch (error) { + done(error); + } + }) + }) + .catch(done); + }); + + it('should return correct number of items with deleted item', (done) => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.removeById(entity2._id)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [], [entity2])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with updated item', (done) => { + const updatedEntity = _.clone(entity2); + updatedEntity.textField = utilities.randomString(); + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(updatedEntity)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [updatedEntity], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with updated and deleted item', (done) => { + const updatedEntity = _.clone(entity2); + updatedEntity.textField = utilities.randomString(); + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(updatedEntity)) + .then(() => deltaNetworkStore.removeById(entity1._id)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [updatedEntity], [entity1])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with query with updated item', (done) => { + const entity4 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const entity5 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const entity6 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const updatedEntity = _.clone(entity5); + updatedEntity.numberField = 5; + const query = new Kinvey.Query(); + query.equalTo('textField', 'queryValue'); + deltaNetworkStore.save(entity4) + .then(() => deltaNetworkStore.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.pull(query) + .then((result) => validatePullOperation(result, [entity4, entity5, entity6])) + .then(() => deltaNetworkStore.save(updatedEntity)) + .then(() => deltaStoreToTest.pull(query)) + .then((result) => validateNewPullOperation(result, [updatedEntity], [])) + .then(() => done())) + .catch(done); + }); + + it('should return correct number of items with query with deleted item', (done) => { + const entity4 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const entity5 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const entity6 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const query = new Kinvey.Query(); + query.equalTo('textField', 'queryValue'); + deltaNetworkStore.save(entity4) + .then(() => deltaNetworkStore.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.pull(query) + .then((result) => validatePullOperation(result, [entity4, entity5, entity6])) + .then(() => deltaNetworkStore.removeById(entity5._id)) + .then(() => deltaStoreToTest.pull(query)) + .then((result) => validateNewPullOperation(result, [], [entity5])) + .then(() => done())) + .catch(done); + }); + + it('should not use deltaset with skip and limit query and should not record X-Kinvey-Request-Start', (done) => { + const entity4 = utilities.getEntity(utilities.randomString(), 'queryValue', 1); + const entity5 = utilities.getEntity(utilities.randomString(), 'queryValue', 2); + const entity6 = utilities.getEntity(utilities.randomString(), 'queryValue', 3); + const query = new Kinvey.Query(); + query.ascending('numberField'); + query.limit = 1; + query.skip = 1; + query.equalTo('textField', 'queryValue'); + const queryWithoutModifiers = new Kinvey.Query(); + queryWithoutModifiers.equalTo('textField', 'queryValue'); + deltaNetworkStore.save(entity4) + .then(() => deltaNetworkStore.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.pull(queryWithoutModifiers)) + .then((result) => validatePullOperation(result, [entity4, entity5, entity6])) + .then(() => deltaNetworkStore.removeById(entity4._id)) + .then(() => deltaStoreToTest.pull(query)) + .then((result) => validatePullOperation(result, [entity6])) + .then(() => deltaStoreToTest.pull(query)) + .then((result) => validatePullOperation(result, [entity6])) + .then(() => deltaStoreToTest.pull(queryWithoutModifiers)) + .then((result) => validateNewPullOperation(result, [], [entity4])) + .then(() => done()) + .catch(done); + + }); + + it('limit and skip query should not delete data', (done) => { + const onNextSpy = sinon.spy(); + const query = new Kinvey.Query(); + query.limit = 2; + query.skip = 1; + deltaNetworkStore.save(entity3) + .then(() => deltaStoreToTest.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2, entity3])) + .then(() => deltaStoreToTest.pull(query)) + .then((result) => validatePullOperation(result, [entity2, entity3])) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [], [])) + .then(() => { + syncStore.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(Kinvey.DataStoreType.Sync, onNextSpy, [entity1, entity2, entity3], [], true); + done(); + } catch (error) { + done(error); + } + }) + }) + .catch(done); + + }); + }); + + describe('sync', () => { + const dataStoreType = currentDataStoreType; + const entity1 = utilities.getEntity(utilities.randomString()); + const entity2 = utilities.getEntity(utilities.randomString()); + const entity3 = utilities.getEntity(utilities.randomString()); + const createdUserIds = []; + let deltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true }); + let taggedDeltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true, tag: tagStore }); + + before((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => Kinvey.User.signup()) + .then((user) => { + createdUserIds.push(user.data._id); + done(); + }) + .catch(done); + }); + + beforeEach((done) => { + utilities.cleanUpCollectionData(deltaCollectionName) + .then(() => deltaNetworkStore.save(entity1)) + .then(() => deltaNetworkStore.save(entity2)) + .then(() => done()) + .catch(done); + }); + + after((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items without changes', (done) => { + deltaStoreToTest.sync() + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with disabled deltaset', (done) => { + let deisabledDeltaSetStore = currentDataStoreType === Kinvey.DataStoreType.Cache ? cacheStore : syncStore; + deisabledDeltaSetStore.sync() + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deisabledDeltaSetStore.sync()) + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with tagged dataStore', (done) => { + const onNextSpy = sinon.spy(); + syncStore.save(entity1) + .then(() => taggedDeltaStoreToTest.sync()) + .then((result) => validatePullOperation(result.pull, [entity1, entity2], 2, tagStore)) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => taggedDeltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [entity3], [], tagStore)) + .then(() => deltaNetworkStore.removeById(entity1._id)) + .then(() => taggedDeltaStoreToTest.sync()) + .then((result) => { validateNewPullOperation(result.pull, [], [entity1], tagStore) }) + .then(() => { + syncStore.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(Kinvey.DataStoreType.Sync, onNextSpy, [entity1]); + done(); + } catch (error) { + done(error); + } + }) + }) + .catch(done); + }); + + it('should return correct number of items with created item', (done) => { + deltaStoreToTest.sync() + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [entity3], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with created item with 3rd request', (done) => { + const entity4 = utilities.getEntity(utilities.randomString()); + deltaStoreToTest.sync() + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [entity3], [])) + .then(() => deltaNetworkStore.save(entity4)) + .then(() => deltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [entity4], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with auto-pagination', (done) => { + deltaStoreToTest.sync(new Kinvey.Query(), { autoPagination: true }) + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.sync(new Kinvey.Query(), { autoPagination: true })) + .then((result) => validateNewPullOperation(result.pull, [entity3], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with auto-pagination and skip and limit', (done) => { + const query = new Kinvey.Query(); + query.skip = 1; + query.limit = 2; + deltaStoreToTest.sync(query, { autoPagination: true }) + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.sync(query, { autoPagination: true })) + .then((result) => validateNewPullOperation(result.pull, [entity1, entity2, entity3], [])) + .then(() => done()) + .catch(done); + }); + + + it('should return correct number of items with tagged dataStore', (done) => { + const onNextSpy = sinon.spy(); + syncStore.save(entity1) + .then(() => taggedDeltaStoreToTest.sync()) + .then((result) => validatePullOperation(result.pull, [entity1, entity2], 2, tagStore)) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => taggedDeltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [entity3], [], tagStore)) + .then(() => deltaNetworkStore.removeById(entity1._id)) + .then(() => taggedDeltaStoreToTest.sync()) + .then((result) => { validateNewPullOperation(result.pull, [], [entity1], tagStore) }) + .then(() => { + syncStore.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(Kinvey.DataStoreType.Sync, onNextSpy, [entity1]); + done(); + } catch (error) { + done(error); + } + }) + }) + .catch(done); + }); + + it('should return correct number of items with deleted item', (done) => { + deltaStoreToTest.sync() + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaNetworkStore.removeById(entity2._id)) + .then(() => deltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [], [entity2])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with updated item', (done) => { + const updatedEntity = _.clone(entity2); + updatedEntity.textField = utilities.randomString(); + deltaStoreToTest.sync() + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaNetworkStore.save(updatedEntity)) + .then(() => deltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [updatedEntity], [])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with updated and deleted item', (done) => { + const updatedEntity = _.clone(entity2); + updatedEntity.textField = utilities.randomString(); + deltaStoreToTest.sync() + .then((result) => validatePullOperation(result.pull, [entity1, entity2])) + .then(() => deltaNetworkStore.save(updatedEntity)) + .then(() => deltaNetworkStore.removeById(entity1._id)) + .then(() => deltaStoreToTest.sync()) + .then((result) => validateNewPullOperation(result.pull, [updatedEntity], [entity1])) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items with query with updated item', (done) => { + const entity4 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const entity5 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const entity6 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const updatedEntity = _.clone(entity5); + updatedEntity.numberField = 5; + const query = new Kinvey.Query(); + query.equalTo('textField', 'queryValue'); + deltaNetworkStore.save(entity4) + .then(() => deltaNetworkStore.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.sync(query) + .then((result) => validatePullOperation(result.pull, [entity4, entity5, entity6])) + .then(() => deltaNetworkStore.save(updatedEntity)) + .then(() => deltaStoreToTest.sync(query)) + .then((result) => validateNewPullOperation(result.pull, [updatedEntity], [])) + .then(() => done())) + .catch(done); + }); + + it('should return correct number of items with query with deleted item', (done) => { + let entity4 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let entity5 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let entity6 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let query = new Kinvey.Query(); + query.equalTo('textField', 'queryValue'); + deltaNetworkStore.save(entity4) + .then(() => deltaNetworkStore.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.sync(query) + .then((result) => validatePullOperation(result.pull, [entity4, entity5, entity6])) + .then(() => deltaNetworkStore.removeById(entity5._id)) + .then(() => deltaStoreToTest.sync(query)) + .then((result) => validateNewPullOperation(result.pull, [], [entity5])) + .then(() => done())) + .catch(done); + }); + + it('should not use deltaset with skip and limit query and should not record X-Kinvey-Request-Start', (done) => { + const entity4 = utilities.getEntity(utilities.randomString(), 'queryValue', 1); + const entity5 = utilities.getEntity(utilities.randomString(), 'queryValue', 2); + const entity6 = utilities.getEntity(utilities.randomString(), 'queryValue', 3); + const query = new Kinvey.Query(); + query.ascending('numberField'); + query.limit = 1; + query.skip = 1; + query.equalTo('textField', 'queryValue'); + const queryWithoutModifiers = new Kinvey.Query(); + queryWithoutModifiers.equalTo('textField', 'queryValue') + deltaNetworkStore.save(entity4) + .then(() => deltaNetworkStore.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.sync(queryWithoutModifiers)) + .then((result) => validatePullOperation(result.pull, [entity4, entity5, entity6])) + .then(() => deltaNetworkStore.removeById(entity4._id)) + .then(() => deltaStoreToTest.sync(query)) + .then((result) => validatePullOperation(result.pull, [entity6])) + .then(() => deltaStoreToTest.sync(query)) + .then((result) => validatePullOperation(result.pull, [entity6])) + .then(() => deltaStoreToTest.sync(queryWithoutModifiers)) + .then((result) => validateNewPullOperation(result.pull, [], [entity4])) + .then(() => done()) + .catch(done); + + }); + }); + + conditionalDescribe('find', () => { + const dataStoreType = currentDataStoreType; + const entity1 = utilities.getEntity(utilities.randomString()); + const entity2 = utilities.getEntity(utilities.randomString()); + const entity3 = utilities.getEntity(utilities.randomString()); + const createdUserIds = []; + let deltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true }); + + before((done) => { + + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => Kinvey.User.signup()) + .then((user) => { + createdUserIds.push(user.data._id); + done(); + }) + .catch(done); + }); + + beforeEach((done) => { + utilities.cleanUpCollectionData(deltaCollectionName) + .then(() => deltaStoreToTest.save(entity1)) + .then(() => deltaNetworkStore.save(entity2)) + .then(() => done()) + .catch(done); + }); + + after((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => done()) + .catch(done); + }); + + it('should return correct number of items without changes', (done) => { + const onNextSpy = sinon.spy(); + deltaStoreToTest.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deltaStoreToTest.find() + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [entity1, entity2], true); + done(); + } + catch (error) { + done(error); + } + }) + } + catch (error) { + done(error); + } + }); + }); + + it('should return correct number of items with disabled deltaset', (done) => { + let deisabledDeltaSetStore = cacheStore; + const onNextSpy = sinon.spy(); + deisabledDeltaSetStore.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deisabledDeltaSetStore.find() + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [entity1, entity2], true); + done(); + } + catch (error) { + done(error); + } + }) + } + catch (error) { + done(error); + } + }); + }); + + + it('should return correct number of items with created item', (done) => { + const onNextSpy = sinon.spy(); + deltaStoreToTest.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.save(entity3) + .then(() => deltaStoreToTest.find() + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [entity1, entity2, entity3], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + }); + }); + + it('should return correct number of items with created item with third request', (done) => { + let entity4 = utilities.getEntity(utilities.randomString(), 'queryValue'); + const onNextSpy = sinon.spy(); + deltaStoreToTest.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.save(entity3) + .then(() => deltaStoreToTest.find() + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [entity1, entity2, entity3], true); + const yetAnotherSpy = sinon.spy(); + deltaNetworkStore.save(entity4) + .then(() => deltaStoreToTest.find() + .subscribe(yetAnotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, yetAnotherSpy, [entity1, entity2, entity3], [entity1, entity2, entity3, entity4], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + }); + }); + + it('should return correct number of items with auto-pagination', (done) => { + const onNextSpy = sinon.spy(); + deltaStoreToTest.find(new Kinvey.Query(), { autoPagination: true }) + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.save(entity3) + .then(() => deltaStoreToTest.find(new Kinvey.Query(), { autoPagination: true }) + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [entity1, entity2, entity3], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + }); + }); + + it('should return correct number of items with deleted item', (done) => { + const onNextSpy = sinon.spy(); + deltaStoreToTest.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.removeById(entity1._id) + .then(() => deltaStoreToTest.find() + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [entity2], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + }); + }); + + it('should return correct number of items with updated item', (done) => { + let updatedEntity = _.clone(entity2); + updatedEntity.textField = utilities.randomString(); + const onNextSpy = sinon.spy(); + deltaStoreToTest.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.save(updatedEntity) + .then(() => deltaStoreToTest.find() + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [entity1, updatedEntity], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + }); + }); + + it('should return correct number of items with updated and deleted item', (done) => { + let updatedEntity = _.clone(entity2); + updatedEntity.textField = utilities.randomString(); + const onNextSpy = sinon.spy(); + deltaStoreToTest.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity1], [entity1, entity2], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.save(updatedEntity) + .then(() => deltaNetworkStore.removeById(entity1._id)) + .then(() => deltaStoreToTest.find() + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity1, entity2], [updatedEntity], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + }); + }); + + it('should return correct number of items with query with updated item', (done) => { + let entity4 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let entity5 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let entity6 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let updatedEntity = _.clone(entity5); + updatedEntity.numberField = 5; + let query = new Kinvey.Query(); + query.equalTo('textField', 'queryValue'); + const onNextSpy = sinon.spy(); + deltaNetworkStore.save(entity4) + .then(() => deltaStoreToTest.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.find(query) + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity5], [entity4, entity5, entity6], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.save(updatedEntity) + .then(() => deltaStoreToTest.find(query) + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity4, entity5, entity6], [entity4, updatedEntity, entity6], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + })); + }); + + it('should return correct number of items with query with deleted item', (done) => { + let entity4 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let entity5 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let entity6 = utilities.getEntity(utilities.randomString(), 'queryValue'); + let updatedEntity = _.clone(entity5); + updatedEntity.numberField = 5; + let query = new Kinvey.Query(); + query.equalTo('textField', 'queryValue'); + const onNextSpy = sinon.spy(); + deltaNetworkStore.save(entity4) + .then(() => deltaStoreToTest.save(entity5)) + .then(() => deltaNetworkStore.save(entity6)) + .then(() => deltaStoreToTest.find(query) + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, onNextSpy, [entity5], [entity4, entity5, entity6], true); + const anotherSpy = sinon.spy(); + deltaNetworkStore.removeById(entity5._id) + .then(() => deltaStoreToTest.find(query) + .subscribe(anotherSpy, done, () => { + try { + utilities.validateReadResult(currentDataStoreType, anotherSpy, [entity4, entity5, entity6], [entity4, entity6], true); + done(); + } + catch (error) { + done(error); + } + })) + } + catch (error) { + done(error); + } + })); + }); + }); + + describe('when switching stores', () => { + const dataStoreType = currentDataStoreType; + const entity1 = utilities.getEntity(utilities.randomString()); + const entity2 = utilities.getEntity(utilities.randomString()); + const entity3 = utilities.getEntity(utilities.randomString()); + const entity4 = utilities.getEntity(utilities.randomString()); + const createdUserIds = []; + let deltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true }); + + before((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => Kinvey.User.signup()) + .then((user) => { + createdUserIds.push(user.data._id); + done(); + }) + .catch(done); + }); + + beforeEach((done) => { + utilities.cleanUpCollectionData(deltaCollectionName) + .then(() => deltaNetworkStore.save(entity1)) + .then(() => deltaNetworkStore.save(entity2)) + .then(() => done()) + .catch(done); + }); + + after((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => done()) + .catch(done); + }); + + if (currentDataStoreType === Kinvey.DataStoreType.Sync) { + it('should use deltaset consistently when switching from sync to cache', (done) => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [entity3], [])) + .then(() => deltaNetworkStore.save(entity4)) + .then(() => deltaCacheStore.pull()) + .then((result) => validateNewPullOperation(result, [entity4], [])) + .then(() => done()) + .catch(done); + }); + } + + if (currentDataStoreType === Kinvey.DataStoreType.Cache) { + it('should use deltaset consistently when switching from cache to sync', (done) => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [entity3], [])) + .then(() => deltaNetworkStore.save(entity4)) + .then(() => deltaSyncStore.pull()) + .then((result) => validateNewPullOperation(result, [entity4], [])) + .then(() => done()) + .catch(done); + }); + } + + if (currentDataStoreType === Kinvey.DataStoreType.Sync) { + it('should use deltaset consistently when switching from network to sync', (done) => { + let onNextSpy = sinon.spy(); + deltaNetworkStore.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(Kinvey.DataStoreType.Network, onNextSpy, [entity1], [entity1, entity2], true); + deltaNetworkStore.save(entity3) + .then(() => deltaStoreToTest.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2, entity3])) + .then(() => deltaNetworkStore.save(entity4)) + .then(() => deltaSyncStore.pull()) + .then((result) => validateNewPullOperation(result, [entity4], [])) + .then(() => done()) + .catch(done); + } + catch (error) { + done(error); + } + }); + }) + } + + if (currentDataStoreType === Kinvey.DataStoreType.Cache) { + it('should use deltaset consistently when switching from network to cache', (done) => { + let onNextSpy = sinon.spy(); + deltaNetworkStore.find() + .subscribe(onNextSpy, done, () => { + try { + utilities.validateReadResult(Kinvey.DataStoreType.Network, onNextSpy, [entity1], [entity1, entity2], true); + deltaNetworkStore.save(entity3) + .then(() => deltaStoreToTest.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2, entity3])) + .then(() => deltaNetworkStore.save(entity4)) + .then(() => deltaSyncStore.pull()) + .then((result) => validateNewPullOperation(result, [entity4], [])) + .then(() => done()) + .catch(done); + } + catch (error) { + done(error); + } + }); + }) + } + }); + + describe('when clearing cache', () => { + const dataStoreType = currentDataStoreType; + const entity1 = utilities.getEntity(utilities.randomString()); + const entity2 = utilities.getEntity(utilities.randomString()); + const entity3 = utilities.getEntity(utilities.randomString()); + const entity4 = utilities.getEntity(utilities.randomString()); + const createdUserIds = []; + let deltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true }); + + before((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => Kinvey.User.signup()) + .then((user) => { + createdUserIds.push(user.data._id); + done(); + }) + .catch(done); + }); + + beforeEach((done) => { + utilities.cleanUpCollectionData(deltaCollectionName) + .then(() => deltaNetworkStore.save(entity1)) + .then(() => deltaNetworkStore.save(entity2)) + .then(() => done()) + .catch(done); + }); + + after((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => done()) + .catch(done); + }); + + it('should send regular GET after clearCache()', (done) => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.save(entity3)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [entity3], [])) + .then(() => Kinvey.DataStore.clearCache()) + .then(() => deltaNetworkStore.save(entity4)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2, entity3, entity4])) + .then(() => deltaNetworkStore.removeById(entity3._id)) + .then(() => deltaStoreToTest.pull()) + .then((result) => validateNewPullOperation(result, [], [entity3])) + .then(() => done()) + .catch((error) => done(error)); + }); + }); + + describe('error handling', function () { + const dataStoreType = currentDataStoreType; + const entity1 = utilities.getEntity(utilities.randomString()); + const entity2 = utilities.getEntity(utilities.randomString()); + const entity3 = utilities.getEntity(utilities.randomString()); + const entity4 = utilities.getEntity(utilities.randomString()); + const createdUserIds = []; + let deltaStoreToTest = Kinvey.DataStore.collection(deltaCollectionName, currentDataStoreType, { useDeltaSet: true }); + let nonDeltaStoreToTest = Kinvey.DataStore.collection(collectionWithoutDelta, currentDataStoreType, { useDeltaSet: true }); + let nonDeltaNetworkStore = Kinvey.DataStore.collection(collectionWithoutDelta, Kinvey.DataStoreType.Network); + + before((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => utilities.cleanUpAppData(collectionWithoutDelta, createdUserIds)) + .then(() => Kinvey.User.signup()) + .then((user) => { + createdUserIds.push(user.data._id); + done(); + }) + .catch(done); + }); + + beforeEach((done) => { + utilities.cleanUpCollectionData(deltaCollectionName) + .then(() => utilities.cleanUpCollectionData(collectionWithoutDelta)) + .then(() => utilities.cleanUpCollectionData(deltaCollectionName)) + .then(() => nonDeltaNetworkStore.save(entity1)) + .then(() => nonDeltaNetworkStore.save(entity2)) + .then(() => deltaNetworkStore.save(entity1)) + .then(() => deltaNetworkStore.save(entity2)) + .then(() => done()) + .catch(done); + }); + + after((done) => { + utilities.cleanUpAppData(deltaCollectionName, createdUserIds) + .then(() => done()) + .catch(done); + }); + + it('should send regular GET after failure for missing configuration', (done) => { + nonDeltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2], null, null, collectionWithoutDelta)) + .then(() => nonDeltaNetworkStore.save(entity3)) + .then(() => nonDeltaStoreToTest.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2, entity3], null, null, collectionWithoutDelta)) + .then(() => nonDeltaNetworkStore.save(entity4)) + .then(() => nonDeltaStoreToTest.pull()) + .then((result) => validatePullOperation(result, [entity1, entity2, entity3, entity4], null, null, collectionWithoutDelta)) + .then(() => done()) + .catch(done); + }); + + if (runner.isDesktopApp()) { + it('should send regular GET after fail for outdated since param', function (done) { + let db = window.openDatabase(externalConfig.appKey, 1, 'Kinvey Cache', 20000); + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => { + db.transaction((tx) => { + try { + tx.executeSql(`SELECT * FROM _QueryCache WHERE value LIKE '%"query":""%'`, [], (tx1, resultSet) => { + try { + let item = resultSet.rows[0]; + let queryParsed = JSON.parse(item.value); + let lastRequest = queryParsed.lastRequest; + let lastRequestDateObject = new Date(lastRequest); + lastRequestDateObject.setDate(lastRequestDateObject.getDate() - 31); + let outdatedTimeToString = lastRequestDateObject.toISOString(); + queryParsed.lastRequest = outdatedTimeToString; + tx.executeSql(`UPDATE _QueryCache SET value = ? WHERE value LIKE '%"query":""%'`, [JSON.stringify(queryParsed)], () => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => done()) + .catch((error) => done(error)); + }); + } + catch (error) { + done(error); + } + }); + } + catch (error) { + done(error); + } + }) + }) + .catch((error) => done(error)); + }); + + it('with outdated since param subsequent pull should delete items in the cache', (done) => { + let db = window.openDatabase(externalConfig.appKey, 1, 'Kinvey Cache', 20000); + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity1, entity2])) + .then(() => deltaNetworkStore.removeById(entity1._id)) + .then(() => { + db.transaction((tx) => { + try { + tx.executeSql(`SELECT * FROM _QueryCache WHERE value LIKE '%"query":""%'`, [], (tx1, resultSet) => { + try { + let item = resultSet.rows[0]; + let queryParsed = JSON.parse(item.value); + let lastRequest = queryParsed.lastRequest; + let lastRequestDateObject = new Date(lastRequest); + lastRequestDateObject.setDate(lastRequestDateObject.getDate() - 31); + let outdatedTimeToString = lastRequestDateObject.toISOString(); + queryParsed.lastRequest = outdatedTimeToString; + tx.executeSql(`UPDATE _QueryCache SET value = ? WHERE value LIKE '%"query":""%'`, [JSON.stringify(queryParsed)], () => { + deltaStoreToTest.pull() + .then((result) => validatePullOperation(result, [entity2])) + .then(() => done()) + .catch((error) => done(error)); + }); + } + catch (error) { + done(error); + } + }); + } + catch (error) { + done(error); + } + }) + }) + .catch((error) => done(error)); + }); + } + + }); + }); + }); +} + +runner.run(testFunc); \ No newline at end of file diff --git a/test/integration/tests/files-common.tests.js b/test/integration/tests/files-common.tests.js index 8ca7b28ae..8f256105f 100644 --- a/test/integration/tests/files-common.tests.js +++ b/test/integration/tests/files-common.tests.js @@ -1,7 +1,7 @@ function testFunc() { const notFoundErrorName = 'NotFoundError'; - const notFoundErrorMessage = 'This blob not found for this app backend'; + const notFoundErrorMessage = 'This blob not found for this app backend.'; const timeoutErrorName = 'TimeoutError'; const timeoutErrorMessage = 'The network request timed out.'; const plainTextMimeType = 'text/plain'; diff --git a/test/integration/tests/users/users.tests.js b/test/integration/tests/users/users.tests.js index 9464f6153..89faf161e 100644 --- a/test/integration/tests/users/users.tests.js +++ b/test/integration/tests/users/users.tests.js @@ -446,7 +446,7 @@ function testFunc() { it('should return the error from the server if the id does not exist', (done) => { Kinvey.User.remove(utilities.randomString()) .catch((error) => { - expect(error.message).to.equal('This user does not exist for this app backend'); + expect(error.message).to.equal('This user does not exist for this app backend.'); done(); }) .catch(done);