From 36c4452c0cbf1e975cb041cef8a88e8d9df4b595 Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Wed, 3 Feb 2016 14:56:48 -0800 Subject: [PATCH] [core] [WIP] Interface and implementation for offline --- gyp/platform-android.gypi | 5 + gyp/platform-ios.gypi | 5 + gyp/platform-linux.gypi | 5 + gyp/platform-osx.gypi | 5 + include/mbgl/storage/default_file_source.hpp | 65 +++++ include/mbgl/storage/offline.hpp | 173 ++++++++++++ platform/default/default_file_source.cpp | 83 +++++- platform/default/mbgl/storage/offline.cpp | 122 +++++++++ .../default/mbgl/storage/offline_database.cpp | 89 ++++++ .../default/mbgl/storage/offline_database.hpp | 15 + .../default/mbgl/storage/offline_download.cpp | 236 ++++++++++++++++ .../default/mbgl/storage/offline_download.hpp | 72 +++++ platform/default/sqlite3.cpp | 18 ++ platform/default/sqlite3.hpp | 8 +- src/mbgl/map/source.hpp | 2 + test/fixtures/offline/empty.style.json | 5 + test/fixtures/offline/geojson.json | 4 + .../offline/geojson_source.style.json | 10 + .../fixtures/offline/inline_source.style.json | 17 ++ test/storage/offline.cpp | 73 +++++ test/storage/offline_database.cpp | 59 ++++ test/storage/offline_download.cpp | 257 ++++++++++++++++++ test/test.gypi | 2 + 23 files changed, 1328 insertions(+), 2 deletions(-) create mode 100644 include/mbgl/storage/offline.hpp create mode 100644 platform/default/mbgl/storage/offline.cpp create mode 100644 platform/default/mbgl/storage/offline_download.cpp create mode 100644 platform/default/mbgl/storage/offline_download.hpp create mode 100644 test/fixtures/offline/empty.style.json create mode 100644 test/fixtures/offline/geojson.json create mode 100644 test/fixtures/offline/geojson_source.style.json create mode 100644 test/fixtures/offline/inline_source.style.json create mode 100644 test/storage/offline.cpp create mode 100644 test/storage/offline_download.cpp diff --git a/gyp/platform-android.gypi b/gyp/platform-android.gypi index 2c1e06a7156..91962ed8568 100644 --- a/gyp/platform-android.gypi +++ b/gyp/platform-android.gypi @@ -21,8 +21,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', ], @@ -35,6 +39,7 @@ '<@(nunicode_cflags)', '<@(boost_cflags)', '<@(sqlite_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(libpng_ldflags)', diff --git a/gyp/platform-ios.gypi b/gyp/platform-ios.gypi index 806f103a17e..b70abd3a13b 100644 --- a/gyp/platform-ios.gypi +++ b/gyp/platform-ios.gypi @@ -16,8 +16,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', '../platform/darwin/log_nslog.mm', @@ -72,6 +76,7 @@ '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(zlib_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(sqlite_ldflags)', diff --git a/gyp/platform-linux.gypi b/gyp/platform-linux.gypi index 2eafe3e821d..0fae25d4b89 100644 --- a/gyp/platform-linux.gypi +++ b/gyp/platform-linux.gypi @@ -23,8 +23,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', ], @@ -38,6 +42,7 @@ '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(webp_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(libpng_ldflags)', diff --git a/gyp/platform-osx.gypi b/gyp/platform-osx.gypi index 53e7047492c..970fd1b5f65 100644 --- a/gyp/platform-osx.gypi +++ b/gyp/platform-osx.gypi @@ -15,8 +15,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', '../platform/darwin/log_nslog.mm', @@ -63,6 +67,7 @@ '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(zlib_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(zlib_ldflags)', diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp index d8fc4b98a34..4d4b1670e76 100644 --- a/include/mbgl/storage/default_file_source.hpp +++ b/include/mbgl/storage/default_file_source.hpp @@ -2,6 +2,9 @@ #define MBGL_STORAGE_DEFAULT_FILE_SOURCE #include +#include + +#include namespace mbgl { @@ -22,6 +25,68 @@ class DefaultFileSource : public FileSource { std::unique_ptr request(const Resource&, Callback) override; + /* + * Retrieve all regions in the offline database. + * + * The query will be executed asynchronously and the results passed to the given + * callback, which will be executed on the database thread; it is the responsibility + * of the SDK bindings to re-execute a user-provided callback on the main thread. + */ + void listOfflineRegions(std::function>)>); + + /* + * Create an offline region in the database. + * + * When the initial database queries have completed, the provided callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + * + * Note that the resulting region will be in an inactive download state; to begin + * downloading resources, call `setOfflineRegionDownloadState(OfflineRegionDownloadState::Active)`, + * optionally registering an `OfflineRegionObserver` beforehand. + */ + void createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function)>); + + /* + * Register an observer to be notified when the state of the region changes. + */ + void setOfflineRegionObserver(OfflineRegion&, std::unique_ptr); + + /* + * Pause or resume downloading of regional resources. + */ + void setOfflineRegionDownloadState(OfflineRegion&, OfflineRegionDownloadState); + + /* + * Retrieve the current status of the region. The query will be executed + * asynchronously and the results passed to the given callback, which will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + void getOfflineRegionStatus(OfflineRegion&, std::function)>) const; + + /* + * Initiate the removal of offline region from the database. All resources required by + * the region, but not also required by other regions, will be deleted. + * + * If an observer is registered for the region, it will receive status notifications + * as the deletion progresses. + * + * Note that this method takes ownership of the input, reflecting the fact that once + * region deletion is initiated, it is not legal to perform further actions with the + * region. + * + * When the operation is complete or encounters an error, the given callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + void deleteOfflineRegion(OfflineRegion&&, std::function); + // For testing only. void put(const Resource&, const Response&); void goOffline(); diff --git a/include/mbgl/storage/offline.hpp b/include/mbgl/storage/offline.hpp new file mode 100644 index 00000000000..7581ecd03d5 --- /dev/null +++ b/include/mbgl/storage/offline.hpp @@ -0,0 +1,173 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace mbgl { + +class TileID; +class SourceInfo; + +/* + * An offline region defined by a style URL, geographic bounding box, zoom range, and + * device pixel ratio. + * + * Both minZoom and maxZoom must be ≥ 0, and maxZoom must be ≥ minZoom. + * + * maxZoom may be ∞, in which case for each tile source, the region will include + * tiles from minZoom up to the maximum zoom level provided by that source. + * + * pixelRatio must be ≥ 0 and should typically be 1.0 or 2.0. + */ +class OfflineTilePyramidRegionDefinition { +public: + OfflineTilePyramidRegionDefinition(const std::string&, const LatLngBounds&, double, double, float); + + /* Private */ + std::vector tileCover(SourceType, uint16_t tileSize, const SourceInfo&) const; + + const std::string styleURL; + const LatLngBounds bounds; + const double minZoom; + const double maxZoom; + const float pixelRatio; +}; + +/* + * For the present, a tile pyramid is the only type of offline region. In the future, + * other definition types will be available and this will be a variant type. + */ +using OfflineRegionDefinition = OfflineTilePyramidRegionDefinition; + +/* + * The encoded format is private. + */ +std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition&); +OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string&); + +/* + * Arbitrary binary region metadata. The contents are opaque to the mbgl implementation; + * it just stores and retrieves a BLOB. SDK bindings should leave the interpretation of + * this data up to the application; they _should not_ enforce a higher-level data format. + * In the future we want offline database to be portable across target platforms, and a + * platform-specific metadata format would prevent that. + */ +using OfflineRegionMetadata = std::vector; + +/* + * A region is either inactive (not downloading, but previously-downloaded + * resources are available for use), or active (resources are being downloaded + * or will be downloaded, if necessary, when network access is available). + * + * This state is independent of whether or not the complete set of resources + * is currently available for offline use. To check if that is the case, use + * `OfflineRegionStatus::complete()`. + */ +enum class OfflineRegionDownloadState { + Inactive, + Active +}; + +/* + * A region's status includes its active/inactive state as well as counts + * of the number of resources that have completed downloading, their total + * size in bytes, and the total number of resources that are required. + * + * Note that the total required size in bytes is not currently available. A + * future API release may provide an estimate of this number. + */ +class OfflineRegionStatus { +public: + OfflineRegionDownloadState downloadState = OfflineRegionDownloadState::Inactive; + + /** + * The number of resources that have been fully downloaded and are ready for + * offline access. + */ + uint64_t completedResourceCount = 0; + + /** + * The cumulative size, in bytes, of all resources that have been fully downloaded. + */ + uint64_t completedResourceSize = 0; + + /** + * The number of resources that are known to be required for this region. Note + * that during early phases of the download, this number is merely a _lower bound_ + * and does not necessarily reflect the complete number of required resources. + * This is due to the fact that certain resources must themselves be downloaded + * before the total number can be calculated. For example, the style must be + * downloaded before any other resources can be determined. + */ + uint64_t requiredResourceCount = 0; + + bool complete() const { + return completedResourceCount == requiredResourceCount; + } +}; + +/* + * A region can have a single observer, which gets notified whenever a change + * to the region's status occurs. + */ +class OfflineRegionObserver { +public: + virtual ~OfflineRegionObserver() = default; + + /* + * Implement this method to be notified of a change in the status of an + * offline region. Status changes include any change in state of the members + * of OfflineRegionStatus. + * + * Note that this method will be executed on the database thread; it is the + * responsibility of the SDK bindings to wrap this object in an interface that + * re-executes the user-provided implementation on the main thread. + */ + virtual void statusChanged(OfflineRegionStatus) {}; + + /* + * Implement this method to be notified of errors encountered while downloading + * regional resources. Such errors may be recoverable; for example the implementation + * will attempt to re-request failed resources based on an exponential backoff + * algorithm, or when it detects that network access has been restored. + * + * Note that this method will be executed on the database thread; it is the + * responsibility of the SDK bindings to wrap this object in an interface that + * re-executes the user-provided implementation on the main thread. + */ + virtual void error(std::exception_ptr) {}; +}; + +class OfflineRegion { +public: + // Move-only; not publicly constructible. + OfflineRegion(OfflineRegion&&); + OfflineRegion& operator=(OfflineRegion&&); + ~OfflineRegion(); + + OfflineRegion() = delete; + OfflineRegion(const OfflineRegion&) = delete; + OfflineRegion& operator=(const OfflineRegion&) = delete; + + int64_t getID() const; + const OfflineRegionDefinition& getDefinition() const; + const OfflineRegionMetadata& getMetadata() const; + +private: + friend class OfflineDatabase; + + OfflineRegion(int64_t id, + const OfflineRegionDefinition&, + const OfflineRegionMetadata&); + + const int64_t id; + const OfflineRegionDefinition definition; + const OfflineRegionMetadata metadata; +}; + +} // namespace mbgl diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp index efe893d49b7..f094e3a0ed3 100644 --- a/platform/default/default_file_source.cpp +++ b/platform/default/default_file_source.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,49 @@ class DefaultFileSource::Impl { return onlineFileSource.getAccessToken(); } + void listRegions(std::function>)> callback) { + try { + callback({}, offlineDatabase.listRegions()); + } catch (...) { + callback(std::current_exception(), {}); + } + } + + void createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function)> callback) { + try { + callback({}, offlineDatabase.createRegion(definition, metadata)); + } catch (...) { + callback(std::current_exception(), {}); + } + } + + void getRegionStatus(int64_t regionID, std::function)> callback) { + try { + callback({}, getDownload(regionID).getStatus()); + } catch (...) { + callback(std::current_exception(), {}); + } + } + + void deleteRegion(OfflineRegion&& region, std::function callback) { + try { + offlineDatabase.deleteRegion(std::move(region)); + callback({}); + } catch (...) { + callback(std::current_exception()); + } + } + + void setRegionObserver(int64_t regionID, std::unique_ptr observer) { + getDownload(regionID).setObserver(std::move(observer)); + } + + void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { + getDownload(regionID).setState(state); + } + void add(FileRequest* req, Resource resource, Callback callback) { tasks[req] = std::make_unique(resource, callback, this); } @@ -77,15 +121,26 @@ class DefaultFileSource::Impl { offline = true; } +private: + OfflineDownload& getDownload(int64_t regionID) { + auto it = downloads.find(regionID); + if (it != downloads.end()) { + return *it->second; + } + return *downloads.emplace(regionID, + std::make_unique(regionID, offlineDatabase.getRegionDefinition(regionID), offlineDatabase, onlineFileSource)).first->second; + } + OfflineDatabase offlineDatabase; OnlineFileSource onlineFileSource; std::unordered_map> tasks; + std::unordered_map> downloads; bool offline = false; }; class DefaultFileRequest : public FileRequest { public: - DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread& thread_) + DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread& thread_) : thread(thread_), workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) { } @@ -129,6 +184,32 @@ std::unique_ptr DefaultFileSource::request(const Resource& resource } } +void DefaultFileSource::listOfflineRegions(std::function>)> callback) { + thread->invoke(&Impl::listRegions, callback); +} + +void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function)> callback) { + thread->invoke(&Impl::createRegion, definition, metadata, callback); +} + +void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function callback) { + thread->invoke(&Impl::deleteRegion, std::move(region), callback); +} + +void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr observer) { + thread->invoke(&Impl::setRegionObserver, region.getID(), std::move(observer)); +} + +void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { + thread->invoke(&Impl::setRegionDownloadState, region.getID(), state); +} + +void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function)> callback) const { + thread->invoke(&Impl::getRegionStatus, region.getID(), callback); +} + void DefaultFileSource::put(const Resource& resource, const Response& response) { thread->invokeSync(&Impl::put, resource, response); } diff --git a/platform/default/mbgl/storage/offline.cpp b/platform/default/mbgl/storage/offline.cpp new file mode 100644 index 00000000000..7311474bcfd --- /dev/null +++ b/platform/default/mbgl/storage/offline.cpp @@ -0,0 +1,122 @@ +#include +#include +#include + +#include +#include +#include + +#include + +namespace mbgl { + +OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition( + const std::string& styleURL_, const LatLngBounds& bounds_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(styleURL_), + bounds(bounds_), + minZoom(minZoom_), + maxZoom(maxZoom_), + pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); + } +} + +std::vector OfflineTilePyramidRegionDefinition::tileCover(SourceType type, uint16_t tileSize, const SourceInfo& info) const { + double minZ = std::max(coveringZoomLevel(minZoom, type, tileSize), info.minZoom); + double maxZ = std::min(coveringZoomLevel(maxZoom, type, tileSize), info.maxZoom); + + assert(minZ >= 0); + assert(maxZ >= 0); + assert(minZ < std::numeric_limits::max()); + assert(maxZ < std::numeric_limits::max()); + + std::vector result; + + for (uint8_t z = minZ; z <= maxZ; z++) { + for (const auto& tile : mbgl::tileCover(bounds, z, z)) { + result.push_back(tile.normalized()); + } + } + + return result; +} + +OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) { + rapidjson::GenericDocument, rapidjson::CrtAllocator> doc; + doc.Parse<0>(region.c_str()); + + if (doc.HasParseError() || + !doc.HasMember("style_url") || !doc["style_url"].IsString() || + !doc.HasMember("bounds") || !doc["bounds"].IsArray() || doc["bounds"].Size() != 4 || + !doc["bounds"][0].IsDouble() || !doc["bounds"][1].IsDouble() || + !doc["bounds"][2].IsDouble() || !doc["bounds"][3].IsDouble() || + !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble() || + (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble()) || + !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) { + throw std::runtime_error("Malformed offline region definition"); + } + + std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() }; + LatLngBounds bounds = LatLngBounds::hull( + LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()), + LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())); + double minZoom = doc["min_zoom"].GetDouble(); + double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY; + float pixelRatio = doc["pixel_ratio"].GetDouble(); + + return { styleURL, bounds, minZoom, maxZoom, pixelRatio }; +} + +std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) { + rapidjson::GenericDocument, rapidjson::CrtAllocator> doc; + doc.SetObject(); + + doc.AddMember("style_url", rapidjson::StringRef(region.styleURL.data(), region.styleURL.length()), doc.GetAllocator()); + + rapidjson::GenericValue, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType); + bounds.PushBack(region.bounds.south(), doc.GetAllocator()); + bounds.PushBack(region.bounds.west(), doc.GetAllocator()); + bounds.PushBack(region.bounds.north(), doc.GetAllocator()); + bounds.PushBack(region.bounds.east(), doc.GetAllocator()); + doc.AddMember("bounds", bounds, doc.GetAllocator()); + + doc.AddMember("min_zoom", region.minZoom, doc.GetAllocator()); + if (std::isfinite(region.maxZoom)) { + doc.AddMember("max_zoom", region.maxZoom, doc.GetAllocator()); + } + + doc.AddMember("pixel_ratio", region.pixelRatio, doc.GetAllocator()); + + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + + return buffer.GetString(); +} + +OfflineRegion::OfflineRegion(int64_t id_, + const OfflineRegionDefinition& definition_, + const OfflineRegionMetadata& metadata_) + : id(id_), + definition(definition_), + metadata(metadata_) { +} + +OfflineRegion::OfflineRegion(OfflineRegion&&) = default; +OfflineRegion::~OfflineRegion() = default; + +const OfflineRegionDefinition& OfflineRegion::getDefinition() const { + return definition; +} + +const OfflineRegionMetadata& OfflineRegion::getMetadata() const { + return metadata; +} + +int64_t OfflineRegion::getID() const { + return id; +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp index cd0f88b7fe9..8831d265f4b 100644 --- a/platform/default/mbgl/storage/offline_database.cpp +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -294,4 +294,93 @@ void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& re } } +std::vector OfflineDatabase::listRegions() { + mapbox::sqlite::Statement& stmt = getStatement( + "SELECT id, definition, description FROM regions"); + + std::vector result; + + while (stmt.run()) { + result.push_back(OfflineRegion( + stmt.get(0), + decodeOfflineRegionDefinition(stmt.get(1)), + stmt.get>(2))); + } + + return std::move(result); +} + +OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata) { + mapbox::sqlite::Statement& stmt = getStatement( + "INSERT INTO regions (definition, description) " + "VALUES (?1, ?2) "); + + stmt.bind(1, encodeOfflineRegionDefinition(definition)); + stmt.bind(2, metadata); + stmt.run(); + + return OfflineRegion(db->lastInsertRowid(), definition, metadata); +} + +void OfflineDatabase::deleteRegion(OfflineRegion&& region) { + mapbox::sqlite::Statement& stmt = getStatement( + "DELETE FROM regions WHERE id = ?"); + + stmt.bind(1, region.getID()); + stmt.run(); +} + +optional OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) { + auto response = get(resource); + + if (response) { + markUsed(regionID, resource); + } + + return response; +} + +void OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) { + put(resource, response); + markUsed(regionID, resource); +} + +void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + mapbox::sqlite::Statement& stmt1 = getStatement( + "REPLACE INTO region_tiles (region_id, tileset_id, x, y, z) " + "SELECT ?1, tilesets.id, ?4, ?5, ?6 " + "FROM tilesets " + "WHERE url_template = ?2 " + "AND pixel_ratio = ?3 "); + + stmt1.bind(1, regionID); + stmt1.bind(2, (*resource.tileData).urlTemplate); + stmt1.bind(3, (*resource.tileData).pixelRatio); + stmt1.bind(4, (*resource.tileData).x); + stmt1.bind(5, (*resource.tileData).y); + stmt1.bind(6, (*resource.tileData).z); + stmt1.run(); + } else { + mapbox::sqlite::Statement& stmt1 = getStatement( + "REPLACE INTO region_resources (region_id, resource_url) " + "VALUES (?1, ?2) "); + + stmt1.bind(1, regionID); + stmt1.bind(2, resource.url); + stmt1.run(); + } +} + +OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) { + mapbox::sqlite::Statement& stmt = getStatement( + "SELECT definition FROM regions WHERE id = ?1"); + + stmt.bind(1, regionID); + stmt.run(); + + return decodeOfflineRegionDefinition(stmt.get(1)); +} + } // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp index bc6f784d50e..c31b1e5c2cf 100644 --- a/platform/default/mbgl/storage/offline_database.hpp +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -2,6 +2,7 @@ #define MBGL_OFFLINE_DATABASE #include +#include #include #include @@ -29,6 +30,18 @@ class OfflineDatabase : private util::noncopyable { optional get(const Resource&); void put(const Resource&, const Response&); + std::vector listRegions(); + + OfflineRegion createRegion(const OfflineRegionDefinition&, + const OfflineRegionMetadata&); + + void deleteRegion(OfflineRegion&&); + + optional getRegionResource(int64_t regionID, const Resource&); + void putRegionResource(int64_t regionID, const Resource&, const Response&); + + OfflineRegionDefinition getRegionDefinition(int64_t regionID); + private: void ensureSchema(); void removeExisting(); @@ -40,6 +53,8 @@ class OfflineDatabase : private util::noncopyable { optional getResource(const Resource&); void putResource(const Resource&, const Response&); + void markUsed(int64_t regionID, const Resource&); + const std::string path; std::unique_ptr<::mapbox::sqlite::Database> db; std::unordered_map> statements; diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp new file mode 100644 index 00000000000..31e51db5dac --- /dev/null +++ b/platform/default/mbgl/storage/offline_download.cpp @@ -0,0 +1,236 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace mbgl { + +OfflineDownload::OfflineDownload(int64_t id_, + OfflineRegionDefinition&& definition_, + OfflineDatabase& offlineDatabase_, + FileSource& onlineFileSource_) + : id(id_), + definition(definition_), + offlineDatabase(offlineDatabase_), + onlineFileSource(onlineFileSource_) { +} + +OfflineDownload::~OfflineDownload() = default; + +void OfflineDownload::setObserver(std::unique_ptr observer_) { + observer = std::move(observer_); +} + +void OfflineDownload::setState(OfflineRegionDownloadState state) { + if (status.downloadState == state) { + return; + } + + status.downloadState = state; + + if (status.downloadState == OfflineRegionDownloadState::Active) { + activateDownload(); + } else { + deactivateDownload(); + } +} + +std::vector OfflineDownload::spriteResources(const StyleParser& parser) const { + std::vector result; + + if (!parser.spriteURL.empty()) { + result.push_back(Resource::spriteImage(parser.spriteURL, definition.pixelRatio)); + result.push_back(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio)); + } + + return result; +} + +std::vector OfflineDownload::glyphResources(const StyleParser& parser) const { + std::vector result; + + if (!parser.glyphURL.empty()) { + for (const auto& fontStack : parser.fontStacks()) { + for (uint32_t i = 0; i < 256; i++) { + result.push_back(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * 256))); + } + } + } + + return result; +} + +std::vector OfflineDownload::tileResources(SourceType type, uint16_t tileSize, const SourceInfo& info) const { + std::vector result; + + for (const auto& tile : definition.tileCover(type, tileSize, info)) { + result.push_back(Resource::tile(info.tiles[0], definition.pixelRatio, tile.x, tile.y, tile.z)); + } + + return result; +} + +OfflineRegionStatus OfflineDownload::getStatus() const { + OfflineRegionStatus result; + result.requiredResourceCount++; + + optional styleResponse = offlineDatabase.get(Resource::style(definition.styleURL)); + if (!styleResponse) { + return result; + } + + result.completedResourceCount += 1; + result.completedResourceSize += styleResponse->data->length(); + + StyleParser parser; + parser.parse(*styleResponse->data); + + result.requiredResourceCount += spriteResources(parser).size(); + result.requiredResourceCount += glyphResources(parser).size(); + + for (const auto& source : parser.sources) { + switch (source->type) { + case SourceType::Vector: + case SourceType::Raster: + if (source->getInfo()) { + result.requiredResourceCount += tileResources(source->type, source->tileSize, *source->getInfo()).size(); + } else { + result.requiredResourceCount += 1; + optional sourceResponse = offlineDatabase.get(Resource::source(source->url)); + if (sourceResponse) { + result.completedResourceCount += 1; + result.completedResourceSize += sourceResponse->data->length(); + result.requiredResourceCount += tileResources(source->type, source->tileSize, + *StyleParser::parseTileJSON(*sourceResponse->data, source->url, source->type)).size(); + } + } + break; + + case SourceType::GeoJSON: + if (!source->url.empty()) { + result.requiredResourceCount += 1; + } + break; + + case SourceType::Video: + case SourceType::Annotations: + break; + } + } + + return result; +} + +void OfflineDownload::activateDownload() { + ensureResource(Resource::style(definition.styleURL), [&] (Response styleResponse) { + StyleParser parser; + parser.parse(*styleResponse.data); + + for (const auto& resource : spriteResources(parser)) { + ensureResource(resource); + } + + for (const auto& resource : glyphResources(parser)) { + ensureResource(resource); + } + + for (const auto& source : parser.sources) { + switch (source->type) { + case SourceType::Vector: + case SourceType::Raster: + ensureTileJSON(*source, [=] (const std::vector& tileResources_) { + for (const auto& resource : tileResources_) { + ensureResource(resource); + } + }); + break; + + case SourceType::GeoJSON: + if (!source->url.empty()) { + ensureResource(Resource::source(source->url)); + } + break; + + case SourceType::Video: + case SourceType::Annotations: + break; + } + } + }); +} + +void OfflineDownload::deactivateDownload() { + requests.clear(); +} + +void OfflineDownload::ensureTileJSON(const Source& source, std::function&)> callback) { + SourceType type = source.type; + uint16_t tileSize = source.tileSize; + std::string url = source.url; + + if (source.getInfo()) { + callback(tileResources(type, tileSize, *source.getInfo())); + } else { + ensureResource(Resource::source(url), [=] (Response sourceResponse) { + callback(tileResources(type, tileSize, *StyleParser::parseTileJSON(*sourceResponse.data, url, type))); + }); + } +} + +void OfflineDownload::ensureResource(const Resource& resource, std::function callback) { + status.requiredResourceCount++; + notifyObserver(); + + optional offlineResponse = offlineDatabase.getRegionResource(id, resource); + + if (offlineResponse) { + if (callback) { + callback(*offlineResponse); + } + + status.completedResourceCount++; + if (offlineResponse->data) { + status.completedResourceSize += offlineResponse->data->length(); + } + + notifyObserver(); + + return; + } + + auto it = requests.insert(requests.begin(), nullptr); + *it = onlineFileSource.request(resource, [=] (Response onlineResponse) { + requests.erase(it); + + if (onlineResponse.error) { + return; + } + + offlineDatabase.putRegionResource(id, resource, onlineResponse); + if (callback) { + callback(onlineResponse); + } + + status.completedResourceCount++; + if (onlineResponse.data) { + status.completedResourceSize += onlineResponse.data->length(); + } + + notifyObserver(); + }); +} + +void OfflineDownload::notifyObserver() { + if (observer) { + observer->statusChanged(status); + } +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_download.hpp b/platform/default/mbgl/storage/offline_download.hpp new file mode 100644 index 00000000000..3925a8aa458 --- /dev/null +++ b/platform/default/mbgl/storage/offline_download.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include +#include + +#include +#include + +namespace mbgl { + +class OfflineDatabase; +class FileSource; +class FileRequest; +class Resource; +class Response; +class SourceInfo; +class StyleParser; +class Source; + +/** + * Coordinates the request and storage of all resources for an offline region. + + * @private + */ +class OfflineDownload { +public: + OfflineDownload(int64_t id, OfflineRegionDefinition&&, OfflineDatabase& offline, FileSource& online); + ~OfflineDownload(); + + void setObserver(std::unique_ptr); + void setState(OfflineRegionDownloadState); + + OfflineRegionStatus getStatus() const; + +private: + void activateDownload(); + void deactivateDownload(); + void notifyObserver(); + + std::vector spriteResources(const StyleParser&) const; + std::vector glyphResources(const StyleParser&) const; + std::vector tileResources(SourceType, uint16_t, const SourceInfo&) const; + + /* + * Recursive async function that iterates over sources, accumulates required resource + * counts in status, and calls the callback when all sources have been (asynchronously) + * checked. + */ + void getTileStatus(OfflineRegionStatus, + std::vector>, + std::function)>) const; + + /* + * Ensure that the resource is stored in the database, requesting it if necessary. + * While the request is in progress, it is recorded in `requests`. If the download + * is deactivated, all in progress requests are cancelled. + */ + void ensureResource(const Resource&, std::function = {}); + + void tryTileJSON(const Source&, std::function&)>); + void ensureTileJSON(const Source&, std::function&)>); + + int64_t id; + OfflineRegionDefinition definition; + OfflineDatabase& offlineDatabase; + FileSource& onlineFileSource; + OfflineRegionStatus status; + std::unique_ptr observer; + std::list> requests; +}; + +} // namespace mbgl diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp index 09301bc4d96..74ceafb00f9 100644 --- a/platform/default/sqlite3.cpp +++ b/platform/default/sqlite3.cpp @@ -77,6 +77,11 @@ Statement Database::prepare(const char *query) { return Statement(db, query); } +int64_t Database::lastInsertRowid() const { + assert(db); + return sqlite3_last_insert_rowid(db); +} + Statement::Statement(sqlite3 *db, const char *sql) { const int err = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr); if (err != SQLITE_OK) { @@ -171,6 +176,12 @@ template <> void Statement::bind(int offset, const char *value) { } void Statement::bind(int offset, const std::string& value, bool retain) { + assert(stmt); + check(sqlite3_bind_text(stmt, offset, value.data(), int(value.size()), + retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); +} + +void Statement::bind(int offset, const std::vector& value, bool retain) { assert(stmt); check(sqlite3_bind_blob(stmt, offset, value.data(), int(value.size()), retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); @@ -234,6 +245,13 @@ template <> std::string Statement::get(int offset) { }; } +template <> std::vector Statement::get(int offset) { + assert(stmt); + const uint8_t* begin = reinterpret_cast(sqlite3_column_blob(stmt, offset)); + const uint8_t* end = begin + sqlite3_column_bytes(stmt, offset); + return { begin, end }; +} + template <> std::chrono::system_clock::time_point Statement::get(int offset) { assert(stmt); return std::chrono::system_clock::from_time_t(sqlite3_column_int64(stmt, offset)); diff --git a/platform/default/sqlite3.hpp b/platform/default/sqlite3.hpp index 29e8967db30..84771f8a8f6 100644 --- a/platform/default/sqlite3.hpp +++ b/platform/default/sqlite3.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include typedef struct sqlite3 sqlite3; @@ -43,6 +44,8 @@ class Database { void exec(const std::string &sql); Statement prepare(const char *query); + int64_t lastInsertRowid() const; + private: sqlite3 *db = nullptr; }; @@ -63,7 +66,10 @@ class Statement { operator bool() const; template void bind(int offset, T value); - void bind(int offset, const std::string &value, bool retain = true); + + void bind(int offset, const std::string&, bool retain = true); + void bind(int offset, const std::vector&, bool retain = true); + template T get(int offset); bool run(); diff --git a/src/mbgl/map/source.hpp b/src/mbgl/map/source.hpp index 5856c7bfabb..1ca233144e1 100644 --- a/src/mbgl/map/source.hpp +++ b/src/mbgl/map/source.hpp @@ -53,6 +53,8 @@ class Source : private util::noncopyable { bool isLoading() const; bool isLoaded() const; + const SourceInfo* getInfo() const { return info.get(); } + // Request or parse all the tiles relevant for the "TransformState". This method // will return true if all the tiles were scheduled for updating of false if // they were not. shouldReparsePartialTiles must be set to "true" if there is diff --git a/test/fixtures/offline/empty.style.json b/test/fixtures/offline/empty.style.json new file mode 100644 index 00000000000..61a8fadcdb0 --- /dev/null +++ b/test/fixtures/offline/empty.style.json @@ -0,0 +1,5 @@ +{ + "version": 8, + "sources": {}, + "layers": [] +} diff --git a/test/fixtures/offline/geojson.json b/test/fixtures/offline/geojson.json new file mode 100644 index 00000000000..8b3698faf7f --- /dev/null +++ b/test/fixtures/offline/geojson.json @@ -0,0 +1,4 @@ +{ + "type": "FeatureCollection", + "features": [] +} diff --git a/test/fixtures/offline/geojson_source.style.json b/test/fixtures/offline/geojson_source.style.json new file mode 100644 index 00000000000..511fca9fd08 --- /dev/null +++ b/test/fixtures/offline/geojson_source.style.json @@ -0,0 +1,10 @@ +{ + "version": 8, + "sources": { + "geojson": { + "type": "geojson", + "data": "http://127.0.0.1:3000/offline/geojson.json" + } + }, + "layers": [] +} diff --git a/test/fixtures/offline/inline_source.style.json b/test/fixtures/offline/inline_source.style.json new file mode 100644 index 00000000000..87155d07d8e --- /dev/null +++ b/test/fixtures/offline/inline_source.style.json @@ -0,0 +1,17 @@ +{ + "version": 8, + "sources": { + "inline": { + "type": "vector", + "maxzoom": 15, + "minzoom": 0, + "tiles": [ "http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf" ] + } + }, + "layers": [{ + "id": "fill", + "type": "fill", + "source": "inline", + "source-layer": "water" + }] +} diff --git a/test/storage/offline.cpp b/test/storage/offline.cpp new file mode 100644 index 00000000000..b34aa02c221 --- /dev/null +++ b/test/storage/offline.cpp @@ -0,0 +1,73 @@ +#include +#include +#include + +#include + +using namespace mbgl; + +static const LatLngBounds sanFrancisco = LatLngBounds::hull( + { 37.6609, -122.5744 }, + { 37.8271, -122.3204 }); + +static const LatLngBounds sanFranciscoWrapped = LatLngBounds::hull( + { 37.6609, 238.5744 }, + { 37.8271, 238.3204 }); + +TEST(OfflineTilePyramidRegionDefinition, TileCoverEmpty) { + OfflineTilePyramidRegionDefinition region("", LatLngBounds::empty(), 0, 20, 1.0); + SourceInfo info; + + auto result = region.tileCover(SourceType::Vector, 512, info); + ASSERT_TRUE(result.empty()); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverZoomIntersection) { + OfflineTilePyramidRegionDefinition region("", sanFrancisco, 2, 2, 1.0); + SourceInfo info; + + info.minZoom = 0; + auto resultIntersection = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, resultIntersection.size()); + + info.minZoom = 3; + auto resultNoIntersection = region.tileCover(SourceType::Vector, 512, info); + ASSERT_TRUE(resultNoIntersection.empty()); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverTileSize) { + OfflineTilePyramidRegionDefinition region("", LatLngBounds::world(), 0, 0, 1.0); + SourceInfo info; + + auto result512 = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, result512.size()); + ASSERT_EQ(0, result512[0].z); + + auto result256 = region.tileCover(SourceType::Vector, 256, info); + ASSERT_EQ(4, result256.size()); + ASSERT_EQ(1, result256[0].z); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverZoomRounding) { + OfflineTilePyramidRegionDefinition region("", sanFrancisco, 0.6, 0.7, 1.0); + SourceInfo info; + + auto resultVector = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, resultVector.size()); + ASSERT_EQ(0, resultVector[0].z); + + auto resultRaster = region.tileCover(SourceType::Raster, 512, info); + ASSERT_EQ(1, resultRaster.size()); + ASSERT_EQ(1, resultRaster[0].z); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverWrapped) { + OfflineTilePyramidRegionDefinition region("", sanFranciscoWrapped, 0, 0, 1.0); + SourceInfo info; + + auto result = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, result.size()); + ASSERT_EQ(0, result[0].z); + ASSERT_EQ(0, result[0].x); + ASSERT_EQ(0, result[0].y); +} diff --git a/test/storage/offline_database.cpp b/test/storage/offline_database.cpp index e2e32ee36be..9435a0b0a59 100644 --- a/test/storage/offline_database.cpp +++ b/test/storage/offline_database.cpp @@ -372,3 +372,62 @@ TEST(OfflineDatabase, PutTileNotFound) { EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason); EXPECT_FALSE(res->data.get()); } + +TEST(OfflineDatabase, CreateRegion) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 }; + OfflineRegionMetadata metadata {{ 1, 2, 3 }}; + OfflineRegion region = db.createRegion(definition, metadata); + + EXPECT_EQ(definition.styleURL, region.getDefinition().styleURL); + EXPECT_EQ(definition.bounds, region.getDefinition().bounds); + EXPECT_EQ(definition.minZoom, region.getDefinition().minZoom); + EXPECT_EQ(definition.maxZoom, region.getDefinition().maxZoom); + EXPECT_EQ(definition.pixelRatio, region.getDefinition().pixelRatio); + EXPECT_EQ(metadata, region.getMetadata()); +} + +TEST(OfflineDatabase, ListRegions) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 }; + OfflineRegionMetadata metadata {{ 1, 2, 3 }}; + + db.createRegion(definition, metadata); + std::vector regions = db.listRegions(); + + ASSERT_EQ(1, regions.size()); + const OfflineRegion& region = regions.at(0); + EXPECT_EQ(definition.styleURL, region.getDefinition().styleURL); + EXPECT_EQ(definition.bounds, region.getDefinition().bounds); + EXPECT_EQ(definition.minZoom, region.getDefinition().minZoom); + EXPECT_EQ(definition.maxZoom, region.getDefinition().maxZoom); + EXPECT_EQ(definition.pixelRatio, region.getDefinition().pixelRatio); + EXPECT_EQ(metadata, region.getMetadata()); +} + +TEST(OfflineDatabase, DeleteRegion) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 }; + OfflineRegionMetadata metadata {{ 1, 2, 3 }}; + db.deleteRegion(db.createRegion(definition, metadata)); + + ASSERT_EQ(0, db.listRegions().size()); +} + +TEST(OfflineDatabase, CreateRegionInfiniteMaxZoom) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "", LatLngBounds::world(), 0, INFINITY, 1.0 }; + OfflineRegionMetadata metadata; + OfflineRegion region = db.createRegion(definition, metadata); + + EXPECT_EQ(0, region.getDefinition().minZoom); + EXPECT_EQ(INFINITY, region.getDefinition().maxZoom); +} diff --git a/test/storage/offline_download.cpp b/test/storage/offline_download.cpp new file mode 100644 index 00000000000..61b72363522 --- /dev/null +++ b/test/storage/offline_download.cpp @@ -0,0 +1,257 @@ +#include "../fixtures/stub_file_source.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +using namespace mbgl; +using namespace std::literals::string_literals; + +class MockObserver : public OfflineRegionObserver { +public: + void statusChanged(OfflineRegionStatus status) override { + if (statusChangedFn) statusChangedFn(status); + }; + + void error(std::exception_ptr err) override { + if (errorFn) errorFn(err); + }; + + std::function statusChangedFn; + std::function errorFn; +}; + +class OfflineTest { +public: + util::RunLoop loop; + StubFileSource fileSource; + OfflineDatabase db { ":memory:" }; + std::size_t size = 0; + + Response response(const std::string& path) { + Response result; + result.data = std::make_shared(util::read_file("test/fixtures/"s + path)); + size += result.data->length(); + return result; + } +}; + +TEST(OfflineDownload, NoSubresources) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/empty.style.json"); + }; + + auto observer = std::make_unique(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(1, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, InlineSource) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/inline_source.style.json"); + }; + + test.fileSource.tileResponse = [&] (const Resource& resource) { + const Resource::TileData& tile = *resource.tileData; + EXPECT_EQ("http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf", tile.urlTemplate); + EXPECT_EQ(1, tile.pixelRatio); + EXPECT_EQ(0, tile.x); + EXPECT_EQ(0, tile.y); + EXPECT_EQ(0, tile.z); + return test.response("offline/0-0-0.vector.pbf"); + }; + + auto observer = std::make_unique(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(2, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, GeoJSONSource) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/geojson_source.style.json"); + }; + + test.fileSource.sourceResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/geojson.json", resource.url); + return test.response("offline/geojson.json"); + }; + + auto observer = std::make_unique(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(2, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, Activate) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/style.json"); + }; + + test.fileSource.spriteImageResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/sprite.png", resource.url); + return test.response("offline/sprite.png"); + }; + + test.fileSource.spriteJSONResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/sprite.json", resource.url); + return test.response("offline/sprite.json"); + }; + + test.fileSource.glyphsResponse = [&] (const Resource&) { + return test.response("offline/glyph.pbf"); + }; + + test.fileSource.sourceResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/streets.json", resource.url); + return test.response("offline/streets.json"); + }; + + test.fileSource.tileResponse = [&] (const Resource& resource) { + const Resource::TileData& tile = *resource.tileData; + EXPECT_EQ("http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf", tile.urlTemplate); + EXPECT_EQ(1, tile.pixelRatio); + EXPECT_EQ(0, tile.x); + EXPECT_EQ(0, tile.y); + EXPECT_EQ(0, tile.z); + return test.response("offline/0-0-0.vector.pbf"); + }; + + auto observer = std::make_unique(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(261, status.completedResourceCount); // 256 glyphs, 1 tile, 1 style, source, sprite image, and sprite json + EXPECT_EQ(test.size, status.completedResourceSize); + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, GetStatusNoResources) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + OfflineRegionStatus status = download.getStatus(); + + EXPECT_EQ(OfflineRegionDownloadState::Inactive, status.downloadState); + EXPECT_EQ(0, status.completedResourceCount); + EXPECT_EQ(0, status.completedResourceSize); + EXPECT_EQ(1, status.requiredResourceCount); + EXPECT_FALSE(status.complete()); +} + +TEST(OfflineDownload, GetStatusStyleComplete) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.db.putRegionResource(1, + Resource::style("http://127.0.0.1:3000/offline/style.json"), + test.response("offline/style.json")); + + OfflineRegionStatus status = download.getStatus(); + + EXPECT_EQ(OfflineRegionDownloadState::Inactive, status.downloadState); + EXPECT_EQ(1, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + EXPECT_EQ(260, status.requiredResourceCount); + EXPECT_FALSE(status.complete()); +} + +TEST(OfflineDownload, GetStatusStyleAndSourceComplete) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.db.putRegionResource(1, + Resource::style("http://127.0.0.1:3000/offline/style.json"), + test.response("offline/style.json")); + + test.db.putRegionResource(1, + Resource::source("http://127.0.0.1:3000/offline/streets.json"), + test.response("offline/streets.json")); + + OfflineRegionStatus status = download.getStatus(); + + EXPECT_EQ(OfflineRegionDownloadState::Inactive, status.downloadState); + EXPECT_EQ(2, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + EXPECT_EQ(261, status.requiredResourceCount); + EXPECT_FALSE(status.complete()); +} diff --git a/test/test.gypi b/test/test.gypi index 883b954a353..9b5bceea72c 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -70,7 +70,9 @@ 'storage/storage.hpp', 'storage/storage.cpp', 'storage/default_file_source.cpp', + 'storage/offline.cpp', 'storage/offline_database.cpp', + 'storage/offline_download.cpp', 'storage/asset_file_source.cpp', 'storage/headers.cpp', 'storage/http_cancel.cpp',