From c7a4827d91d2b7f5a156da930ad2d91f37b8a10c Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 3 Sep 2024 16:01:02 -0400 Subject: [PATCH] Get the driver building again. --- src/CMakeLists.txt | 90 +-- src/driver/zarr.cpp | 937 ------------------------ src/driver/zarr.hh | 104 --- src/driver/zarr.storage.cpp | 1099 ++++++++++++++++++++++++++++ src/driver/zarr.storage.hh | 39 + src/driver/zarr.v2.cpp | 152 ---- src/driver/zarr.v2.hh | 33 - src/driver/zarr.v3.cpp | 159 ---- src/driver/zarr.v3.hh | 32 - tests/CMakeLists.txt | 2 +- tests/driver/CMakeLists.txt | 1 - tests/driver/unit-tests.cpp | 120 --- tests/driver/write-zarr-v2-raw.cpp | 8 +- 13 files changed, 1175 insertions(+), 1601 deletions(-) delete mode 100644 src/driver/zarr.cpp delete mode 100644 src/driver/zarr.hh create mode 100644 src/driver/zarr.storage.cpp create mode 100644 src/driver/zarr.storage.hh delete mode 100644 src/driver/zarr.v2.cpp delete mode 100644 src/driver/zarr.v2.hh delete mode 100644 src/driver/zarr.v3.cpp delete mode 100644 src/driver/zarr.v3.hh delete mode 100644 tests/driver/unit-tests.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c27c4f48..dc67174f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,61 +62,35 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ ####### Acquire Zarr Driver ####### -#if (NOT TARGET acquire-core-logger) -# add_subdirectory(../acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) -#endif () -# -#set(tgt-driver acquire-driver-zarr) -#add_library(${tgt-driver} MODULE -# common/dimension.hh -# common/dimension.cpp -# common/thread.pool.hh -# common/thread.pool.cpp -# common/s3.connection.hh -# common/s3.connection.cpp -# common/utilities.hh -# common/utilities.cpp -# writers/sink.hh -# writers/sink.creator.hh -# writers/sink.creator.cpp -# writers/file.sink.hh -# writers/file.sink.cpp -# writers/s3.sink.hh -# writers/s3.sink.cpp -# writers/array.writer.hh -# writers/array.writer.cpp -# writers/zarrv2.array.writer.hh -# writers/zarrv2.array.writer.cpp -# writers/zarrv3.array.writer.hh -# writers/zarrv3.array.writer.cpp -# writers/blosc.compressor.hh -# writers/blosc.compressor.cpp -# zarr.hh -# zarr.cpp -# zarr.v2.hh -# zarr.v2.cpp -# zarr.v3.hh -# zarr.v3.cpp -# zarr.driver.c -#) -# -#target_include_directories(${tgt-driver} PRIVATE -# $ -#) -# -#target_enable_simd(${tgt-driver}) -#target_link_libraries(${tgt-driver} PRIVATE -# acquire-core-logger -# acquire-core-platform -# acquire-device-kit -# acquire-device-properties -# blosc_static -# nlohmann_json::nlohmann_json -# miniocpp::miniocpp -#) -# -#set_target_properties(${tgt-driver} PROPERTIES -# MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -#) -# -#install(TARGETS ${tgt-driver} LIBRARY DESTINATION lib) +if (NOT TARGET acquire-core-logger) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) +endif () + +set(tgt-driver acquire-driver-zarr) +add_library(${tgt-driver} MODULE + driver/zarr.storage.hh + driver/zarr.storage.cpp + driver/zarr.driver.c +) + +target_include_directories(${tgt-driver} PRIVATE + $ +) + +target_enable_simd(${tgt-driver}) +target_link_libraries(${tgt-driver} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + ${tgt} + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp +) + +set_target_properties(${tgt-driver} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt-driver} LIBRARY DESTINATION lib) diff --git a/src/driver/zarr.cpp b/src/driver/zarr.cpp deleted file mode 100644 index be85d613..00000000 --- a/src/driver/zarr.cpp +++ /dev/null @@ -1,937 +0,0 @@ -#include "zarr.hh" - -#include "writers/zarrv2.array.writer.hh" -#include "nlohmann/json.hpp" - -#include // std::ignore - -namespace zarr = acquire::sink::zarr; -namespace common = zarr::common; -using json = nlohmann::json; - -namespace { -/// \brief Get the filename from a StorageProperties as fs::path. -/// \param props StorageProperties for the Zarr Storage device. -/// \return fs::path representation of the Zarr data directory. -fs::path -as_path(const StorageProperties& props) -{ - if (!props.uri.str) { - return {}; - } - - std::string uri{ props.uri.str, props.uri.nbytes - 1 }; - - if (uri.find("file://") == std::string::npos) { - return uri; - } - - return uri.substr(7); // strlen("file://") == 7 -} - -/// \brief Check that the JSON string is valid. (Valid can mean empty.) -/// \param str Putative JSON metadata string. -/// \param nbytes Size of the JSON metadata char array -void -validate_json(const char* str, size_t nbytes) -{ - // Empty strings are valid (no metadata is fine). - if (nbytes <= 1 || nullptr == str) { - return; - } - - std::ignore = json::parse(str, - str + nbytes, - nullptr, // callback - true, // allow exceptions - true // ignore comments - ); -} - -/// \brief Check that the StorageProperties are valid. -/// \details Assumes either an empty or valid JSON metadata string and a -/// filename string that points to a writable directory. \param props Storage -/// properties for Zarr. \throw std::runtime_error if the parent of the Zarr -/// data directory is not an existing directory. -void -validate_props(const StorageProperties* props) -{ - EXPECT(props->uri.str, "URI string is NULL."); - EXPECT(props->uri.nbytes, "URI string is zero size."); - - // check that JSON is correct (throw std::exception if not) - validate_json(props->external_metadata_json.str, - props->external_metadata_json.nbytes); - - std::string uri{ props->uri.str, props->uri.nbytes - 1 }; - - if (common::is_web_uri(uri)) { - std::vector tokens = common::split_uri(uri); - CHECK(tokens.size() > 2); // http://endpoint/bucket - } else { - const fs::path path = as_path(*props); - fs::path parent_path = path.parent_path(); - if (parent_path.empty()) - parent_path = "."; - - EXPECT(fs::is_directory(parent_path), - "Expected \"%s\" to be a directory.", - parent_path.string().c_str()); - - // check directory is writable - const auto perms = fs::status(fs::path(parent_path)).permissions(); - - EXPECT((perms & (fs::perms::owner_write | fs::perms::group_write | - fs::perms::others_write)) != fs::perms::none, - "Expected \"%s\" to have write permissions.", - parent_path.c_str()); - } -} - -void -validate_dimension(const zarr::Dimension& dim, bool is_append) -{ - if (is_append) { - EXPECT(dim.array_size_px == 0, - "Append dimension array size must be 0."); - } else { - EXPECT(dim.array_size_px > 0, "Dimension array size must be positive."); - } - - EXPECT(dim.chunk_size_px > 0, "Dimension chunk size must be positive."); -} - -[[nodiscard]] bool -is_multiscale_supported(const std::vector& dims) -{ - // 0. Must have at least 3 dimensions. - if (dims.size() < 3) { - return false; - } - - // 1. The first two dimensions must be space dimensions. - if (dims.at(0).kind != DimensionType_Space || - dims.at(1).kind != DimensionType_Space) { - return false; - } - - // 2. Interior dimensions must have size 1 - for (auto i = 2; i < dims.size() - 1; ++i) { - if (dims.at(i).array_size_px != 1) { - return false; - } - } - - return true; -} - -template -VideoFrame* -scale_image(const uint8_t* const data, - size_t bytes_of_data, - const struct ImageShape& shape) -{ - CHECK(data); - CHECK(bytes_of_data); - - const int downscale = 2; - constexpr size_t bytes_of_type = sizeof(T); - const auto factor = 0.25f; - - const auto width = shape.dims.width; - const auto w_pad = width + (width % downscale); - - const auto height = shape.dims.height; - const auto h_pad = height + (height % downscale); - - const auto size_of_image = - static_cast(w_pad * h_pad * factor * bytes_of_type); - - const size_t bytes_of_frame = - common::align_up(sizeof(VideoFrame) + size_of_image, 8); - - auto* dst = (VideoFrame*)malloc(bytes_of_frame); - CHECK(dst); - dst->bytes_of_frame = bytes_of_frame; - - { - dst->shape = shape; - dst->shape.dims = { - .width = w_pad / downscale, - .height = h_pad / downscale, - }; - dst->shape.strides = { - .height = dst->shape.dims.width, - .planes = dst->shape.dims.width * dst->shape.dims.height, - }; - - CHECK(bytes_of_image(&dst->shape) == size_of_image); - } - - const auto* src_img = (T*)data; - auto* dst_img = (T*)dst->data; - memset(dst_img, 0, size_of_image); - - size_t dst_idx = 0; - for (auto row = 0; row < height; row += downscale) { - const bool pad_height = (row == height - 1 && height != h_pad); - - for (auto col = 0; col < width; col += downscale) { - const bool pad_width = (col == width - 1 && width != w_pad); - - size_t idx = row * width + col; - dst_img[dst_idx++] = - (T)(factor * - ((float)src_img[idx] + - (float)src_img[idx + (1 - (int)pad_width)] + - (float)src_img[idx + width * (1 - (int)pad_height)] + - (float)src_img[idx + width * (1 - (int)pad_height) + - (1 - (int)pad_width)])); - } - } - - return dst; -} - -/// @brief Average both `dst` and `src` into `dst`. -template -void -average_two_frames(VideoFrame* dst, const VideoFrame* src) -{ - CHECK(dst); - CHECK(src); - CHECK(dst->bytes_of_frame == src->bytes_of_frame); - - const auto nbytes_image = bytes_of_image(&dst->shape); - const auto num_pixels = nbytes_image / sizeof(T); - for (auto i = 0; i < num_pixels; ++i) { - dst->data[i] = (T)(0.5f * ((float)dst->data[i] + (float)src->data[i])); - } -} - -DeviceState -zarr_set(Storage* self_, const StorageProperties* props) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->set(props); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - return DeviceState_AwaitingConfiguration; - } catch (...) { - LOGE("Exception: (unknown)"); - return DeviceState_AwaitingConfiguration; - } - - return DeviceState_Armed; -} - -void -zarr_get(const Storage* self_, StorageProperties* props) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->get(props); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} - -void -zarr_get_meta(const Storage* self_, StoragePropertyMetadata* meta) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->get_meta(meta); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} - -DeviceState -zarr_start(Storage* self_) noexcept -{ - DeviceState state{ DeviceState_AwaitingConfiguration }; - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->start(); - state = self->state; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return state; -} - -DeviceState -zarr_append(Storage* self_, const VideoFrame* frames, size_t* nbytes) noexcept -{ - DeviceState state{ DeviceState_AwaitingConfiguration }; - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - *nbytes = self->append(frames, *nbytes); - state = self->state; - } catch (const std::exception& exc) { - *nbytes = 0; - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - *nbytes = 0; - LOGE("Exception: (unknown)"); - } - - return state; -} - -DeviceState -zarr_stop(Storage* self_) noexcept -{ - DeviceState state{ DeviceState_AwaitingConfiguration }; - - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - CHECK(self->stop()); // state is set to DeviceState_Armed here - state = self->state; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return state; -} - -void -zarr_destroy(Storage* self_) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - if (self_->stop) - self_->stop(self_); - - delete self; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} - -void -zarr_reserve_image_shape(Storage* self_, const ImageShape* shape) noexcept -{ - try { - CHECK(self_); - auto* self = (zarr::Zarr*)self_; - self->reserve_image_shape(shape); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } -} -} // end ::{anonymous} namespace - -void -zarr::Zarr::set(const StorageProperties* props) -{ - EXPECT(state != DeviceState_Running, - "Cannot set properties while running."); - CHECK(props); - - // checks the directory exists and is writable - validate_props(props); - - std::string uri(props->uri.str, props->uri.nbytes - 1); - - if (common::is_web_uri(uri)) { - dataset_root_ = uri; - } else { - dataset_root_ = as_path(*props).string(); - } - - if (props->access_key_id.str) { - s3_access_key_id_ = std::string(props->access_key_id.str, - props->access_key_id.nbytes - 1); - } - - if (props->secret_access_key.str) { - s3_secret_access_key_ = std::string( - props->secret_access_key.str, props->secret_access_key.nbytes - 1); - } - - if (props->external_metadata_json.str) { - external_metadata_json_ = - std::string(props->external_metadata_json.str, - props->external_metadata_json.nbytes - 1); - } - - pixel_scale_um_ = props->pixel_scale_um; - - set_dimensions_(props); - enable_multiscale_ = props->enable_multiscale && - is_multiscale_supported(acquisition_dimensions_); -} - -void -zarr::Zarr::get(StorageProperties* props) const -{ - CHECK(props); - storage_properties_destroy(props); - - std::string uri; - if (common::is_web_uri(dataset_root_)) { - uri = dataset_root_; - } else if (!dataset_root_.empty()) { - fs::path dataset_root_abs = fs::absolute(dataset_root_); - uri = "file://" + dataset_root_abs.string(); - } - - const size_t bytes_of_filename = uri.empty() ? 0 : uri.size() + 1; - - const char* metadata = external_metadata_json_.empty() - ? nullptr - : external_metadata_json_.c_str(); - const size_t bytes_of_metadata = - metadata ? external_metadata_json_.size() + 1 : 0; - - CHECK(storage_properties_init(props, - 0, - uri.c_str(), - bytes_of_filename, - metadata, - bytes_of_metadata, - pixel_scale_um_, - acquisition_dimensions_.size())); - - // set access key and secret - { - const char* access_key_id = - s3_access_key_id_.has_value() ? s3_access_key_id_->c_str() : nullptr; - const size_t bytes_of_access_key_id = - access_key_id ? s3_access_key_id_->size() + 1 : 0; - - const char* secret_access_key = s3_secret_access_key_.has_value() - ? s3_secret_access_key_->c_str() - : nullptr; - const size_t bytes_of_secret_access_key = - secret_access_key ? s3_secret_access_key_->size() + 1 : 0; - - if (access_key_id && secret_access_key) { - CHECK(storage_properties_set_access_key_and_secret( - props, - access_key_id, - bytes_of_access_key_id, - secret_access_key, - bytes_of_secret_access_key)); - } - } - - for (auto i = 0; i < acquisition_dimensions_.size(); ++i) { - const auto dim = acquisition_dimensions_.at(i); - CHECK(storage_properties_set_dimension(props, - i, - dim.name.c_str(), - dim.name.length() + 1, - dim.kind, - dim.array_size_px, - dim.chunk_size_px, - dim.shard_size_chunks)); - } - - storage_properties_set_enable_multiscale(props, - (uint8_t)enable_multiscale_); -} - -void -zarr::Zarr::get_meta(StoragePropertyMetadata* meta) const -{ - CHECK(meta); - memset(meta, 0, sizeof(*meta)); - - meta->chunking_is_supported = 1; - meta->multiscale_is_supported = 1; - meta->s3_is_supported = 1; -} - -void -zarr::Zarr::start() -{ - error_ = true; - - thread_pool_ = std::make_shared( - std::thread::hardware_concurrency(), - [this](const std::string& err) { this->set_error(err); }); - - if (common::is_web_uri(dataset_root_)) { - std::vector tokens = common::split_uri(dataset_root_); - CHECK(tokens.size() > 1); - const std::string endpoint = tokens[0] + "//" + tokens[1]; - connection_pool_ = std::make_shared( - 8, endpoint, *s3_access_key_id_, *s3_secret_access_key_); - } else { - // remove the folder if it exists - if (fs::exists(dataset_root_)) { - std::error_code ec; - EXPECT(fs::remove_all(dataset_root_, ec), - R"(Failed to remove folder for "%s": %s)", - dataset_root_.c_str(), - ec.message().c_str()); - } - - // create the dataset folder - fs::create_directories(dataset_root_); - } - - allocate_writers_(); - - make_metadata_sinks_(); - write_fixed_metadata_(); - - state = DeviceState_Running; - error_ = false; -} - -int -zarr::Zarr::stop() noexcept -{ - int is_ok = 1; - - if (DeviceState_Running == state) { - state = DeviceState_Armed; - is_ok = 0; - - try { - // must precede close of chunk file - write_group_metadata_(); - metadata_sinks_.clear(); - - for (auto& writer : writers_) { - writer->finalize(); - } - - // call await_stop() before destroying to give jobs a chance to - // finish - thread_pool_->await_stop(); - thread_pool_ = nullptr; - - connection_pool_ = nullptr; - - // don't clear before all working threads have shut down - writers_.clear(); - - // should be empty, but just in case - for (auto& [_, frame] : scaled_frames_) { - if (frame && *frame) { - free(*frame); - } - } - scaled_frames_.clear(); - - error_ = false; - error_msg_.clear(); - - is_ok = 1; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - } - - return is_ok; -} - -size_t -zarr::Zarr::append(const VideoFrame* frames, size_t nbytes) -{ - CHECK(DeviceState_Running == state); - EXPECT(!error_, "%s", error_msg_.c_str()); - - if (0 == nbytes) { - return nbytes; - } - - const VideoFrame* cur = nullptr; - const auto* end = (const VideoFrame*)((uint8_t*)frames + nbytes); - auto next = [&]() -> const VideoFrame* { - const uint8_t* p = ((const uint8_t*)cur) + cur->bytes_of_frame; - return (const VideoFrame*)p; - }; - - for (cur = frames; cur < end; cur = next()) { - const size_t bytes_of_frame = bytes_of_image(&cur->shape); - const size_t bytes_written = append_frame( - const_cast(cur->data), bytes_of_frame, cur->shape); - EXPECT(bytes_written == bytes_of_frame, - "Expected to write %zu bytes, but wrote %zu.", - bytes_of_frame, - bytes_written); - } - - return nbytes; -} - -size_t -zarr::Zarr::append_frame(const uint8_t* const data, - size_t bytes_of_data, - const ImageShape& shape) -{ - CHECK(DeviceState_Running == state); - EXPECT(!error_, "%s", error_msg_.c_str()); - - if (!data || !bytes_of_data) { - return 0; - } - - const size_t bytes_written = writers_.at(0)->write(data, bytes_of_data); - if (bytes_written != bytes_of_data) { - set_error("Failed to write frame."); - return bytes_written; - } - - // multiscale - if (writers_.size() > 1) { - write_multiscale_frames_(data, bytes_written, shape); - } - - return bytes_written; -} - -void -zarr::Zarr::reserve_image_shape(const ImageShape* shape) -{ - EXPECT(state != DeviceState_Running, - "Cannot reserve image shape while running."); - - // `shape` should be verified nonnull in storage_reserve_image_shape, but - // let's check anyway - CHECK(shape); - - // image shape should be compatible with first two acquisition dimensions - EXPECT(shape->dims.width == acquisition_dimensions_.at(0).array_size_px, - "Image width must match first acquisition dimension."); - EXPECT(shape->dims.height == acquisition_dimensions_.at(1).array_size_px, - "Image height must match second acquisition dimension."); - - image_shape_ = *shape; -} - -/// Zarr - -zarr::Zarr::Zarr() - : Storage { - .state = DeviceState_AwaitingConfiguration, - .set = ::zarr_set, - .get = ::zarr_get, - .get_meta = ::zarr_get_meta, - .start = ::zarr_start, - .append = ::zarr_append, - .stop = ::zarr_stop, - .destroy = ::zarr_destroy, - .reserve_image_shape = ::zarr_reserve_image_shape, - } - , thread_pool_{ nullptr } - , pixel_scale_um_{ 1, 1 } - , enable_multiscale_{ false } - , error_{ false } -{ -} - -zarr::Zarr::Zarr(BloscCompressionParams&& compression_params) - : Zarr() -{ - blosc_compression_params_ = std::move(compression_params); -} - -void -zarr::Zarr::set_dimensions_(const StorageProperties* props) -{ - const auto dimension_count = props->acquisition_dimensions.size; - EXPECT(dimension_count > 2, "Expected at least 3 dimensions."); - - acquisition_dimensions_.clear(); - - for (auto i = 0; i < dimension_count; ++i) { - CHECK(props->acquisition_dimensions.data[i].name.str); - Dimension dim(props->acquisition_dimensions.data[i]); - validate_dimension(dim, i == dimension_count - 1); - - acquisition_dimensions_.push_back(dim); - } -} - -void -zarr::Zarr::set_error(const std::string& msg) noexcept -{ - std::scoped_lock lock(mutex_); - - // don't overwrite the first error - if (!error_) { - error_ = true; - error_msg_ = msg; - } -} - -void -zarr::Zarr::write_fixed_metadata_() const -{ - write_base_metadata_(); - write_external_metadata_(); -} - -json -zarr::Zarr::make_multiscale_metadata_() const -{ - json multiscales = json::array({ json::object() }); - // write multiscale metadata - multiscales[0]["version"] = "0.4"; - - auto& axes = multiscales[0]["axes"]; - for (auto dim = acquisition_dimensions_.rbegin(); - dim != acquisition_dimensions_.rend(); - ++dim) { - std::string type; - switch (dim->kind) { - case DimensionType_Space: - type = "space"; - break; - case DimensionType_Channel: - type = "channel"; - break; - case DimensionType_Time: - type = "time"; - break; - case DimensionType_Other: - type = "other"; - break; - default: - throw std::runtime_error("Unknown dimension type"); - } - - if (dim < acquisition_dimensions_.rend() - 2) { - axes.push_back({ { "name", dim->name }, { "type", type } }); - } else { - axes.push_back({ { "name", dim->name }, - { "type", type }, - { "unit", "micrometer" } }); - } - } - - // spatial multiscale metadata - if (writers_.empty()) { - std::vector scales; - for (auto i = 0; i < acquisition_dimensions_.size() - 2; ++i) { - scales.push_back(1.); - } - scales.push_back(pixel_scale_um_.y); - scales.push_back(pixel_scale_um_.x); - - multiscales[0]["datasets"] = { - { - { "path", "0" }, - { "coordinateTransformations", - { - { - { "type", "scale" }, - { "scale", scales }, - }, - } - }, - }, - }; - } else { - for (auto i = 0; i < writers_.size(); ++i) { - std::vector scales; - scales.push_back(std::pow(2, i)); // append - for (auto k = 0; k < acquisition_dimensions_.size() - 3; ++k) { - scales.push_back(1.); - } - scales.push_back(std::pow(2, i) * pixel_scale_um_.y); // y - scales.push_back(std::pow(2, i) * pixel_scale_um_.x); // x - - multiscales[0]["datasets"].push_back({ - { "path", std::to_string(i) }, - { "coordinateTransformations", - { - { - { "type", "scale" }, - { "scale", scales }, - }, - } - }, - }); - } - - // downsampling metadata - multiscales[0]["type"] = "local_mean"; - multiscales[0]["metadata"] = { - { "description", - "The fields in the metadata describe how to reproduce this " - "multiscaling in scikit-image. The method and its parameters are " - "given here." }, - { "method", "skimage.transform.downscale_local_mean" }, - { "version", "0.21.0" }, - { "args", "[2]" }, - { "kwargs", { "cval", 0 } }, - }; - } - - return multiscales; -} - -void -zarr::Zarr::write_multiscale_frames_(const uint8_t* const data_, - size_t bytes_of_data, - const ImageShape& shape_) -{ - auto* data = const_cast(data_); - ImageShape shape = shape_; - struct VideoFrame* dst; - - std::function scale; - std::function average2; - switch (shape.type) { - case SampleType_u10: - case SampleType_u12: - case SampleType_u14: - case SampleType_u16: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_i8: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_i16: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_f32: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - case SampleType_u8: - scale = ::scale_image; - average2 = ::average_two_frames; - break; - default: - char err_msg[64]; - snprintf(err_msg, - sizeof(err_msg), - "Unsupported pixel type: %s", - common::sample_type_to_string(shape.type)); - throw std::runtime_error(err_msg); - } - - for (auto i = 1; i < writers_.size(); ++i) { - dst = scale(data, bytes_of_data, shape); - if (scaled_frames_.at(i).has_value()) { - // average - average2(dst, scaled_frames_.at(i).value()); - - // write the downsampled frame - const size_t bytes_of_frame = bytes_of_image(&dst->shape); - CHECK(writers_.at(i)->write(dst->data, bytes_of_frame)); - - // clean up this level of detail - free(scaled_frames_.at(i).value()); - scaled_frames_.at(i).reset(); - - // setup for next iteration - if (i + 1 < writers_.size()) { - data = dst->data; - shape = dst->shape; - bytes_of_data = bytes_of_image(&shape); - } else { - // no longer needed - free(dst); - } - } else { - scaled_frames_.at(i) = dst; - break; - } - } -} - -#ifndef NO_UNIT_TESTS -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -///< Test that a single frame with 1 plane is padded and averaged correctly. -template -void -test_average_frame_inner(const SampleType& stype) -{ - const size_t bytes_of_frame = - common::align_up(sizeof(VideoFrame) + 9 * sizeof(T), 8); - auto* src = (VideoFrame*)malloc(bytes_of_frame); - CHECK(src); - - src->bytes_of_frame = bytes_of_frame; - src->shape = { - .dims = { - .channels = 1, - .width = 3, - .height = 3, - .planes = 1, - }, - .strides = { - .channels = 1, - .width = 1, - .height = 3, - .planes = 9 - }, - .type = stype - }; - - for (auto i = 0; i < 9; ++i) { - ((T*)src->data)[i] = (T)(i + 1); - } - - auto dst = - scale_image(src->data, bytes_of_image(&src->shape), src->shape); - CHECK(((T*)dst->data)[0] == (T)3); - CHECK(((T*)dst->data)[1] == (T)4.5); - CHECK(((T*)dst->data)[2] == (T)7.5); - CHECK(((T*)dst->data)[3] == (T)9); - - free(src); - free(dst); -} - -extern "C" acquire_export int -unit_test__average_frame() -{ - try { - test_average_frame_inner(SampleType_u8); - test_average_frame_inner(SampleType_i8); - test_average_frame_inner(SampleType_u16); - test_average_frame_inner(SampleType_i16); - test_average_frame_inner(SampleType_f32); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - return 0; - } catch (...) { - LOGE("Exception: (unknown)"); - return 0; - } - - return 1; -} -#endif diff --git a/src/driver/zarr.hh b/src/driver/zarr.hh deleted file mode 100644 index da59a8db..00000000 --- a/src/driver/zarr.hh +++ /dev/null @@ -1,104 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_V0 -#define H_ACQUIRE_STORAGE_ZARR_V0 - -#include "device/kit/storage.h" - -#include "common/utilities.hh" -#include "internal/thread.pool.hh" -#include "common/s3.connection.hh" -#include "writers/array.writer.hh" -#include "writers/blosc.compressor.hh" - -#include -#include -#include -#include -#include // std::pair -#include - -#include - -namespace fs = std::filesystem; -using json = nlohmann::json; - -namespace acquire::sink::zarr { -struct Zarr : public Storage -{ - public: - Zarr(); - explicit Zarr(BloscCompressionParams&& compression_params); - virtual ~Zarr() noexcept = default; - - /// Storage interface - void set(const StorageProperties* props); - void get(StorageProperties* props) const; - virtual void get_meta(StoragePropertyMetadata* meta) const; - void start(); - int stop() noexcept; - size_t append(const VideoFrame* frames, size_t nbytes); - size_t append_frame(const uint8_t* data, - size_t bytes_of_data, - const ImageShape& shape); - void reserve_image_shape(const ImageShape* shape); - - /// Error state - void set_error(const std::string& msg) noexcept; - - protected: - /// static - set on construction - std::optional blosc_compression_params_; - - /// changes on set - std::string dataset_root_; - std::optional s3_access_key_id_; - std::optional s3_secret_access_key_; - std::string external_metadata_json_; - PixelScale pixel_scale_um_; - bool enable_multiscale_; - - /// changes on reserve_image_shape - struct ImageShape image_shape_; - std::vector acquisition_dimensions_; - std::vector> writers_; - - /// changes on append - // scaled frames, keyed by level-of-detail - std::unordered_map> scaled_frames_; - - /// changes on start - std::shared_ptr thread_pool_; - std::shared_ptr connection_pool_; - std::unordered_map> metadata_sinks_; - - /// Multithreading - mutable std::mutex mutex_; // for error_ / error_msg_ - - /// Error state - bool error_; - std::string error_msg_; - - /// Setup - void set_dimensions_(const StorageProperties* props); - virtual void allocate_writers_() = 0; - - /// Metadata - virtual void make_metadata_sinks_() = 0; - - // fixed metadata - void write_fixed_metadata_() const; - virtual void write_base_metadata_() const = 0; - virtual void write_external_metadata_() const = 0; - - // mutable metadata, changes on flush - virtual void write_group_metadata_() const = 0; - - /// Multiscale - json make_multiscale_metadata_() const; - void write_multiscale_frames_(const uint8_t* data_, - size_t bytes_of_data, - const ImageShape& shape_); -}; - -} // namespace acquire::sink::zarr - -#endif // H_ACQUIRE_STORAGE_ZARR_V0 diff --git a/src/driver/zarr.storage.cpp b/src/driver/zarr.storage.cpp new file mode 100644 index 00000000..234bc4e5 --- /dev/null +++ b/src/driver/zarr.storage.cpp @@ -0,0 +1,1099 @@ +#include "zarr.storage.hh" +#include "logger.h" + +#include + +#include +#include + +#define LOG(...) aq_logger(0, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) +#define LOGE(...) aq_logger(1, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) +#define EXPECT(e, ...) \ + do { \ + if (!(e)) { \ + LOGE(__VA_ARGS__); \ + throw std::runtime_error("Check failed: " #e); \ + } \ + } while (0) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) + +#define ZARR_OK(e) \ + do { \ + ZarrError __err = (e); \ + EXPECT( \ + __err == ZarrError_Success, "%s", Zarr_get_error_message(__err)); \ + } while (0) + +namespace sink = acquire::sink; +namespace fs = std::filesystem; + +using json = nlohmann::json; + +struct S3URI +{ + std::string endpoint; + std::string bucket; + std::string store_path; +}; + +namespace { +/** + * @brief Align a size to a given alignment. + * @param n Size to align. + * @param align Alignment. + * @return Aligned size. + */ +size_t +align_up(size_t n, size_t align) +{ + EXPECT(align > 0, "Alignment must be greater than zero."); + return align * ((n + align - 1) / align); +} + +/** + * @brief Get the filename from a StorageProperties as fs::path. + * @param props StorageProperties for the Zarr Storage device. + * @return fs::path representation of the Zarr data directory. + */ +fs::path +as_path(const StorageProperties* props) +{ + if (!props->uri.str) { + return {}; + } + + std::string uri{ props->uri.str, props->uri.nbytes - 1 }; + + while (uri.find("file://") != std::string::npos) { + uri = uri.substr(7); // strlen("file://") == 7 + } + + return uri; +} + +/// \brief Check that the JSON string is valid. (Valid can mean empty.) +/// \param str Putative JSON metadata string. +/// \param nbytes Size of the JSON metadata char array +void +validate_json(const char* str, size_t nbytes) +{ + // Empty strings are valid (no metadata is fine). + if (nbytes <= 1 || nullptr == str) { + return; + } + + std::ignore = json::parse(str, + str + nbytes, + nullptr, // callback + true, // allow exceptions + true // ignore comments + ); +} + +/** + * @brief Check if a URI is a web URI (e.g., for S3). + * @param uri String to check. + * @return True if the URI is a web URI, false otherwise. + */ +bool +is_web_uri(std::string_view uri) +{ + return uri.starts_with("http://") || uri.starts_with("https://"); +} + +/** + * @brief Split a URI into its components. + * @param uri String to split. + * @return Vector of strings representing the components of the URI. + */ +std::vector +split_uri(const std::string& uri) +{ + const char delim = '/'; + + std::vector out; + size_t begin = 0, end = uri.find_first_of(delim); + + while (end != std::string::npos) { + std::string part = uri.substr(begin, end - begin); + if (!part.empty()) + out.push_back(part); + + begin = end + 1; + end = uri.find_first_of(delim, begin); + } + + // Add the last segment of the URI (if any) after the last '/' + std::string last_part = uri.substr(begin); + if (!last_part.empty()) { + out.push_back(last_part); + } + + return out; +} + +/// \brief Check that the StorageProperties are valid. +/// \details Assumes either an empty or valid JSON metadata string and a +/// filename string that points to a writable directory. \param props Storage +/// properties for Zarr. \throw std::runtime_error if the parent of the Zarr +/// data directory is not an existing directory. +void +validate_props(const StorageProperties* props) +{ + EXPECT(props->uri.str, "URI string is NULL."); + EXPECT(props->uri.nbytes, "URI string is zero size."); + + // check that JSON is correct (throw std::exception if not) + validate_json(props->external_metadata_json.str, + props->external_metadata_json.nbytes); + + std::string uri{ props->uri.str, props->uri.nbytes - 1 }; + + if (is_web_uri(uri)) { + std::vector tokens = split_uri(uri); + CHECK(tokens.size() > 2); // http://endpoint/bucket + } else { + const fs::path path = as_path(props); + fs::path parent_path = path.parent_path(); + if (parent_path.empty()) + parent_path = "."; + + EXPECT(fs::is_directory(parent_path), + "Expected \"%s\" to be a directory.", + parent_path.string().c_str()); + + // check directory is writable + const auto perms = fs::status(fs::path(parent_path)).permissions(); + + EXPECT((perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none, + "Expected \"%s\" to have write permissions.", + parent_path.c_str()); + } +} + +void +validate_dimension(const struct StorageDimension* dim, bool is_append) +{ + EXPECT(dim, "Dimension is NULL."); + + if (is_append) { + EXPECT(dim->array_size_px == 0, + "Append dimension array size must be 0."); + } else { + EXPECT(dim->array_size_px > 0, + "Dimension array size must be positive."); + } + + EXPECT(dim->chunk_size_px > 0, "Dimension chunk size must be positive."); + + EXPECT(dim->name.str, "Dimension name is NULL."); + EXPECT(dim->name.nbytes > 1, "Dimension name is empty."); +} + +[[nodiscard]] bool +is_multiscale_supported(struct StorageDimension* dims, size_t ndims) +{ + EXPECT(dims, "Dimensions are NULL."); + EXPECT(ndims > 2, "Expected at least 3 dimensions."); + + // 1. The final two dimensions must be space dimensions. + if (dims[ndims - 1].kind != DimensionType_Space || + dims[ndims - 2].kind != DimensionType_Space) { + return false; + } + + // 2. Interior dimensions must have size 1 + for (auto i = 1; i < ndims - 2; ++i) { + if (dims[i].array_size_px != 1) { + return false; + } + } + + return true; +} + +template +VideoFrame* +scale_image(const uint8_t* const data, + size_t bytes_of_data, + const struct ImageShape& shape) +{ + CHECK(data); + CHECK(bytes_of_data); + + const int downscale = 2; + constexpr size_t bytes_of_type = sizeof(T); + const auto factor = 0.25f; + + const auto width = shape.dims.width; + const auto w_pad = width + (width % downscale); + + const auto height = shape.dims.height; + const auto h_pad = height + (height % downscale); + + const auto size_of_image = + static_cast(w_pad * h_pad * factor * bytes_of_type); + + const size_t bytes_of_frame = + align_up(sizeof(VideoFrame) + size_of_image, 8); + + auto* dst = (VideoFrame*)malloc(bytes_of_frame); + CHECK(dst); + dst->bytes_of_frame = bytes_of_frame; + + { + dst->shape = shape; + dst->shape.dims = { + .width = w_pad / downscale, + .height = h_pad / downscale, + }; + dst->shape.strides = { + .height = dst->shape.dims.width, + .planes = dst->shape.dims.width * dst->shape.dims.height, + }; + + CHECK(bytes_of_image(&dst->shape) == size_of_image); + } + + const auto* src_img = (T*)data; + auto* dst_img = (T*)dst->data; + memset(dst_img, 0, size_of_image); + + size_t dst_idx = 0; + for (auto row = 0; row < height; row += downscale) { + const bool pad_height = (row == height - 1 && height != h_pad); + + for (auto col = 0; col < width; col += downscale) { + const bool pad_width = (col == width - 1 && width != w_pad); + + size_t idx = row * width + col; + dst_img[dst_idx++] = + (T)(factor * + ((float)src_img[idx] + + (float)src_img[idx + (1 - (int)pad_width)] + + (float)src_img[idx + width * (1 - (int)pad_height)] + + (float)src_img[idx + width * (1 - (int)pad_height) + + (1 - (int)pad_width)])); + } + } + + return dst; +} + +/// @brief Average both `dst` and `src` into `dst`. +template +void +average_two_frames(VideoFrame* dst, const VideoFrame* src) +{ + CHECK(dst); + CHECK(src); + CHECK(dst->bytes_of_frame == src->bytes_of_frame); + + const auto nbytes_image = bytes_of_image(&dst->shape); + const auto num_pixels = nbytes_image / sizeof(T); + for (auto i = 0; i < num_pixels; ++i) { + dst->data[i] = (T)(0.5f * ((float)dst->data[i] + (float)src->data[i])); + } +} + +DeviceState +zarr_set(Storage* self_, const StorageProperties* props) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->set(props); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + return DeviceState_AwaitingConfiguration; + } catch (...) { + LOGE("Exception: (unknown)"); + return DeviceState_AwaitingConfiguration; + } + + return DeviceState_Armed; +} + +void +zarr_get(const Storage* self_, StorageProperties* props) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->get(props); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} + +void +zarr_get_meta(const Storage* self_, StoragePropertyMetadata* meta) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->get_meta(meta); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} + +DeviceState +zarr_start(Storage* self_) noexcept +{ + DeviceState state{ DeviceState_AwaitingConfiguration }; + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->start(); + state = self->state; + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + + return state; +} + +DeviceState +zarr_append(Storage* self_, const VideoFrame* frames, size_t* nbytes) noexcept +{ + DeviceState state{ DeviceState_AwaitingConfiguration }; + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + *nbytes = self->append(frames, *nbytes); + state = self->state; + } catch (const std::exception& exc) { + *nbytes = 0; + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + *nbytes = 0; + LOGE("Exception: (unknown)"); + } + + return state; +} + +DeviceState +zarr_stop(Storage* self_) noexcept +{ + DeviceState state{ DeviceState_AwaitingConfiguration }; + + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->stop(); // state is set to DeviceState_Armed here + state = self->state; + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + + return state; +} + +void +zarr_destroy(Storage* self_) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + if (self_->stop) + self_->stop(self_); + + delete self; + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} + +void +zarr_reserve_image_shape(Storage* self_, const ImageShape* shape) noexcept +{ + try { + CHECK(self_); + auto* self = (sink::Zarr*)self_; + self->reserve_image_shape(shape); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } +} +} // end ::{anonymous} namespace + +sink::Zarr::Zarr(ZarrVersion version, + ZarrCompressionCodec compression_codec, + uint8_t compression_level, + uint8_t shuffle) + : Storage { + .state = DeviceState_AwaitingConfiguration, + .set = ::zarr_set, + .get = ::zarr_get, + .get_meta = ::zarr_get_meta, + .start = ::zarr_start, + .append = ::zarr_append, + .stop = ::zarr_stop, + .destroy = ::zarr_destroy, + .reserve_image_shape = ::zarr_reserve_image_shape, + } + , version_(version) + , compression_codec_(compression_codec) + , compression_level_(compression_level) + , shuffle_(shuffle) + , stream_settings_(nullptr) + , stream_(nullptr) +{ + EXPECT( + version_ < ZarrVersionCount, "Unsupported Zarr version: %d", version); + EXPECT(compression_codec_ < ZarrCompressionCodecCount, + "Unsupported compression codec: %d", + compression_codec); + EXPECT( + compression_level_ <= 9, + "Invalid compression level: %d. Compression level must be in [0, 9].", + compression_level_); + EXPECT(shuffle_ <= 2, + "Invalid shuffle value: %d. Shuffle must be 0, 1, or 2.", + shuffle_); + + stream_settings_ = ZarrStreamSettings_create(); +} + +sink::Zarr::~Zarr() +{ + stop(); +} + +void +sink::Zarr::set(const StorageProperties* props) +{ + EXPECT(state != DeviceState_Running, + "Cannot set properties while running."); + EXPECT(props, "StorageProperties is NULL."); + + if (!stream_settings_) { + stream_settings_ = ZarrStreamSettings_create(); + CHECK(stream_settings_); + } + + EXPECT(props->uri.str, "URI string is NULL."); + EXPECT(props->uri.nbytes > 1, "URI string is empty."); + std::string uri(props->uri.str, props->uri.nbytes - 1); + + std::string store_path; + + if (is_web_uri(uri)) { + auto components = split_uri(uri); + EXPECT(components.size() > 3, "Invalid URI: %s", uri.c_str()); + + std::string s3_endpoint = components[0] + "//" + components[1]; + ZARR_OK(ZarrStreamSettings_set_s3_endpoint( + stream_settings_, s3_endpoint.c_str(), s3_endpoint.size() + 1)); + + const std::string& s3_bucket_name = components[2]; + ZARR_OK(ZarrStreamSettings_set_s3_bucket_name( + stream_settings_, s3_bucket_name.c_str(), s3_bucket_name.size() + 1)); + + store_path = components[3]; + for (auto i = 4; i < components.size(); ++i) { + store_path += "/" + components[i]; + } + ZARR_OK(ZarrStreamSettings_set_store_path( + stream_settings_, store_path.c_str(), store_path.size() + 1)); + } else { + if (uri.find("file://") != std::string::npos) { + uri = uri.substr(7); // strlen("file://") == 7 + } + store_path = uri; + + if (fs::exists(store_path)) { + std::error_code ec; + EXPECT(fs::remove_all(store_path, ec), + R"(Failed to remove folder for "%s": %s)", + store_path.c_str(), + ec.message().c_str()); + } + + fs::path parent_path = fs::path(store_path).parent_path(); + if (parent_path.empty()) + parent_path = "."; + + EXPECT(fs::is_directory(parent_path), + "Expected \"%s\" to be a directory.", + parent_path.string().c_str()); + + // check directory is writable + const auto perms = fs::status(fs::path(parent_path)).permissions(); + + EXPECT((perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none, + "Expected \"%s\" to have write permissions.", + parent_path.c_str()); + + ZarrStreamSettings_set_store_path( + stream_settings_, store_path.c_str(), store_path.size() + 1); + } + + if (props->access_key_id.str) { + ZARR_OK( + ZarrStreamSettings_set_s3_access_key_id(stream_settings_, + props->access_key_id.str, + props->access_key_id.nbytes)); + } + + if (props->secret_access_key.str) { + ZARR_OK(ZarrStreamSettings_set_s3_secret_access_key( + stream_settings_, + props->secret_access_key.str, + props->secret_access_key.nbytes)); + } + + if (props->external_metadata_json.str) { + external_metadata_json_ = + std::string(props->external_metadata_json.str, + props->external_metadata_json.nbytes - 1); + } + + ZARR_OK(ZarrStreamSettings_reserve_dimensions( + stream_settings_, props->acquisition_dimensions.size)); + for (auto i = 0; i < props->acquisition_dimensions.size; ++i) { + const auto* dim = props->acquisition_dimensions.data + i; + validate_dimension(dim, i == 0); + + ZarrDimensionType kind; + switch (dim->kind) { + case DimensionType_Space: + kind = ZarrDimensionType_Space; + break; + case DimensionType_Channel: + kind = ZarrDimensionType_Channel; + break; + case DimensionType_Time: + kind = ZarrDimensionType_Time; + break; + case DimensionType_Other: + kind = ZarrDimensionType_Other; + break; + default: + throw std::runtime_error("Invalid dimension type: " + + std::to_string(dim->kind)); + } + + ZARR_OK(ZarrStreamSettings_set_dimension(stream_settings_, + i, + dim->name.str, + dim->name.nbytes, + kind, + dim->array_size_px, + dim->chunk_size_px, + dim->shard_size_chunks)); + } + + ZARR_OK(ZarrStreamSettings_set_multiscale(stream_settings_, + props->enable_multiscale)); +} + +void +sink::Zarr::get(StorageProperties* props) const +{ + EXPECT(props, "StorageProperties is NULL."); + EXPECT(stream_settings_ || stream_, "No stream or stream settings."); + + storage_properties_destroy(props); + + std::string s3_endpoint, s3_bucket, store_path; + std::string access_key_id, secret_access_key; + uint8_t multiscale; + size_t ndims; + + if (stream_) { + s3_endpoint = ZarrStream_get_s3_endpoint(stream_); + s3_bucket = ZarrStream_get_s3_bucket_name(stream_); + store_path = ZarrStream_get_store_path(stream_); + + access_key_id = ZarrStream_get_s3_access_key_id(stream_); + secret_access_key = ZarrStream_get_s3_secret_access_key(stream_); + + ndims = ZarrStream_get_dimension_count(stream_); + + multiscale = ZarrStream_get_multiscale(stream_); + } else { // stream_settings_ + s3_endpoint = ZarrStreamSettings_get_s3_endpoint(stream_settings_); + s3_bucket = ZarrStreamSettings_get_s3_bucket_name(stream_settings_); + store_path = ZarrStreamSettings_get_store_path(stream_settings_); + + access_key_id = + ZarrStreamSettings_get_s3_access_key_id(stream_settings_); + secret_access_key = + ZarrStreamSettings_get_s3_secret_access_key(stream_settings_); + + ndims = ZarrStreamSettings_get_dimension_count(stream_settings_); + + multiscale = ZarrStreamSettings_get_multiscale(stream_settings_); + } + + std::string uri; + if (!s3_endpoint.empty() && !s3_bucket.empty() && !store_path.empty()) { + uri = s3_endpoint + "/" + s3_bucket + "/" + store_path; + } else { + uri = "file://" + fs::absolute(store_path).string(); + } + + const size_t bytes_of_filename = uri.empty() ? 0 : uri.size() + 1; + + const char* metadata = external_metadata_json_.empty() + ? nullptr + : external_metadata_json_.c_str(); + const size_t bytes_of_metadata = + metadata ? external_metadata_json_.size() + 1 : 0; + + CHECK(storage_properties_init(props, + 0, + uri.c_str(), + bytes_of_filename, + metadata, + bytes_of_metadata, + {}, + ndims)); + + // set access key and secret + if (!access_key_id.empty() && !secret_access_key.empty()) { + CHECK(storage_properties_set_access_key_and_secret( + props, + access_key_id.c_str(), + access_key_id.size() + 1, + secret_access_key.c_str(), + secret_access_key.size() + 1)); + } + + for (auto i = 0; i < ndims; ++i) { + char name[64]; + ZarrDimensionType kind_; + size_t array_size_px, chunk_size_px, shard_size_chunks; + + if (stream_) { + ZARR_OK(ZarrStream_get_dimension(stream_, + i, + name, + sizeof(name), + &kind_, + &array_size_px, + &chunk_size_px, + &shard_size_chunks)); + } else { + ZARR_OK(ZarrStreamSettings_get_dimension(stream_settings_, + i, + name, + sizeof(name), + &kind_, + &array_size_px, + &chunk_size_px, + &shard_size_chunks)); + } + + DimensionType kind; + switch (kind_) { + case ZarrDimensionType_Space: + kind = DimensionType_Space; + break; + case ZarrDimensionType_Channel: + kind = DimensionType_Channel; + break; + case ZarrDimensionType_Time: + kind = DimensionType_Time; + break; + case ZarrDimensionType_Other: + kind = DimensionType_Other; + break; + default: + throw std::runtime_error("Invalid dimension type: " + + std::to_string(kind_)); + } + + CHECK(storage_properties_set_dimension(props, + i, + name, + sizeof(name), + kind, + array_size_px, + chunk_size_px, + shard_size_chunks)); + } + + CHECK(storage_properties_set_enable_multiscale(props, multiscale)); +} + +void +sink::Zarr::get_meta(StoragePropertyMetadata* meta) const +{ + CHECK(meta); + memset(meta, 0, sizeof(*meta)); + + meta->chunking_is_supported = 1; + meta->multiscale_is_supported = 1; + meta->s3_is_supported = 1; + meta->sharding_is_supported = + static_cast(version_ == ZarrVersion_3); +} + +void +sink::Zarr::start() +{ + EXPECT(state == DeviceState_Armed, "Device is not armed."); + EXPECT(stream_settings_, "No stream settings."); + + if (stream_) { + ZarrStream_destroy(stream_); + stream_ = nullptr; + } + + stream_ = ZarrStream_create(stream_settings_, version_); + CHECK(stream_); + stream_settings_ = nullptr; + + state = DeviceState_Running; +} + +void +sink::Zarr::stop() noexcept +{ + if (DeviceState_Running == state) { + state = DeviceState_Armed; + ZarrStream_destroy(stream_); + } +} + +size_t +sink::Zarr::append(const VideoFrame* frames, size_t nbytes) +{ + EXPECT(DeviceState_Running == state, "Device is not running."); + + if (0 == nbytes) { + return nbytes; + } + + const VideoFrame* cur = nullptr; + const auto* end = (const VideoFrame*)((uint8_t*)frames + nbytes); + auto next = [&]() -> const VideoFrame* { + const uint8_t* p = ((const uint8_t*)cur) + cur->bytes_of_frame; + return (const VideoFrame*)p; + }; + + for (cur = frames; cur < end; cur = next()) { + const size_t bytes_of_frame = bytes_of_image(&cur->shape); + size_t bytes_written; + ZARR_OK(ZarrStream_append( + stream_, cur->data, bytes_of_frame, &bytes_written)); + EXPECT(bytes_written == bytes_of_frame, + "Expected to write %zu bytes, but wrote %zu.", + bytes_of_frame, + bytes_written); + } + + return nbytes; +} + +void +sink::Zarr::reserve_image_shape(const ImageShape* shape) +{ + EXPECT(state != DeviceState_Running, + "Cannot reserve image shape while running."); + + EXPECT(stream_settings_, "No stream settings."); + + // `shape` should be verified nonnull in storage_reserve_image_shape, but + // let's check anyway + CHECK(shape); + + size_t ndims = ZarrStreamSettings_get_dimension_count(stream_settings_); + + // check that the configured dimensions match the image shape + { + char name[64]; + ZarrDimensionType kind; + size_t array_size_px, chunk_size_px, shard_size_chunks; + + ZARR_OK(ZarrStreamSettings_get_dimension(stream_settings_, + ndims - 2, + name, + sizeof(name), + &kind, + &array_size_px, + &chunk_size_px, + &shard_size_chunks)); + EXPECT(array_size_px == shape->dims.height, "Image height mismatch."); + + ZARR_OK(ZarrStreamSettings_get_dimension(stream_settings_, + ndims - 1, + name, + sizeof(name), + &kind, + &array_size_px, + &chunk_size_px, + &shard_size_chunks)); + EXPECT(array_size_px == shape->dims.width, "Image width mismatch."); + } + + ZarrDataType kind; + switch (shape->type) { + case SampleType_u8: + kind = ZarrDataType_uint8; + break; + case SampleType_u10: + case SampleType_u12: + case SampleType_u14: + case SampleType_u16: + kind = ZarrDataType_uint16; + break; + case SampleType_i8: + kind = ZarrDataType_int8; + break; + case SampleType_i16: + kind = ZarrDataType_int16; + break; + case SampleType_f32: + kind = ZarrDataType_float32; + break; + default: + throw std::runtime_error("Unsupported image type: " + + std::to_string(shape->type)); + } + + ZARR_OK(ZarrStreamSettings_set_data_type(stream_settings_, kind)); +} + +extern "C" +{ + struct Storage* zarr_v2_init() + { + try { + return new sink::Zarr( + ZarrVersion_2, ZarrCompressionCodec_None, 0, 0); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + struct Storage* compressed_zarr_v2_zstd_init() + { + try { + return new sink::Zarr( + ZarrVersion_2, ZarrCompressionCodec_BloscZstd, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + + struct Storage* compressed_zarr_v2_lz4_init() + { + try { + return new sink::Zarr( + ZarrVersion_2, ZarrCompressionCodec_BloscLZ4, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + + struct Storage* zarr_v3_init() + { + try { + return new sink::Zarr( + ZarrVersion_3, ZarrCompressionCodec_None, 0, 0); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + struct Storage* compressed_zarr_v3_zstd_init() + { + try { + return new sink::Zarr( + ZarrVersion_3, ZarrCompressionCodec_BloscZstd, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } + + struct Storage* compressed_zarr_v3_lz4_init() + { + try { + return new sink::Zarr( + ZarrVersion_3, ZarrCompressionCodec_BloscLZ4, 1, 1); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + } catch (...) { + LOGE("Exception: (unknown)"); + } + return nullptr; + } +} + +/* +void +sink::Zarr::write_multiscale_frames_(const uint8_t* const data_, + size_t bytes_of_data, + const ImageShape& shape_) +{ + auto* data = const_cast(data_); + ImageShape shape = shape_; + struct VideoFrame* dst; + + std::function scale; + std::function average2; + switch (shape.type) { + case SampleType_u10: + case SampleType_u12: + case SampleType_u14: + case SampleType_u16: + scale = ::scale_image; + average2 = ::average_two_frames; + break; + case SampleType_i8: + scale = ::scale_image; + average2 = ::average_two_frames; + break; + case SampleType_i16: + scale = ::scale_image; + average2 = ::average_two_frames; + break; + case SampleType_f32: + scale = ::scale_image; + average2 = ::average_two_frames; + break; + case SampleType_u8: + scale = ::scale_image; + average2 = ::average_two_frames; + break; + default: + char err_msg[64]; + snprintf(err_msg, + sizeof(err_msg), + "Unsupported pixel type: %s", + common::sample_type_to_string(shape.type)); + throw std::runtime_error(err_msg); + } + + for (auto i = 1; i < writers_.size(); ++i) { + dst = scale(data, bytes_of_data, shape); + if (scaled_frames_[i].has_value()) { + // average + average2(dst, scaled_frames_[i].value()); + + // write the downsampled frame + const size_t bytes_of_frame = bytes_of_image(&dst->shape); + CHECK(writers_[i]->write(dst->data, bytes_of_frame)); + + // clean up this level of detail + free(scaled_frames_[i].value()); + scaled_frames_[i].reset(); + + // setup for next iteration + if (i + 1 < writers_.size()) { + data = dst->data; + shape = dst->shape; + bytes_of_data = bytes_of_image(&shape); + } else { + // no longer needed + free(dst); + } + } else { + scaled_frames_[i] = dst; + break; + } + } +} + +#ifndef NO_UNIT_TESTS +#ifdef _WIN32 +#define acquire_export __declspec(dllexport) +#else +#define acquire_export +#endif + +///< Test that a single frame with 1 plane is padded and averaged correctly. +template +void +test_average_frame_inner(const SampleType& stype) +{ + const size_t bytes_of_frame = + common::align_up(sizeof(VideoFrame) + 9 * sizeof(T), 8); + auto* src = (VideoFrame*)malloc(bytes_of_frame); + CHECK(src); + + src->bytes_of_frame = bytes_of_frame; + src->shape = { + .dims = { + .channels = 1, + .width = 3, + .height = 3, + .planes = 1, + }, + .strides = { + .channels = 1, + .width = 1, + .height = 3, + .planes = 9 + }, + .type = stype + }; + + for (auto i = 0; i < 9; ++i) { + ((T*)src->data)[i] = (T)(i + 1); + } + + auto dst = + scale_image(src->data, bytes_of_image(&src->shape), src->shape); + CHECK(((T*)dst->data)[0] == (T)3); + CHECK(((T*)dst->data)[1] == (T)4.5); + CHECK(((T*)dst->data)[2] == (T)7.5); + CHECK(((T*)dst->data)[3] == (T)9); + + free(src); + free(dst); +} + +extern "C" acquire_export int +unit_test__average_frame() +{ + try { + test_average_frame_inner(SampleType_u8); + test_average_frame_inner(SampleType_i8); + test_average_frame_inner(SampleType_u16); + test_average_frame_inner(SampleType_i16); + test_average_frame_inner(SampleType_f32); + } catch (const std::exception& exc) { + LOGE("Exception: %s\n", exc.what()); + return 0; + } catch (...) { + LOGE("Exception: (unknown)"); + return 0; + } + + return 1; +} +#endif +*/ \ No newline at end of file diff --git a/src/driver/zarr.storage.hh b/src/driver/zarr.storage.hh new file mode 100644 index 00000000..15baccf8 --- /dev/null +++ b/src/driver/zarr.storage.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "device/kit/storage.h" + +#include "zarr.h" + +#include + +namespace acquire::sink { +struct Zarr : public Storage +{ + public: + Zarr(ZarrVersion version, + ZarrCompressionCodec compression_codec, + uint8_t compression_level, + uint8_t shuffle); + ~Zarr(); + + /// Storage interface + void set(const StorageProperties* props); + void get(StorageProperties* props) const; + void get_meta(StoragePropertyMetadata* meta) const; + void start(); + void stop() noexcept; + size_t append(const VideoFrame* frames, size_t nbytes); + void reserve_image_shape(const ImageShape* shape); + + private: + ZarrVersion version_; + ZarrCompressionCodec compression_codec_; + uint8_t compression_level_; + uint8_t shuffle_; + + std::string external_metadata_json_; + + ZarrStreamSettings* stream_settings_; + ZarrStream* stream_; +}; +} // namespace acquire::sink diff --git a/src/driver/zarr.v2.cpp b/src/driver/zarr.v2.cpp deleted file mode 100644 index 6641f1ba..00000000 --- a/src/driver/zarr.v2.cpp +++ /dev/null @@ -1,152 +0,0 @@ -#include "zarr.v2.hh" -#include "writers/zarrv2.array.writer.hh" -#include "writers/sink.creator.hh" - -#include "nlohmann/json.hpp" - -#include - -namespace zarr = acquire::sink::zarr; - -namespace { -template -struct Storage* -compressed_zarr_v2_init() -{ - try { - zarr::BloscCompressionParams params( - zarr::compression_codec_as_string(), 1, 1); - return new zarr::ZarrV2(std::move(params)); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; -} -} // end ::{anonymous} namespace - -/// ZarrV2 -zarr::ZarrV2::ZarrV2(BloscCompressionParams&& compression_params) - : Zarr(std::move(compression_params)) -{ -} - -void -zarr::ZarrV2::get_meta(StoragePropertyMetadata* meta) const -{ - Zarr::get_meta(meta); - meta->sharding_is_supported = 0; -} - -void -zarr::ZarrV2::allocate_writers_() -{ - writers_.clear(); - - ArrayWriterConfig config = { - .image_shape = image_shape_, - .dimensions = acquisition_dimensions_, - .level_of_detail = 0, - .dataset_root = dataset_root_, - .compression_params = blosc_compression_params_, - }; - - writers_.push_back(std::make_shared( - config, thread_pool_, connection_pool_)); - - if (enable_multiscale_) { - ArrayWriterConfig downsampled_config; - - bool do_downsample = true; - int level = 1; - while (do_downsample) { - do_downsample = downsample(config, downsampled_config); - writers_.push_back(std::make_shared( - downsampled_config, thread_pool_, connection_pool_)); - scaled_frames_.emplace(level++, std::nullopt); - - config = std::move(downsampled_config); - downsampled_config = {}; - } - } -} - -void -zarr::ZarrV2::make_metadata_sinks_() -{ - SinkCreator creator(thread_pool_, connection_pool_); - CHECK(creator.make_metadata_sinks( - ZarrVersion::V2, dataset_root_, metadata_sinks_)); -} - -void -zarr::ZarrV2::write_base_metadata_() const -{ - namespace fs = std::filesystem; - - json metadata; - metadata["multiscales"] = make_multiscale_metadata_(); - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at(".zattrs"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -void -zarr::ZarrV2::write_external_metadata_() const -{ - namespace fs = std::filesystem; - - std::string metadata_str = external_metadata_json_.empty() - ? "{}" - : json::parse(external_metadata_json_, - nullptr, // callback - true, // allow exceptions - true // ignore comments - ) - .dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at("0/.zattrs"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -void -zarr::ZarrV2::write_group_metadata_() const -{ - namespace fs = std::filesystem; - - const json metadata = { { "zarr_format", 2 } }; - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at(".zgroup"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -extern "C" -{ - struct Storage* zarr_v2_init() - { - try { - return new zarr::ZarrV2(); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; - } - struct Storage* compressed_zarr_v2_zstd_init() - { - return compressed_zarr_v2_init(); - } - - struct Storage* compressed_zarr_v2_lz4_init() - { - return compressed_zarr_v2_init(); - } -} diff --git a/src/driver/zarr.v2.hh b/src/driver/zarr.v2.hh deleted file mode 100644 index a83bb630..00000000 --- a/src/driver/zarr.v2.hh +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_V2_V0 -#define H_ACQUIRE_STORAGE_ZARR_V2_V0 - -#include "zarr.hh" - -namespace acquire::sink::zarr { -struct ZarrV2 final : public Zarr -{ - public: - ZarrV2() = default; - ZarrV2(BloscCompressionParams&& compression_params); - ~ZarrV2() override = default; - - /// StorageInterface - void get_meta(StoragePropertyMetadata* meta) const override; - - private: - /// Setup - void allocate_writers_() override; - - /// Metadata - void make_metadata_sinks_() override; - - // fixed metadata - void write_base_metadata_() const override; - void write_external_metadata_() const override; - - // mutable metadata, changes on flush - void write_group_metadata_() const override; -}; -} // namespace acquire::sink::zarr - -#endif // H_ACQUIRE_STORAGE_ZARR_V2_V0 diff --git a/src/driver/zarr.v3.cpp b/src/driver/zarr.v3.cpp deleted file mode 100644 index 3a35a958..00000000 --- a/src/driver/zarr.v3.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include "zarr.v3.hh" -#include "writers/zarrv3.array.writer.hh" -#include "writers/sink.creator.hh" - -#include "nlohmann/json.hpp" - -#include - -namespace zarr = acquire::sink::zarr; - -namespace { -template -struct Storage* -compressed_zarr_v3_init() -{ - try { - zarr::BloscCompressionParams params( - zarr::compression_codec_as_string(), 1, 1); - return new zarr::ZarrV3(std::move(params)); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; -} -} // end ::{anonymous} namespace - -zarr::ZarrV3::ZarrV3(BloscCompressionParams&& compression_params) - : Zarr(std::move(compression_params)) -{ -} - -void -zarr::ZarrV3::allocate_writers_() -{ - writers_.clear(); - - ArrayWriterConfig config = { - .image_shape = image_shape_, - .dimensions = acquisition_dimensions_, - .level_of_detail = 0, - .dataset_root = dataset_root_, - .compression_params = blosc_compression_params_, - }; - - writers_.push_back(std::make_shared( - config, thread_pool_, connection_pool_)); - - if (enable_multiscale_) { - ArrayWriterConfig downsampled_config; - - bool do_downsample = true; - int level = 1; - while (do_downsample) { - do_downsample = downsample(config, downsampled_config); - writers_.push_back(std::make_shared( - downsampled_config, thread_pool_, connection_pool_)); - scaled_frames_.emplace(level++, std::nullopt); - - config = std::move(downsampled_config); - downsampled_config = {}; - } - } -} - -void -zarr::ZarrV3::get_meta(StoragePropertyMetadata* meta) const -{ - Zarr::get_meta(meta); - meta->sharding_is_supported = 1; -} - -void -zarr::ZarrV3::make_metadata_sinks_() -{ - SinkCreator creator(thread_pool_, connection_pool_); - CHECK(creator.make_metadata_sinks( - ZarrVersion::V3, dataset_root_, metadata_sinks_)); -} - -/// @brief Write the metadata for the dataset. -void -zarr::ZarrV3::write_base_metadata_() const -{ - namespace fs = std::filesystem; - - json metadata; - metadata["extensions"] = json::array(); - metadata["metadata_encoding"] = - "https://purl.org/zarr/spec/protocol/core/3.0"; - metadata["metadata_key_suffix"] = ".json"; - metadata["zarr_format"] = "https://purl.org/zarr/spec/protocol/core/3.0"; - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = metadata_sinks_.at("zarr.json"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -/// @brief Write the external metadata. -/// @details This is a no-op for ZarrV3. Instead, external metadata is -/// stored in the group metadata. -void -zarr::ZarrV3::write_external_metadata_() const -{ - // no-op -} - -/// @brief Write the metadata for the group. -/// @details Zarr v3 stores group metadata in -/// /meta/{group_name}.group.json. We will call the group "root". -void -zarr::ZarrV3::write_group_metadata_() const -{ - namespace fs = std::filesystem; - - json metadata; - metadata["attributes"]["acquire"] = - external_metadata_json_.empty() ? "" - : json::parse(external_metadata_json_, - nullptr, // callback - true, // allow exceptions - true // ignore comments - ); - metadata["attributes"]["multiscales"] = make_multiscale_metadata_(); - - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); - const std::unique_ptr& sink = - metadata_sinks_.at("meta/root.group.json"); - CHECK(sink); - CHECK(sink->write(0, metadata_bytes, metadata_str.size())); -} - -extern "C" -{ - struct Storage* zarr_v3_init() - { - try { - return new zarr::ZarrV3(); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - return nullptr; - } - struct Storage* compressed_zarr_v3_zstd_init() - { - return compressed_zarr_v3_init(); - } - - struct Storage* compressed_zarr_v3_lz4_init() - { - return compressed_zarr_v3_init(); - } -} diff --git a/src/driver/zarr.v3.hh b/src/driver/zarr.v3.hh deleted file mode 100644 index d85d6f5b..00000000 --- a/src/driver/zarr.v3.hh +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef H_ACQUIRE_STORAGE_ZARR_V3_V0 -#define H_ACQUIRE_STORAGE_ZARR_V3_V0 - -#include "zarr.hh" - -namespace acquire::sink::zarr { -struct ZarrV3 final : public Zarr -{ - public: - ZarrV3() = default; - explicit ZarrV3(BloscCompressionParams&& compression_params); - ~ZarrV3() override = default; - - /// Storage interface - void get_meta(StoragePropertyMetadata* meta) const override; - - private: - /// Setup - void allocate_writers_() override; - - /// Metadata - void make_metadata_sinks_() override; - - // fixed metadata - void write_base_metadata_() const override; - void write_external_metadata_() const override; - - // mutable metadata, changes on flush - void write_group_metadata_() const override; -}; -} // namespace acquire::sink::zarr -#endif // H_ACQUIRE_STORAGE_ZARR_V3_V0 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c98967ff..ba0157ff 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,5 +3,5 @@ if (${NOTEST}) else () add_subdirectory(unit-tests) add_subdirectory(integration) -# add_subdirectory(driver) + add_subdirectory(driver) endif () diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt index 68850c70..ef7ec66e 100644 --- a/tests/driver/CMakeLists.txt +++ b/tests/driver/CMakeLists.txt @@ -12,7 +12,6 @@ set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is # set(tests list-devices - unit-tests get get-meta get-set-get diff --git a/tests/driver/unit-tests.cpp b/tests/driver/unit-tests.cpp deleted file mode 100644 index e02880b8..00000000 --- a/tests/driver/unit-tests.cpp +++ /dev/null @@ -1,120 +0,0 @@ -// This is a "unit test" driver. -// -// Adding unit test functions here will run them as part of the CTest suite -// in a standardized fashion. -// -// Unit tests should be focused on testing the smallest logically isolated -// parts of the code. Practically, this means they should live close to the -// code they're testing. That is usually under the public interface -// defined by this module - if you're test uses a private interface that's a -// good sign it might be a unit test. -// -// Adding a new unit test: -// 1. Define your unit test in the same source file as what you're testing. -// 2. Add it to the declarations list below. See TEST DECLARATIONS. -// 3. Add it to the test list. See TEST LIST. -// -// Template: -// -// ```c -// #ifndef NO_UNIT_TESTS -// int -// unit_test__my_descriptive_test_name() -// { -// // do stuff -// return 1; // success -// Error: -// return 0; // failure -// } -// #endif // NO_UNIT_TESTS -// ``` - -#include "platform.h" -#include "logger.h" - -#include -#include -#include - -#define L (aq_logger) -#define LOG(...) L(0, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) -#define ERR(...) L(1, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__) - -void -reporter(int is_error, - const char* file, - int line, - const char* function, - const char* msg) -{ - fprintf(is_error ? stderr : stdout, - "%s%s(%d) - %s: %s\n", - is_error ? "ERROR " : "", - file, - line, - function, - msg); -} - -typedef struct Driver* (*init_func_t)(void (*reporter)(int is_error, - const char* file, - int line, - const char* function, - const char* msg)); -// -// TEST DRIVER -// - -int -main() -{ - logger_set_reporter(reporter); - struct lib lib = { 0 }; - if (!lib_open_by_name(&lib, "acquire-driver-zarr")) { - ERR("Failed to open \"acquire-driver-zarr\"."); - exit(2); - } - - struct testcase - { - const char* name; - int (*test)(); - }; - const std::vector tests{ -#define CASE(e) { .name = #e, .test = (int (*)())lib_load(&lib, #e) } - CASE(unit_test__trim), - CASE(unit_test__split_uri), - CASE(unit_test__shard_index_for_chunk), - CASE(unit_test__shard_internal_index), - CASE(unit_test__average_frame), - CASE(unit_test__thread_pool__push_to_job_queue), - CASE(unit_test__s3_connection__put_object), - CASE(unit_test__s3_connection__upload_multipart_object), - CASE(unit_test__sink_creator__create_chunk_file_sinks), - CASE(unit_test__sink_creator__create_shard_file_sinks), - CASE(unit_test__chunk_lattice_index), - CASE(unit_test__tile_group_offset), - CASE(unit_test__chunk_internal_offset), - CASE(unit_test__writer__write_frame_to_chunks), - CASE(unit_test__downsample_writer_config), - CASE(unit_test__writer__write_frame_to_chunks), - CASE(unit_test__zarrv2_writer__write_even), - CASE(unit_test__zarrv2_writer__write_ragged_append_dim), - CASE(unit_test__zarrv2_writer__write_ragged_internal_dim), - CASE(unit_test__zarrv3_writer__write_even), - CASE(unit_test__zarrv3_writer__write_ragged_append_dim), - CASE(unit_test__zarrv3_writer__write_ragged_internal_dim), -#undef CASE - }; - - bool any = false; - for (const auto& test : tests) { - LOG("Running %s", test.name); - if (!(test.test())) { - ERR("unit test failed: %s", test.name); - any = true; - } - } - lib_close(&lib); - return any; -} diff --git a/tests/driver/write-zarr-v2-raw.cpp b/tests/driver/write-zarr-v2-raw.cpp index d213ecad..0a352b21 100644 --- a/tests/driver/write-zarr-v2-raw.cpp +++ b/tests/driver/write-zarr-v2-raw.cpp @@ -104,28 +104,28 @@ acquire(AcquireRuntime* runtime, const char* filename) 4); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 0, + 3, SIZED("x") + 1, DimensionType_Space, frame_width, frame_width, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 1, + 2, SIZED("y") + 1, DimensionType_Space, frame_height, frame_height, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 2, + 1, SIZED("c") + 1, DimensionType_Channel, 1, 1, 0)); CHECK(storage_properties_set_dimension(&props.video[0].storage.settings, - 3, + 0, SIZED("t") + 1, DimensionType_Time, 0,