From c7832d724950c7ce0ebec3797d716dd203ece9bc Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 09:59:22 -0400 Subject: [PATCH 01/39] Move driver source files and tests to a separate directory. --- src/CMakeLists.txt | 58 +------------ src/driver/CMakeLists.txt | 61 +++++++++++++ src/{ => driver}/README.md | 0 src/{ => driver}/common/dimension.cpp | 0 src/{ => driver}/common/dimension.hh | 0 src/{ => driver}/common/macros.hh | 0 src/{ => driver}/common/s3.connection.cpp | 0 src/{ => driver}/common/s3.connection.hh | 0 src/{ => driver}/common/thread.pool.cpp | 0 src/{ => driver}/common/thread.pool.hh | 0 src/{ => driver}/common/utilities.cpp | 0 src/{ => driver}/common/utilities.hh | 0 src/{ => driver}/writers/array.writer.cpp | 0 src/{ => driver}/writers/array.writer.hh | 0 src/{ => driver}/writers/blosc.compressor.cpp | 0 src/{ => driver}/writers/blosc.compressor.hh | 0 src/{ => driver}/writers/file.sink.cpp | 0 src/{ => driver}/writers/file.sink.hh | 0 src/{ => driver}/writers/s3.sink.cpp | 0 src/{ => driver}/writers/s3.sink.hh | 0 src/{ => driver}/writers/sink.creator.cpp | 0 src/{ => driver}/writers/sink.creator.hh | 0 src/{ => driver}/writers/sink.hh | 0 .../writers/zarrv2.array.writer.cpp | 0 .../writers/zarrv2.array.writer.hh | 0 .../writers/zarrv3.array.writer.cpp | 0 .../writers/zarrv3.array.writer.hh | 0 src/{ => driver}/zarr.cpp | 0 src/{ => driver}/zarr.driver.c | 0 src/{ => driver}/zarr.hh | 0 src/{ => driver}/zarr.v2.cpp | 0 src/{ => driver}/zarr.v2.hh | 0 src/{ => driver}/zarr.v3.cpp | 0 src/{ => driver}/zarr.v3.hh | 0 tests/CMakeLists.txt | 87 +------------------ tests/driver/CMakeLists.txt | 87 +++++++++++++++++++ tests/{ => driver}/README.md | 0 .../external-metadata-with-whitespace-ok.cpp | 0 tests/{ => driver}/get-meta.cpp | 0 tests/{ => driver}/get-set-get.cpp | 0 tests/{ => driver}/get.cpp | 0 tests/{ => driver}/list-devices.cpp | 0 .../{ => driver}/metadata-dimension-sizes.cpp | 0 tests/{ => driver}/multiscales-metadata.cpp | 0 tests/{ => driver}/repeat-start.cpp | 0 ...restart-stopped-zarr-resets-threadpool.cpp | 0 tests/{ => driver}/unit-tests.cpp | 0 .../write-zarr-v2-compressed-multiscale.cpp | 0 ...-compressed-with-chunking-and-rollover.cpp | 0 ...write-zarr-v2-compressed-with-chunking.cpp | 0 ...-raw-chunk-size-larger-than-frame-size.cpp | 0 ...-raw-multiscale-with-trivial-tile-size.cpp | 0 .../write-zarr-v2-raw-multiscale.cpp | 0 ...v2-raw-with-even-chunking-and-rollover.cpp | 0 .../write-zarr-v2-raw-with-even-chunking.cpp | 0 ...write-zarr-v2-raw-with-ragged-chunking.cpp | 0 tests/{ => driver}/write-zarr-v2-raw.cpp | 0 tests/{ => driver}/write-zarr-v2-to-s3.cpp | 0 .../write-zarr-v2-with-lz4-compression.cpp | 0 .../write-zarr-v2-with-zstd-compression.cpp | 0 .../{ => driver}/write-zarr-v3-compressed.cpp | 0 .../write-zarr-v3-raw-chunk-exceeds-array.cpp | 0 .../write-zarr-v3-raw-multiscale.cpp | 0 ...write-zarr-v3-raw-with-ragged-sharding.cpp | 0 tests/{ => driver}/write-zarr-v3-raw.cpp | 0 tests/{ => driver}/write-zarr-v3-to-s3.cpp | 0 66 files changed, 151 insertions(+), 142 deletions(-) create mode 100644 src/driver/CMakeLists.txt rename src/{ => driver}/README.md (100%) rename src/{ => driver}/common/dimension.cpp (100%) rename src/{ => driver}/common/dimension.hh (100%) rename src/{ => driver}/common/macros.hh (100%) rename src/{ => driver}/common/s3.connection.cpp (100%) rename src/{ => driver}/common/s3.connection.hh (100%) rename src/{ => driver}/common/thread.pool.cpp (100%) rename src/{ => driver}/common/thread.pool.hh (100%) rename src/{ => driver}/common/utilities.cpp (100%) rename src/{ => driver}/common/utilities.hh (100%) rename src/{ => driver}/writers/array.writer.cpp (100%) rename src/{ => driver}/writers/array.writer.hh (100%) rename src/{ => driver}/writers/blosc.compressor.cpp (100%) rename src/{ => driver}/writers/blosc.compressor.hh (100%) rename src/{ => driver}/writers/file.sink.cpp (100%) rename src/{ => driver}/writers/file.sink.hh (100%) rename src/{ => driver}/writers/s3.sink.cpp (100%) rename src/{ => driver}/writers/s3.sink.hh (100%) rename src/{ => driver}/writers/sink.creator.cpp (100%) rename src/{ => driver}/writers/sink.creator.hh (100%) rename src/{ => driver}/writers/sink.hh (100%) rename src/{ => driver}/writers/zarrv2.array.writer.cpp (100%) rename src/{ => driver}/writers/zarrv2.array.writer.hh (100%) rename src/{ => driver}/writers/zarrv3.array.writer.cpp (100%) rename src/{ => driver}/writers/zarrv3.array.writer.hh (100%) rename src/{ => driver}/zarr.cpp (100%) rename src/{ => driver}/zarr.driver.c (100%) rename src/{ => driver}/zarr.hh (100%) rename src/{ => driver}/zarr.v2.cpp (100%) rename src/{ => driver}/zarr.v2.hh (100%) rename src/{ => driver}/zarr.v3.cpp (100%) rename src/{ => driver}/zarr.v3.hh (100%) create mode 100644 tests/driver/CMakeLists.txt rename tests/{ => driver}/README.md (100%) rename tests/{ => driver}/external-metadata-with-whitespace-ok.cpp (100%) rename tests/{ => driver}/get-meta.cpp (100%) rename tests/{ => driver}/get-set-get.cpp (100%) rename tests/{ => driver}/get.cpp (100%) rename tests/{ => driver}/list-devices.cpp (100%) rename tests/{ => driver}/metadata-dimension-sizes.cpp (100%) rename tests/{ => driver}/multiscales-metadata.cpp (100%) rename tests/{ => driver}/repeat-start.cpp (100%) rename tests/{ => driver}/restart-stopped-zarr-resets-threadpool.cpp (100%) rename tests/{ => driver}/unit-tests.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-with-chunking-and-rollover.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-with-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-even-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-ragged-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw.cpp (100%) rename tests/{ => driver}/write-zarr-v2-to-s3.cpp (100%) rename tests/{ => driver}/write-zarr-v2-with-lz4-compression.cpp (100%) rename tests/{ => driver}/write-zarr-v2-with-zstd-compression.cpp (100%) rename tests/{ => driver}/write-zarr-v3-compressed.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-chunk-exceeds-array.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-with-ragged-sharding.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw.cpp (100%) rename tests/{ => driver}/write-zarr-v3-to-s3.cpp (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9e6668d6..f9a3b42a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,57 +1,3 @@ -if (NOT TARGET acquire-core-logger) - add_subdirectory(../acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) -endif () +set(CMAKE_POSITION_INDEPENDENT_CODE ON) -set(tgt acquire-driver-zarr) -add_library(${tgt} MODULE - common/dimension.hh - common/dimension.cpp - common/thread.pool.hh - common/thread.pool.cpp - common/s3.connection.hh - common/s3.connection.cpp - common/utilities.hh - common/utilities.cpp - writers/sink.hh - writers/sink.creator.hh - writers/sink.creator.cpp - writers/file.sink.hh - writers/file.sink.cpp - writers/s3.sink.hh - writers/s3.sink.cpp - writers/array.writer.hh - writers/array.writer.cpp - writers/zarrv2.array.writer.hh - writers/zarrv2.array.writer.cpp - writers/zarrv3.array.writer.hh - writers/zarrv3.array.writer.cpp - writers/blosc.compressor.hh - writers/blosc.compressor.cpp - zarr.hh - zarr.cpp - zarr.v2.hh - zarr.v2.cpp - zarr.v3.hh - zarr.v3.cpp - zarr.driver.c -) - -target_include_directories(${tgt} PRIVATE - $ -) - -target_enable_simd(${tgt}) -target_link_libraries(${tgt} PRIVATE - acquire-core-logger - acquire-core-platform - acquire-device-kit - acquire-device-properties - blosc_static - nlohmann_json::nlohmann_json - miniocpp::miniocpp -) -set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -) - -install(TARGETS ${tgt} LIBRARY DESTINATION lib) +add_subdirectory(driver) diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt new file mode 100644 index 00000000..e223b5db --- /dev/null +++ b/src/driver/CMakeLists.txt @@ -0,0 +1,61 @@ +option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) + +if (BUILD_ACQUIRE_DRIVER_ZARR) + if (NOT TARGET acquire-core-logger) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) + endif () + + set(tgt acquire-driver-zarr) + add_library(${tgt} MODULE + common/dimension.hh + common/dimension.cpp + common/thread.pool.hh + common/thread.pool.cpp + common/s3.connection.hh + common/s3.connection.cpp + common/utilities.hh + common/utilities.cpp + writers/sink.hh + writers/sink.creator.hh + writers/sink.creator.cpp + writers/file.sink.hh + writers/file.sink.cpp + writers/s3.sink.hh + writers/s3.sink.cpp + writers/array.writer.hh + writers/array.writer.cpp + writers/zarrv2.array.writer.hh + writers/zarrv2.array.writer.cpp + writers/zarrv3.array.writer.hh + writers/zarrv3.array.writer.cpp + writers/blosc.compressor.hh + writers/blosc.compressor.cpp + zarr.hh + zarr.cpp + zarr.v2.hh + zarr.v2.cpp + zarr.v3.hh + zarr.v3.cpp + zarr.driver.c + ) + + target_include_directories(${tgt} PRIVATE + $ + ) + + target_enable_simd(${tgt}) + target_link_libraries(${tgt} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + + install(TARGETS ${tgt} LIBRARY DESTINATION lib) +endif () \ No newline at end of file diff --git a/src/README.md b/src/driver/README.md similarity index 100% rename from src/README.md rename to src/driver/README.md diff --git a/src/common/dimension.cpp b/src/driver/common/dimension.cpp similarity index 100% rename from src/common/dimension.cpp rename to src/driver/common/dimension.cpp diff --git a/src/common/dimension.hh b/src/driver/common/dimension.hh similarity index 100% rename from src/common/dimension.hh rename to src/driver/common/dimension.hh diff --git a/src/common/macros.hh b/src/driver/common/macros.hh similarity index 100% rename from src/common/macros.hh rename to src/driver/common/macros.hh diff --git a/src/common/s3.connection.cpp b/src/driver/common/s3.connection.cpp similarity index 100% rename from src/common/s3.connection.cpp rename to src/driver/common/s3.connection.cpp diff --git a/src/common/s3.connection.hh b/src/driver/common/s3.connection.hh similarity index 100% rename from src/common/s3.connection.hh rename to src/driver/common/s3.connection.hh diff --git a/src/common/thread.pool.cpp b/src/driver/common/thread.pool.cpp similarity index 100% rename from src/common/thread.pool.cpp rename to src/driver/common/thread.pool.cpp diff --git a/src/common/thread.pool.hh b/src/driver/common/thread.pool.hh similarity index 100% rename from src/common/thread.pool.hh rename to src/driver/common/thread.pool.hh diff --git a/src/common/utilities.cpp b/src/driver/common/utilities.cpp similarity index 100% rename from src/common/utilities.cpp rename to src/driver/common/utilities.cpp diff --git a/src/common/utilities.hh b/src/driver/common/utilities.hh similarity index 100% rename from src/common/utilities.hh rename to src/driver/common/utilities.hh diff --git a/src/writers/array.writer.cpp b/src/driver/writers/array.writer.cpp similarity index 100% rename from src/writers/array.writer.cpp rename to src/driver/writers/array.writer.cpp diff --git a/src/writers/array.writer.hh b/src/driver/writers/array.writer.hh similarity index 100% rename from src/writers/array.writer.hh rename to src/driver/writers/array.writer.hh diff --git a/src/writers/blosc.compressor.cpp b/src/driver/writers/blosc.compressor.cpp similarity index 100% rename from src/writers/blosc.compressor.cpp rename to src/driver/writers/blosc.compressor.cpp diff --git a/src/writers/blosc.compressor.hh b/src/driver/writers/blosc.compressor.hh similarity index 100% rename from src/writers/blosc.compressor.hh rename to src/driver/writers/blosc.compressor.hh diff --git a/src/writers/file.sink.cpp b/src/driver/writers/file.sink.cpp similarity index 100% rename from src/writers/file.sink.cpp rename to src/driver/writers/file.sink.cpp diff --git a/src/writers/file.sink.hh b/src/driver/writers/file.sink.hh similarity index 100% rename from src/writers/file.sink.hh rename to src/driver/writers/file.sink.hh diff --git a/src/writers/s3.sink.cpp b/src/driver/writers/s3.sink.cpp similarity index 100% rename from src/writers/s3.sink.cpp rename to src/driver/writers/s3.sink.cpp diff --git a/src/writers/s3.sink.hh b/src/driver/writers/s3.sink.hh similarity index 100% rename from src/writers/s3.sink.hh rename to src/driver/writers/s3.sink.hh diff --git a/src/writers/sink.creator.cpp b/src/driver/writers/sink.creator.cpp similarity index 100% rename from src/writers/sink.creator.cpp rename to src/driver/writers/sink.creator.cpp diff --git a/src/writers/sink.creator.hh b/src/driver/writers/sink.creator.hh similarity index 100% rename from src/writers/sink.creator.hh rename to src/driver/writers/sink.creator.hh diff --git a/src/writers/sink.hh b/src/driver/writers/sink.hh similarity index 100% rename from src/writers/sink.hh rename to src/driver/writers/sink.hh diff --git a/src/writers/zarrv2.array.writer.cpp b/src/driver/writers/zarrv2.array.writer.cpp similarity index 100% rename from src/writers/zarrv2.array.writer.cpp rename to src/driver/writers/zarrv2.array.writer.cpp diff --git a/src/writers/zarrv2.array.writer.hh b/src/driver/writers/zarrv2.array.writer.hh similarity index 100% rename from src/writers/zarrv2.array.writer.hh rename to src/driver/writers/zarrv2.array.writer.hh diff --git a/src/writers/zarrv3.array.writer.cpp b/src/driver/writers/zarrv3.array.writer.cpp similarity index 100% rename from src/writers/zarrv3.array.writer.cpp rename to src/driver/writers/zarrv3.array.writer.cpp diff --git a/src/writers/zarrv3.array.writer.hh b/src/driver/writers/zarrv3.array.writer.hh similarity index 100% rename from src/writers/zarrv3.array.writer.hh rename to src/driver/writers/zarrv3.array.writer.hh diff --git a/src/zarr.cpp b/src/driver/zarr.cpp similarity index 100% rename from src/zarr.cpp rename to src/driver/zarr.cpp diff --git a/src/zarr.driver.c b/src/driver/zarr.driver.c similarity index 100% rename from src/zarr.driver.c rename to src/driver/zarr.driver.c diff --git a/src/zarr.hh b/src/driver/zarr.hh similarity index 100% rename from src/zarr.hh rename to src/driver/zarr.hh diff --git a/src/zarr.v2.cpp b/src/driver/zarr.v2.cpp similarity index 100% rename from src/zarr.v2.cpp rename to src/driver/zarr.v2.cpp diff --git a/src/zarr.v2.hh b/src/driver/zarr.v2.hh similarity index 100% rename from src/zarr.v2.hh rename to src/driver/zarr.v2.hh diff --git a/src/zarr.v3.cpp b/src/driver/zarr.v3.cpp similarity index 100% rename from src/zarr.v3.cpp rename to src/driver/zarr.v3.cpp diff --git a/src/zarr.v3.hh b/src/driver/zarr.v3.hh similarity index 100% rename from src/zarr.v3.hh rename to src/driver/zarr.v3.hh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc4c2f4b..87878b37 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,90 +1,5 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () - set(NOTEST "TRUE") - add_subdirectory(../acquire-common/acquire-driver-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-driver-common) - add_subdirectory(../acquire-common/acquire-video-runtime ${CMAKE_CURRENT_BINARY_DIR}/acquire-video-runtime) - set(NOTEST "FALSE") - - # - # PARAMETERS - # - set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project - - # - # Tests - # - set(tests - list-devices - unit-tests - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 - ) - - foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") - endforeach () - - # - # Copy driver to tests - # - list(POP_FRONT tests onename) - - foreach (driver - acquire-driver-common - acquire-driver-zarr - ) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" - ) - - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) - endforeach () - endforeach () + add_subdirectory(driver) endif () diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt new file mode 100644 index 00000000..1fd661a2 --- /dev/null +++ b/tests/driver/CMakeLists.txt @@ -0,0 +1,87 @@ +if (BUILD_ACQUIRE_DRIVER_ZARR) + set(NOTEST "TRUE") + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) + set(NOTEST "FALSE") + + # + # PARAMETERS + # + set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project + + # + # Tests + # + set(tests + list-devices + unit-tests + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 + ) + + foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") + endforeach () + + # + # Copy driver to tests + # + list(POP_FRONT tests onename) + + foreach (driver + acquire-driver-common + acquire-driver-zarr + ) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" + ) + + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) + endforeach () + endforeach () +endif () \ No newline at end of file diff --git a/tests/README.md b/tests/driver/README.md similarity index 100% rename from tests/README.md rename to tests/driver/README.md diff --git a/tests/external-metadata-with-whitespace-ok.cpp b/tests/driver/external-metadata-with-whitespace-ok.cpp similarity index 100% rename from tests/external-metadata-with-whitespace-ok.cpp rename to tests/driver/external-metadata-with-whitespace-ok.cpp diff --git a/tests/get-meta.cpp b/tests/driver/get-meta.cpp similarity index 100% rename from tests/get-meta.cpp rename to tests/driver/get-meta.cpp diff --git a/tests/get-set-get.cpp b/tests/driver/get-set-get.cpp similarity index 100% rename from tests/get-set-get.cpp rename to tests/driver/get-set-get.cpp diff --git a/tests/get.cpp b/tests/driver/get.cpp similarity index 100% rename from tests/get.cpp rename to tests/driver/get.cpp diff --git a/tests/list-devices.cpp b/tests/driver/list-devices.cpp similarity index 100% rename from tests/list-devices.cpp rename to tests/driver/list-devices.cpp diff --git a/tests/metadata-dimension-sizes.cpp b/tests/driver/metadata-dimension-sizes.cpp similarity index 100% rename from tests/metadata-dimension-sizes.cpp rename to tests/driver/metadata-dimension-sizes.cpp diff --git a/tests/multiscales-metadata.cpp b/tests/driver/multiscales-metadata.cpp similarity index 100% rename from tests/multiscales-metadata.cpp rename to tests/driver/multiscales-metadata.cpp diff --git a/tests/repeat-start.cpp b/tests/driver/repeat-start.cpp similarity index 100% rename from tests/repeat-start.cpp rename to tests/driver/repeat-start.cpp diff --git a/tests/restart-stopped-zarr-resets-threadpool.cpp b/tests/driver/restart-stopped-zarr-resets-threadpool.cpp similarity index 100% rename from tests/restart-stopped-zarr-resets-threadpool.cpp rename to tests/driver/restart-stopped-zarr-resets-threadpool.cpp diff --git a/tests/unit-tests.cpp b/tests/driver/unit-tests.cpp similarity index 100% rename from tests/unit-tests.cpp rename to tests/driver/unit-tests.cpp diff --git a/tests/write-zarr-v2-compressed-multiscale.cpp b/tests/driver/write-zarr-v2-compressed-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-multiscale.cpp rename to tests/driver/write-zarr-v2-compressed-multiscale.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking.cpp diff --git a/tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp b/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp rename to tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp b/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp rename to tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale.cpp b/tests/driver/write-zarr-v2-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale.cpp rename to tests/driver/write-zarr-v2-raw-multiscale.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking.cpp diff --git a/tests/write-zarr-v2-raw-with-ragged-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-ragged-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp diff --git a/tests/write-zarr-v2-raw.cpp b/tests/driver/write-zarr-v2-raw.cpp similarity index 100% rename from tests/write-zarr-v2-raw.cpp rename to tests/driver/write-zarr-v2-raw.cpp diff --git a/tests/write-zarr-v2-to-s3.cpp b/tests/driver/write-zarr-v2-to-s3.cpp similarity index 100% rename from tests/write-zarr-v2-to-s3.cpp rename to tests/driver/write-zarr-v2-to-s3.cpp diff --git a/tests/write-zarr-v2-with-lz4-compression.cpp b/tests/driver/write-zarr-v2-with-lz4-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-lz4-compression.cpp rename to tests/driver/write-zarr-v2-with-lz4-compression.cpp diff --git a/tests/write-zarr-v2-with-zstd-compression.cpp b/tests/driver/write-zarr-v2-with-zstd-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-zstd-compression.cpp rename to tests/driver/write-zarr-v2-with-zstd-compression.cpp diff --git a/tests/write-zarr-v3-compressed.cpp b/tests/driver/write-zarr-v3-compressed.cpp similarity index 100% rename from tests/write-zarr-v3-compressed.cpp rename to tests/driver/write-zarr-v3-compressed.cpp diff --git a/tests/write-zarr-v3-raw-chunk-exceeds-array.cpp b/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp similarity index 100% rename from tests/write-zarr-v3-raw-chunk-exceeds-array.cpp rename to tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp diff --git a/tests/write-zarr-v3-raw-multiscale.cpp b/tests/driver/write-zarr-v3-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v3-raw-multiscale.cpp rename to tests/driver/write-zarr-v3-raw-multiscale.cpp diff --git a/tests/write-zarr-v3-raw-with-ragged-sharding.cpp b/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp similarity index 100% rename from tests/write-zarr-v3-raw-with-ragged-sharding.cpp rename to tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp diff --git a/tests/write-zarr-v3-raw.cpp b/tests/driver/write-zarr-v3-raw.cpp similarity index 100% rename from tests/write-zarr-v3-raw.cpp rename to tests/driver/write-zarr-v3-raw.cpp diff --git a/tests/write-zarr-v3-to-s3.cpp b/tests/driver/write-zarr-v3-to-s3.cpp similarity index 100% rename from tests/write-zarr-v3-to-s3.cpp rename to tests/driver/write-zarr-v3-to-s3.cpp From a811e221925bf2f4886c13a830bf038289788bfb Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 10:06:22 -0400 Subject: [PATCH 02/39] Define the Zarr streaming API. --- include/zarr.h | 216 +++++++++++++++++++++++++++++++++++++++++++ include/zarr.types.h | 133 ++++++++++++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 include/zarr.h create mode 100644 include/zarr.types.h diff --git a/include/zarr.h b/include/zarr.h new file mode 100644 index 00000000..eac50f35 --- /dev/null +++ b/include/zarr.h @@ -0,0 +1,216 @@ +#ifndef H_ACQUIRE_ZARR_V0 +#define H_ACQUIRE_ZARR_V0 + +#include "zarr.types.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef struct ZarrStreamSettings_s ZarrStreamSettings; + typedef struct ZarrStream_s ZarrStream; + + /** + * @brief Get the version of the Zarr API. + * @return The version of the Zarr API. + */ + uint32_t Zarr_get_api_version(); + + /** + * @brief Get the message for the given status code. + * @param status The status code. + * @return A human-readable status message. + */ + const char* Zarr_get_error_message(ZarrStatus status); + + /** + * @brief Create a Zarr stream settings struct. + * @return A pointer to the Zarr stream settings struct, or NULL on failure. + */ + ZarrStreamSettings* ZarrStreamSettings_create(); + + /** + * @brief Destroy a Zarr stream settings struct. + * @details This function frees the memory allocated for the Zarr stream + * settings struct. + * @param[in] settings The Zarr stream settings struct. + */ + void ZarrStreamSettings_destroy(ZarrStreamSettings* settings); + + /** + * @brief Copy a Zarr stream settings struct. + * @param[in] settings The Zarr stream settings struct to copy. + * @return A copy of the Zarr stream settings struct. + */ + ZarrStreamSettings* ZarrStreamSettings_copy( + const ZarrStreamSettings* settings); + + /** + * @brief Set store path and S3 settings for the Zarr stream. + * @param[in, out] settings + * @param[in] store_path The store path for the Zarr stream. Directory path + * when acquiring to the filesystem, key prefix when acquiring to S3. + * @param[in] bytes_of_store_path The length of @p store_path in bytes, + * including the null terminator. + * @param[in] s3_settings Optional S3 settings. If NULL, the store path is + * assumed to be a directory path. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings); + + /** + * @brief Set the data type, compressor, codec, compression_settings level, + * and shuffle for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] compression_settings The compression_settings settings. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_compression( + ZarrStreamSettings* settings, + const ZarrCompressionSettings* compression_settings); + + /** + * @brief Set the data type for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] data_type The data type. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type); + + /** + * @brief Reserve space for dimensions in the Zarr stream settings struct. + * @detail *Must* precede calls to ZarrStreamSettings_set_dimension. We + * require at least 3 dimensions to validate settings, but you may set up to + * 32 dimensions. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] count The number of dimensions to reserve space for. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_reserve_dimensions( + ZarrStreamSettings* settings, + size_t count); + + /** + * @brief Set properties for an acquisition dimension. + * @detail The order of the dimensions in the Zarr stream is the order in + * which they are set. The first dimension set is the slowest varying + * dimension, and the last dimension set is the fastest varying dimension. + * For example, if the dimensions are set in the order z, y, x, the fastest + * varying dimension is x, the next fastest varying dimension is y, and the + * slowest varying dimension is z. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] index The index of the dimension to set. Must be less than the + * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. + * @param[in] dimension The dimension's settings. + */ + ZarrStatus ZarrStreamSettings_set_dimension( + ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension); + + /** + * @brief Set the multiscale flag for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] multiscale A flag indicating whether to stream to multiple + * levels of detail. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale); + + /** + * @brief Set JSON-formatted custom metadata for the Zarr stream. + * @details This metadata will be written to acquire-zarr.json in the + * metadata directory of the Zarr store. This parameter is optional. + * @param settings[in, out] settings The Zarr stream settings struct. + * @param external_metadata JSON-formatted external metadata. + * @param bytes_of_external_metadata The length of @p custom_metadata in + * bytes, including the null terminator. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_custom_metadata( + ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata); + + const char* ZarrStreamSettings_get_store_path( + const ZarrStreamSettings* settings); + + ZarrS3Settings ZarrStreamSettings_get_s3_settings( + const ZarrStreamSettings* settings); + + ZarrCompressionSettings ZarrStreamSettings_get_compression( + const ZarrStreamSettings* settings); + + ZarrDataType ZarrStreamSettings_get_data_type( + const ZarrStreamSettings* settings); + + size_t ZarrStreamSettings_get_dimension_count( + const ZarrStreamSettings* settings); + + ZarrDimensionProperties ZarrStreamSettings_get_dimension( + const ZarrStreamSettings* settings, + size_t index); + + bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); + + const char* ZarrStreamSettings_get_custom_metadata( + const ZarrStreamSettings* settings); + + /** + * @brief Create a Zarr stream. + * @param[in, out] settings The settings for the Zarr stream. + * @param[in] version The version of the Zarr stream. 2 or 3. + * @return A pointer to the Zarr stream struct, or NULL on failure. + */ + ZarrStream* ZarrStream_create(ZarrStreamSettings* settings, + ZarrVersion version); + + /** + * @brief Destroy a Zarr stream. + * @details This function frees the memory allocated for the Zarr stream. + * @param stream The Zarr stream struct to destroy. + */ + void ZarrStream_destroy(ZarrStream* stream); + + /** + * @brief Append data to the Zarr stream. + * @param[in, out] stream The Zarr stream struct. + * @param[in] data The data to append. + * @param[in] bytes_in The number of bytes in @p data. It should be at least + * the size of a single frame. + * @param[out] bytes_out The number of bytes written to the stream. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStream_append(ZarrStream* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out); + + /** + * @brief Get the version (i.e., 2 or 3) of the Zarr stream. + * @param stream The Zarr stream struct. + * @return The version of the Zarr stream. + */ + ZarrVersion ZarrStream_get_version(const ZarrStream* stream); + + /** + * @brief Get a copy of the settings for the Zarr stream. + * @param stream The Zarr stream struct. + * @return A copy of the settings for the Zarr stream. + */ + ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); + + ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + ZarrLogLevel Zarr_get_log_level(); + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_V0 diff --git a/include/zarr.types.h b/include/zarr.types.h new file mode 100644 index 00000000..fdb6c715 --- /dev/null +++ b/include/zarr.types.h @@ -0,0 +1,133 @@ +#ifndef H_ACQUIRE_ZARR_TYPES_V0 +#define H_ACQUIRE_ZARR_TYPES_V0 + +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef enum + { + ZarrStatus_Success = 0, + ZarrStatus_InvalidArgument, + ZarrStatus_Overflow, + ZarrStatus_InvalidIndex, + ZarrStatus_NotYetImplemented, + ZarrStatus_InternalError, + ZarrStatus_OutOfMemory, + ZarrStatus_IOError, + ZarrStatus_CompressionError, + ZarrStatus_InvalidSettings, + ZarrStatusCount, + } ZarrStatus; + + typedef enum + { + ZarrVersion_2 = 2, + ZarrVersion_3, + ZarrVersionCount + } ZarrVersion; + + typedef enum + { + ZarrLogLevel_Debug, + ZarrLogLevel_Info, + ZarrLogLevel_Warning, + ZarrLogLevel_Error, + ZarrLogLevel_None, + ZarrLogLevelCount + } ZarrLogLevel; + + typedef enum + { + ZarrDataType_uint8, + ZarrDataType_uint16, + ZarrDataType_uint32, + ZarrDataType_uint64, + ZarrDataType_int8, + ZarrDataType_int16, + ZarrDataType_int32, + ZarrDataType_int64, + ZarrDataType_float32, + ZarrDataType_float64, + ZarrDataTypeCount + } ZarrDataType; + + typedef enum + { + ZarrCompressor_None = 0, + ZarrCompressor_Blosc1, + ZarrCompressorCount + } ZarrCompressor; + + typedef enum + { + ZarrCompressionCodec_None = 0, + ZarrCompressionCodec_BloscLZ4, + ZarrCompressionCodec_BloscZstd, + ZarrCompressionCodecCount + } ZarrCompressionCodec; + + typedef enum + { + ZarrDimensionType_Space = 0, + ZarrDimensionType_Channel, + ZarrDimensionType_Time, + ZarrDimensionType_Other, + ZarrDimensionTypeCount + } ZarrDimensionType; + + /** + * @brief S3 settings for streaming to Zarr. + */ + typedef struct + { + const char* endpoint; + size_t bytes_of_endpoint; + const char* bucket_name; + size_t bytes_of_bucket_name; + const char* access_key_id; + size_t bytes_of_access_key_id; + const char* secret_access_key; + size_t bytes_of_secret_access_key; + } ZarrS3Settings; + + /** + * @brief Compression settings for a Zarr array. + * @detail The compressor is not the same as the codec. A codec is + * a specific implementation of a compression algorithm, while a compressor + * is a library that implements one or more codecs. + */ + typedef struct + { + ZarrCompressor compressor; /**< Compressor to use */ + ZarrCompressionCodec codec; /**< Codec to use */ + uint8_t level; /**< Compression level */ + uint8_t shuffle; /**< Whether to shuffle the data before compressing */ + } ZarrCompressionSettings; + + /** + * @brief Properties of a dimension of the Zarr array. + */ + typedef struct + { + const char* name; /**< Name of the dimension */ + size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ + ZarrDimensionType kind; /**< Type of the dimension */ + uint32_t array_size_px; /**< Size of the array along this dimension in + pixels */ + uint32_t chunk_size_px; /**< Size of the chunks along this dimension in + pixels */ + uint32_t shard_size_chunks; /**< Number of chunks in a shard along this + dimension */ + } ZarrDimensionProperties; + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_TYPES_V0 \ No newline at end of file From cf89e13d638b2a445815bd4222e0bce9a2045456 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 10:50:56 -0400 Subject: [PATCH 03/39] Define the Zarr logger. --- src/CMakeLists.txt | 1 + src/logger/CMakeLists.txt | 23 +++++++++++ src/logger/logger.cpp | 83 +++++++++++++++++++++++++++++++++++++++ src/logger/logger.hh | 30 ++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 src/logger/CMakeLists.txt create mode 100644 src/logger/logger.cpp create mode 100644 src/logger/logger.hh diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9a3b42a..1323f310 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,4 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) +add_subdirectory(logger) add_subdirectory(driver) diff --git a/src/logger/CMakeLists.txt b/src/logger/CMakeLists.txt new file mode 100644 index 00000000..37b28416 --- /dev/null +++ b/src/logger/CMakeLists.txt @@ -0,0 +1,23 @@ +set(tgt acquire-zarr-logger) + +add_library(${tgt} + logger.hh + logger.cpp +) + +set(PUBLIC_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include/) + +target_include_directories(${tgt} + PUBLIC + $ + PRIVATE + $ +) + +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt} + LIBRARY DESTINATION lib +) \ No newline at end of file diff --git a/src/logger/logger.cpp b/src/logger/logger.cpp new file mode 100644 index 00000000..c316fcad --- /dev/null +++ b/src/logger/logger.cpp @@ -0,0 +1,83 @@ +#include "logger.hh" + +#include +#include +#include +#include +#include +#include + +ZarrLogLevel Logger::current_level_ = ZarrLogLevel_Info; + +void +Logger::set_log_level(ZarrLogLevel level) +{ + current_level_ = level; +} + +ZarrLogLevel +Logger::get_log_level() +{ + return current_level_; +} + +std::string +Logger::log(ZarrLogLevel level, + const char* file, + int line, + const char* func, + const char* format, + ...) +{ + std::scoped_lock lock(log_mutex_); + if (current_level_ == ZarrLogLevel_None || level < current_level_) { + return {}; // Suppress logs + } + + va_list args; + va_start(args, format); + + std::string prefix; + std::ostream* stream = &std::cout; + + switch (level) { + case ZarrLogLevel_Debug: + prefix = "[DEBUG] "; + break; + case ZarrLogLevel_Info: + prefix = "[INFO] "; + break; + case ZarrLogLevel_Warning: + prefix = "[WARNING] "; + stream = &std::cerr; + break; + case ZarrLogLevel_Error: + prefix = "[ERROR] "; + stream = &std::cerr; + break; + } + + // Get current time + auto now = std::chrono::system_clock::now(); + auto time = std::chrono::system_clock::to_time_t(now); + auto ms = std::chrono::duration_cast( + now.time_since_epoch()) % + 1000; + + // Get filename without path + std::filesystem::path filepath(file); + std::string filename = filepath.filename().string(); + + // Output timestamp, log level, filename + *stream << std::put_time(std::localtime(&time), "%Y-%m-%d %H:%M:%S") << '.' + << std::setfill('0') << std::setw(3) << ms.count() << " " << prefix + << filename << ":" << line << " " << func << ": "; + + char buffer[1024]; + vsnprintf(buffer, sizeof(buffer), format, args); + *stream << buffer << std::endl; + + va_end(args); + + return buffer; +} \ No newline at end of file diff --git a/src/logger/logger.hh b/src/logger/logger.hh new file mode 100644 index 00000000..1f1f339f --- /dev/null +++ b/src/logger/logger.hh @@ -0,0 +1,30 @@ +#include "zarr.types.h" + +#include + +class Logger +{ + public: + static void set_log_level(ZarrLogLevel level); + static ZarrLogLevel get_log_level(); + + static std::string log(ZarrLogLevel level, + const char* file, + int line, + const char* func, + const char* format, + ...); + + private: + static ZarrLogLevel current_level_; + static std::mutex log_mutex_; +}; + +#define LOG_DEBUG(...) \ + Logger::log(ZarrLogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_INFO(...) \ + Logger::log(LogLevel_Info, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_WARNING(...) \ + Logger::log(ZarrLogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_ERROR(...) \ + Logger::log(ZarrLogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) From fff86c61fb3798f6894b60d251e420fea30d57af Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 10:52:59 -0400 Subject: [PATCH 04/39] Rename zarr.h to acquire.zarr.h. --- include/{zarr.h => acquire.zarr.h} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename include/{zarr.h => acquire.zarr.h} (100%) diff --git a/include/zarr.h b/include/acquire.zarr.h similarity index 100% rename from include/zarr.h rename to include/acquire.zarr.h From bdd0c51dba2756f8907c023a7459b84d9dfc1c50 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 11:23:38 -0400 Subject: [PATCH 05/39] Instantiate the logger's mutex. --- src/logger/logger.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/logger/logger.cpp b/src/logger/logger.cpp index c316fcad..2dfa5e2d 100644 --- a/src/logger/logger.cpp +++ b/src/logger/logger.cpp @@ -8,6 +8,7 @@ #include ZarrLogLevel Logger::current_level_ = ZarrLogLevel_Info; +std::mutex Logger::log_mutex_{}; void Logger::set_log_level(ZarrLogLevel level) From d23fe1a4aed5b219573222dfbfdda3854e10a7c7 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 13:35:06 -0400 Subject: [PATCH 06/39] Implement and test stream settings, with API setters and getters. --- include/zarr.types.h | 2 +- src/CMakeLists.txt | 1 + src/streaming/CMakeLists.txt | 36 ++ src/streaming/macros.hh | 28 ++ src/streaming/stream.settings.cpp | 513 +++++++++++++++++++++ src/streaming/stream.settings.hh | 46 ++ tests/CMakeLists.txt | 1 + tests/unit-tests/CMakeLists.txt | 28 ++ tests/unit-tests/get-stream-parameters.cpp | 236 ++++++++++ tests/unit-tests/set-stream-parameters.cpp | 274 +++++++++++ tests/unit-tests/unit.test.macros.hh | 34 ++ 11 files changed, 1198 insertions(+), 1 deletion(-) create mode 100644 src/streaming/CMakeLists.txt create mode 100644 src/streaming/macros.hh create mode 100644 src/streaming/stream.settings.cpp create mode 100644 src/streaming/stream.settings.hh create mode 100644 tests/unit-tests/CMakeLists.txt create mode 100644 tests/unit-tests/get-stream-parameters.cpp create mode 100644 tests/unit-tests/set-stream-parameters.cpp create mode 100644 tests/unit-tests/unit.test.macros.hh diff --git a/include/zarr.types.h b/include/zarr.types.h index fdb6c715..56a54a56 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -117,7 +117,7 @@ extern "C" { const char* name; /**< Name of the dimension */ size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ - ZarrDimensionType kind; /**< Type of the dimension */ + ZarrDimensionType type; /**< Type of the dimension */ uint32_t array_size_px; /**< Size of the array along this dimension in pixels */ uint32_t chunk_size_px; /**< Size of the chunks along this dimension in diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1323f310..0c51ab98 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_subdirectory(logger) +add_subdirectory(streaming) add_subdirectory(driver) diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt new file mode 100644 index 00000000..2f9fb4fd --- /dev/null +++ b/src/streaming/CMakeLists.txt @@ -0,0 +1,36 @@ +set(tgt acquire-zarr) + +add_library(${tgt} + macros.hh + stream.settings.hh + stream.settings.cpp +) + +target_include_directories(${tgt} + PUBLIC + $ + PRIVATE + $ + $ +) + +target_link_libraries(${tgt} PRIVATE + acquire-zarr-logger + blosc_static + miniocpp::miniocpp +) + +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt} + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) + +# Install public header files +install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/ + DESTINATION include + FILES_MATCHING PATTERN "*.h" +) \ No newline at end of file diff --git a/src/streaming/macros.hh b/src/streaming/macros.hh new file mode 100644 index 00000000..05b094b3 --- /dev/null +++ b/src/streaming/macros.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "logger.hh" + +#define EXPECT(e, ...) \ + do { \ + if (!(e)) { \ + const std::string __err = LOG_ERROR(__VA_ARGS__); \ + throw std::runtime_error(__err); \ + } \ + } while (0) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) + +#define EXPECT_VALID_ARGUMENT(e, ...) \ + do { \ + if (!(e)) { \ + LOG_ERROR(__VA_ARGS__); \ + return ZarrStatus_InvalidArgument; \ + } \ + } while (0) + +#define EXPECT_VALID_INDEX(e, ...) \ + do { \ + if (!(e)) { \ + LOG_ERROR(__VA_ARGS__); \ + return ZarrStatus_InvalidIndex; \ + } \ + } while (0) \ No newline at end of file diff --git a/src/streaming/stream.settings.cpp b/src/streaming/stream.settings.cpp new file mode 100644 index 00000000..09c8bdb8 --- /dev/null +++ b/src/streaming/stream.settings.cpp @@ -0,0 +1,513 @@ +#include "macros.hh" +#include "stream.settings.hh" +#include "acquire.zarr.h" + +#include +#include + +#include // memcpy, strnlen +#include + +#define SETTINGS_GET_STRING(settings, member) \ + do { \ + if (!settings) { \ + LOG_ERROR("Null pointer: %s", #settings); \ + return nullptr; \ + } \ + return settings->member.c_str(); \ + } while (0) + +namespace fs = std::filesystem; + +namespace { +const size_t zarr_dimension_min = 3; +const size_t zarr_dimension_max = 32; + +[[nodiscard]] +std::string +trim(const char* s, size_t bytes_of_s) +{ + // trim left + std::string trimmed(s, bytes_of_s); + trimmed.erase(trimmed.begin(), + std::find_if(trimmed.begin(), trimmed.end(), [](char c) { + return !std::isspace(c); + })); + + // trim right + trimmed.erase(std::find_if(trimmed.rbegin(), + trimmed.rend(), + [](char c) { return !std::isspace(c); }) + .base(), + trimmed.end()); + + return trimmed; +} + +[[nodiscard]] +bool +validate_s3_settings(const ZarrS3Settings* settings) +{ + size_t len; + + if (len = strnlen(settings->endpoint, settings->bytes_of_endpoint); + len == 0) { + LOG_ERROR("S3 endpoint is empty"); + return false; + } + + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html + if (len = strnlen(settings->bucket_name, settings->bytes_of_bucket_name); + len < 4 || len > 64) { + LOG_ERROR("Invalid length for S3 bucket name: %zu. Must be between 3 " + "and 63 characters", + len); + return false; + } + + if (len = + strnlen(settings->access_key_id, settings->bytes_of_access_key_id); + len == 0) { + LOG_ERROR("S3 access key ID is empty"); + return false; + } + + if (len = strnlen(settings->secret_access_key, + settings->bytes_of_secret_access_key); + len == 0) { + LOG_ERROR("S3 secret access key is empty"); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_filesystem_store_path(std::string_view data_root) +{ + fs::path path(data_root); + fs::path parent_path = path.parent_path(); + if (parent_path.empty()) { + parent_path = "."; + } + + // parent path must exist and be a directory + if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { + LOG_ERROR("Parent path '%s' does not exist or is not a directory", + parent_path.c_str()); + return false; + } + + // parent path must be writable + const auto perms = fs::status(parent_path).permissions(); + const bool is_writable = + (perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none; + + if (!is_writable) { + LOG_ERROR("Parent path '%s' is not writable", parent_path.c_str()); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_compression_settings(const ZarrCompressionSettings* settings) +{ + if (settings->compressor >= ZarrCompressorCount) { + LOG_ERROR("Invalid compressor: %d", settings->compressor); + return false; + } + + if (settings->codec >= ZarrCompressionCodecCount) { + LOG_ERROR("Invalid compression codec: %d", settings->codec); + return false; + } + + // if compressing, we require a compression codec + if (settings->compressor != ZarrCompressor_None && + settings->codec == ZarrCompressionCodec_None) { + LOG_ERROR("Compression codec must be set when using a compressor"); + return false; + } + + if (settings->level > 9) { + LOG_ERROR("Invalid compression level: %d. Must be between 0 and 9", + settings->level); + return false; + } + + if (settings->shuffle != BLOSC_NOSHUFFLE && + settings->shuffle != BLOSC_SHUFFLE && + settings->shuffle != BLOSC_BITSHUFFLE) { + LOG_ERROR("Invalid shuffle: %d. Must be %d (no shuffle), %d (byte " + "shuffle), or %d (bit shuffle)", + settings->shuffle, + BLOSC_NOSHUFFLE, + BLOSC_SHUFFLE, + BLOSC_BITSHUFFLE); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_dimension(const ZarrDimensionProperties* dimension) +{ + std::string trimmed = trim(dimension->name, dimension->bytes_of_name - 1); + if (trimmed.empty()) { + LOG_ERROR("Invalid name. Must not be empty"); + return false; + } + + if (dimension->type >= ZarrDimensionTypeCount) { + LOG_ERROR("Invalid dimension type: %d", dimension->type); + return false; + } + + if (dimension->chunk_size_px == 0) { + LOG_ERROR("Invalid chunk size: %zu", dimension->chunk_size_px); + return false; + } + + return true; +} +} // namespace + +ZarrStreamSettings_s::ZarrStreamSettings_s() + : store_path() + , s3_endpoint() + , s3_bucket_name() + , s3_access_key_id() + , s3_secret_access_key() + , custom_metadata("{}") + , dtype(ZarrDataType_uint8) + , compressor(ZarrCompressor_None) + , compression_codec(ZarrCompressionCodec_None) + , compression_level(0) + , compression_shuffle(BLOSC_NOSHUFFLE) + , dimensions() + , multiscale(false) +{ +} + +/* Lifecycle */ +ZarrStreamSettings* +ZarrStreamSettings_create() +{ + try { + return new ZarrStreamSettings(); + } catch (const std::bad_alloc&) { + return nullptr; + } +} + +void +ZarrStreamSettings_destroy(ZarrStreamSettings* settings) +{ + delete settings; +} + +ZarrStreamSettings* +ZarrStreamSettings_copy(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return nullptr; + } + + ZarrStreamSettings* copy = ZarrStreamSettings_create(); + if (!copy) { + LOG_ERROR("Failed to allocate memory for copy"); + return nullptr; + } + + *copy = *settings; + + return copy; +} + +/* Setters */ +ZarrStatus +ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); + + bytes_of_store_path = strnlen(store_path, bytes_of_store_path); + EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, + "Invalid store path. Must not be empty"); + + std::string_view store_path_sv(store_path, bytes_of_store_path); + if (store_path_sv.empty()) { + LOG_ERROR("Invalid store path. Must not be empty"); + return ZarrStatus_InvalidArgument; + } + + if (nullptr != s3_settings) { + if (!validate_s3_settings(s3_settings)) { + return ZarrStatus_InvalidArgument; + } + } else if (!validate_filesystem_store_path(store_path)) { + return ZarrStatus_InvalidArgument; + } + + if (nullptr != s3_settings) { + settings->s3_endpoint = s3_settings->endpoint; + settings->s3_bucket_name = s3_settings->bucket_name; + settings->s3_access_key_id = s3_settings->access_key_id; + settings->s3_secret_access_key = s3_settings->secret_access_key; + } + + settings->store_path = store_path; + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_compression( + ZarrStreamSettings* settings, + const ZarrCompressionSettings* compression_settings) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(compression_settings, + "Null pointer: compression_settings"); + + if (!validate_compression_settings(compression_settings)) { + return ZarrStatus_InvalidArgument; + } + + settings->compressor = compression_settings->compressor; + settings->compression_codec = compression_settings->codec; + settings->compression_level = compression_settings->level; + settings->compression_shuffle = compression_settings->shuffle; + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_custom_metadata(ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata) +{ + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return ZarrStatus_InvalidArgument; + } + + if (!external_metadata) { + LOG_ERROR("Null pointer: custom_metadata"); + return ZarrStatus_InvalidArgument; + } + + if (bytes_of_external_metadata == 0) { + LOG_ERROR("Invalid length: %zu. Must be greater than 0", + bytes_of_external_metadata); + return ZarrStatus_InvalidArgument; + } + + size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); + if (nbytes < 2) { + settings->custom_metadata = "{}"; + return ZarrStatus_Success; + } + + auto val = nlohmann::json::parse(external_metadata, + external_metadata + nbytes, + nullptr, // callback + false, // allow exceptions + true // ignore comments + ); + + if (val.is_discarded()) { + LOG_ERROR("Invalid JSON: %s", external_metadata); + return ZarrStatus_InvalidArgument; + } + settings->custom_metadata = val.dump(); + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); + + settings->dtype = data_type; + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_reserve_dimensions(ZarrStreamSettings* settings, + size_t count) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(count >= zarr_dimension_min && + count <= zarr_dimension_max, + "Invalid count: %zu. Count must be between %d and %d", + count, + zarr_dimension_min, + zarr_dimension_max); + + settings->dimensions.resize(count); + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_dimension(ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); + EXPECT_VALID_INDEX(index < settings->dimensions.size(), + "Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + + if (!validate_dimension(dimension)) { + return ZarrStatus_InvalidArgument; + } + + struct ZarrDimension_s& dim = settings->dimensions[index]; + + dim.name = trim(dimension->name, dimension->bytes_of_name - 1); + dim.type = dimension->type; + dim.array_size_px = dimension->array_size_px; + dim.chunk_size_px = dimension->chunk_size_px; + dim.shard_size_chunks = dimension->shard_size_chunks; + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + + settings->multiscale = multiscale > 0; + return ZarrStatus_Success; +} + +/* Getters */ +const char* +ZarrStreamSettings_get_store_path(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, store_path); +} + +ZarrS3Settings +ZarrStreamSettings_get_s3_settings(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); + return {}; + } + + ZarrS3Settings s3_settings = { + settings->s3_endpoint.c_str(), + settings->s3_endpoint.length() + 1, + settings->s3_bucket_name.c_str(), + settings->s3_bucket_name.length() + 1, + settings->s3_access_key_id.c_str(), + settings->s3_access_key_id.length() + 1, + settings->s3_secret_access_key.c_str(), + settings->s3_secret_access_key.length() + 1, + }; + return s3_settings; +} + +const char* +ZarrStreamSettings_get_custom_metadata(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, custom_metadata); +} + +ZarrDataType +ZarrStreamSettings_get_data_type(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); + return ZarrDataType_uint8; + } + return static_cast(settings->dtype); +} + +ZarrCompressionSettings +ZarrStreamSettings_get_compression(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty compression."); + return {}; + } + + ZarrCompressionSettings compression = { + .compressor = settings->compressor, + .codec = static_cast(settings->compression_codec), + .level = settings->compression_level, + .shuffle = settings->compression_shuffle, + }; + return compression; +} + +size_t +ZarrStreamSettings_get_dimension_count(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return settings->dimensions.size(); +} + +ZarrDimensionProperties +ZarrStreamSettings_get_dimension(const ZarrStreamSettings* settings, + size_t index) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty dimension."); + return {}; + } + + if (index >= settings->dimensions.size()) { + LOG_ERROR("Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + return {}; + } + + const auto& dim = settings->dimensions[index]; + + ZarrDimensionProperties dimension = { + .name = dim.name.c_str(), + .bytes_of_name = dim.name.size() + 1, + .type = dim.type, + .array_size_px = dim.array_size_px, + .chunk_size_px = dim.chunk_size_px, + .shard_size_chunks = dim.shard_size_chunks, + }; + + return dimension; +} + +bool +ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning false."); + return false; + } + return settings->multiscale; +} diff --git a/src/streaming/stream.settings.hh b/src/streaming/stream.settings.hh new file mode 100644 index 00000000..a23e48cf --- /dev/null +++ b/src/streaming/stream.settings.hh @@ -0,0 +1,46 @@ +#pragma once + +#include "zarr.types.h" + +#include // size_t +#include // uint8_t +#include +#include + +struct ZarrDimension_s +{ + std::string name; /* Name of the dimension */ + ZarrDimensionType type; /* Type of dimension */ + + uint32_t array_size_px; /* Size of the array along this dimension */ + uint32_t chunk_size_px; /* Size of a chunk along this dimension */ + uint32_t shard_size_chunks; /* Number of chunks in a shard along this + dimension */ +}; + +struct ZarrStreamSettings_s +{ + public: + ZarrStreamSettings_s(); + std::string store_path; /* Path to the Zarr store on the local filesystem */ + + std::string s3_endpoint; /* Endpoint for the S3 service */ + std::string s3_bucket_name; /* Name of the S3 bucket */ + std::string s3_access_key_id; /* Access key ID for the S3 service */ + std::string s3_secret_access_key; /* Secret access key for the S3 service */ + + std::string custom_metadata; /* JSON formatted external metadata for the + base array */ + + ZarrDataType dtype; /* Data type of the base array */ + + ZarrCompressor compressor; /* Compression library to use */ + ZarrCompressionCodec compression_codec; /* Compression codec to use */ + uint8_t compression_level; /* Compression level to use */ + uint8_t compression_shuffle; /* Whether and how to shuffle the data before + compressing */ + + std::vector dimensions; /* Dimensions of the base array */ + + bool multiscale; /* Whether to stream to multiple resolutions */ +}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 87878b37..f7713371 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,6 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () + add_subdirectory(unit-tests) add_subdirectory(driver) endif () diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt new file mode 100644 index 00000000..f5d9264b --- /dev/null +++ b/tests/unit-tests/CMakeLists.txt @@ -0,0 +1,28 @@ +set(project acquire-zarr) + +set(tests + set-stream-parameters + get-stream-parameters +) + +foreach (name ${tests}) + set(tgt "${project}-unit-test-${name}") + add_executable(${tgt} ${name}.cpp unit.test.macros.hh) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/logger + ${CMAKE_SOURCE_DIR}/src/streaming + ) + target_link_libraries(${tgt} PRIVATE + acquire-zarr-logger + acquire-zarr + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;unit-tests;acquire-zarr") +endforeach () \ No newline at end of file diff --git a/tests/unit-tests/get-stream-parameters.cpp b/tests/unit-tests/get-stream-parameters.cpp new file mode 100644 index 00000000..c3d287a4 --- /dev/null +++ b/tests/unit-tests/get-stream-parameters.cpp @@ -0,0 +1,236 @@ +#include "acquire.zarr.h" +#include "stream.settings.hh" +#include "unit.test.macros.hh" + +void +check_preliminaries(ZarrStreamSettings* settings) +{ + CHECK(settings); + + CHECK(settings->store_path.empty()); + + CHECK(settings->s3_endpoint.empty()); + CHECK(settings->s3_bucket_name.empty()); + CHECK(settings->s3_access_key_id.empty()); + CHECK(settings->s3_secret_access_key.empty()); + + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint8); + + EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_None); + EXPECT_EQ( + int, "%d", settings->compression_codec, ZarrCompressionCodec_None); + EXPECT_EQ(int, "%d", settings->compression_level, 0); + EXPECT_EQ(int, "%d", settings->compression_shuffle, 0); + + CHECK(settings->dimensions.empty()); + + CHECK(!settings->multiscale); +} + +void +get_store_path(ZarrStreamSettings* settings) +{ + EXPECT_STR_EQ(ZarrStreamSettings_get_store_path(settings), ""); + + settings->store_path = TEST ".zarr"; + EXPECT_STR_EQ(ZarrStreamSettings_get_store_path(settings), + settings->store_path.c_str()); +} + +void +get_s3_settings(ZarrStreamSettings* settings) +{ + auto s3_settings = ZarrStreamSettings_get_s3_settings(settings); + EXPECT_STR_EQ(s3_settings.endpoint, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_endpoint, 1); + + EXPECT_STR_EQ(s3_settings.bucket_name, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_bucket_name, 1); + + EXPECT_STR_EQ(s3_settings.access_key_id, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_access_key_id, 1); + + EXPECT_STR_EQ(s3_settings.secret_access_key, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_secret_access_key, 1); + + settings->s3_endpoint = "https://s3.amazonaws.com"; + settings->s3_bucket_name = "bucket"; + settings->s3_access_key_id = "access_key"; + settings->s3_secret_access_key = "secret_access_key"; + + s3_settings = ZarrStreamSettings_get_s3_settings(settings); + EXPECT_STR_EQ(s3_settings.endpoint, settings->s3_endpoint.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_endpoint, + settings->s3_endpoint.size() + 1); + + EXPECT_STR_EQ(s3_settings.bucket_name, settings->s3_bucket_name.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_bucket_name, + settings->s3_bucket_name.size() + 1); + + EXPECT_STR_EQ(s3_settings.access_key_id, + settings->s3_access_key_id.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_access_key_id, + settings->s3_access_key_id.size() + 1); + + EXPECT_STR_EQ(s3_settings.secret_access_key, + settings->s3_secret_access_key.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_secret_access_key, + settings->s3_secret_access_key.size() + 1); +} + +void +get_compression(ZarrStreamSettings* settings) +{ + auto compression_settings = ZarrStreamSettings_get_compression(settings); + + EXPECT_EQ(int, "%d", compression_settings.compressor, ZarrCompressor_None); + EXPECT_EQ(int, "%d", compression_settings.codec, ZarrCompressionCodec_None); + EXPECT_EQ(int, "%d", compression_settings.level, 0); + EXPECT_EQ(int, "%d", compression_settings.shuffle, 0); + + settings->compressor = ZarrCompressor_Blosc1; + settings->compression_codec = ZarrCompressionCodec_BloscZstd; + settings->compression_level = 8; + settings->compression_shuffle = 2; + + compression_settings = ZarrStreamSettings_get_compression(settings); + EXPECT_EQ( + int, "%d", compression_settings.compressor, ZarrCompressor_Blosc1); + EXPECT_EQ( + int, "%d", compression_settings.codec, ZarrCompressionCodec_BloscZstd); + EXPECT_EQ(int, "%d", compression_settings.level, 8); + EXPECT_EQ(int, "%d", compression_settings.shuffle, 2); +} + +void +get_data_type(ZarrStreamSettings* settings) +{ + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_get_data_type(settings), + ZarrDataType_uint8); + + settings->dtype = ZarrDataType_float32; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_get_data_type(settings), + ZarrDataType_float32); +} + +void +get_dimensions(ZarrStreamSettings* settings) +{ + EXPECT_EQ(int, "%d", ZarrStreamSettings_get_dimension_count(settings), 0); + + settings->dimensions.resize(3); + EXPECT_EQ(int, "%d", ZarrStreamSettings_get_dimension_count(settings), 3); + + { + auto& dim = settings->dimensions[0]; + dim.name = "time"; + dim.type = ZarrDimensionType_Time; + dim.array_size_px = 100; + dim.chunk_size_px = 13; + dim.shard_size_chunks = 7; + } + { + auto& dim = settings->dimensions[1]; + dim.name = "height"; + dim.array_size_px = 300; + dim.chunk_size_px = 19; + dim.shard_size_chunks = 13; + } + { + auto& dim = settings->dimensions[2]; + dim.name = "width"; + dim.array_size_px = 200; + dim.chunk_size_px = 17; + dim.shard_size_chunks = 11; + } + + // can't get beyond the last dimension + auto dim = ZarrStreamSettings_get_dimension(settings, 3); + EXPECT_STR_EQ(dim.name, ""); + EXPECT_EQ(int, "%d", dim.bytes_of_name, 0); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", dim.array_size_px, 0); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 0); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 0); + + dim = ZarrStreamSettings_get_dimension(settings, 0); + EXPECT_STR_EQ(dim.name, "time"); + EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("time")); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", dim.array_size_px, 100); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 13); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 7); + + dim = ZarrStreamSettings_get_dimension(settings, 1); + EXPECT_STR_EQ(dim.name, "height"); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("height")); + EXPECT_EQ(int, "%d", dim.array_size_px, 300); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 19); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 13); + + dim = ZarrStreamSettings_get_dimension(settings, 2); + EXPECT_STR_EQ(dim.name, "width"); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("width")); + EXPECT_EQ(int, "%d", dim.array_size_px, 200); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 17); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 11); +} + +void +get_multiscale(ZarrStreamSettings* settings) +{ + CHECK(!ZarrStreamSettings_get_multiscale(settings)); + + settings->multiscale = true; + CHECK(ZarrStreamSettings_get_multiscale(settings)); +} + +void +get_custom_metadata(ZarrStreamSettings* settings) +{ + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + settings->custom_metadata = "this ain't even json"; // oops + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "this ain't even json"); +} + +int +main() +{ + int retval = 1; + + ZarrStreamSettings* settings = ZarrStreamSettings_create(); + try { + CHECK(settings); + check_preliminaries(settings); + get_store_path(settings); + get_s3_settings(settings); + get_compression(settings); + get_data_type(settings); + get_dimensions(settings); + get_multiscale(settings); + get_custom_metadata(settings); + + retval = 0; + } catch (const std::exception& exception) { + LOG_ERROR("%s", exception.what()); + } + ZarrStreamSettings_destroy(settings); + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/set-stream-parameters.cpp b/tests/unit-tests/set-stream-parameters.cpp new file mode 100644 index 00000000..d7ee16b7 --- /dev/null +++ b/tests/unit-tests/set-stream-parameters.cpp @@ -0,0 +1,274 @@ +#include "acquire.zarr.h" +#include "stream.settings.hh" +#include "unit.test.macros.hh" + +void +check_preliminaries(ZarrStreamSettings* settings) +{ + CHECK(settings); + + CHECK(settings->store_path.empty()); + + CHECK(settings->s3_endpoint.empty()); + CHECK(settings->s3_bucket_name.empty()); + CHECK(settings->s3_access_key_id.empty()); + CHECK(settings->s3_secret_access_key.empty()); + + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint8); + + EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_None); + EXPECT_EQ( + int, "%d", settings->compression_codec, ZarrCompressionCodec_None); + EXPECT_EQ(int, "%d", settings->compression_level, 0); + EXPECT_EQ(int, "%d", settings->compression_shuffle, 0); + + CHECK(settings->dimensions.empty()); + + CHECK(!settings->multiscale); +} + +void +set_store(ZarrStreamSettings* settings) +{ + std::string store_path = TEST ".zarr"; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_store( + settings, store_path.c_str(), store_path.size() + 1, nullptr), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->store_path.c_str(), store_path.c_str()); + settings->store_path = ""; // reset + + ZarrS3Settings s3_settings{ + .endpoint = "https://s3.amazonaws.com", + .bytes_of_endpoint = sizeof("https://s3.amazonaws.com"), + .bucket_name = "bucket", + .bytes_of_bucket_name = sizeof("bucket"), + .access_key_id = "access_key", + .bytes_of_access_key_id = sizeof("access_key"), + .secret_access_key = "secret_access_key", + .bytes_of_secret_access_key = sizeof("secret_access_key"), + }; + + EXPECT_EQ( + int, + "%d", + ZarrStreamSettings_set_store( + settings, store_path.c_str(), store_path.size() + 1, &s3_settings), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->store_path.c_str(), store_path.c_str()); + EXPECT_STR_EQ(settings->s3_endpoint.c_str(), s3_settings.endpoint); + EXPECT_STR_EQ(settings->s3_bucket_name.c_str(), s3_settings.bucket_name); + EXPECT_STR_EQ(settings->s3_access_key_id.c_str(), + s3_settings.access_key_id); + EXPECT_STR_EQ(settings->s3_secret_access_key.c_str(), + s3_settings.secret_access_key); +} + +void +set_compression(ZarrStreamSettings* settings) +{ + ZarrCompressionSettings compression_settings{ + .compressor = ZarrCompressor_Blosc1, + .codec = ZarrCompressionCodec_BloscLZ4, + .level = 5, + .shuffle = 1, + }; + + EXPECT_EQ( + int, + "%d", + ZarrStreamSettings_set_compression(settings, &compression_settings), + ZarrStatus_Success); + + EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_Blosc1); + EXPECT_EQ( + int, "%d", settings->compression_codec, ZarrCompressionCodec_BloscLZ4); + EXPECT_EQ(int, "%d", settings->compression_level, 5); + EXPECT_EQ(int, "%d", settings->compression_shuffle, 1); +} + +void +set_data_type(ZarrStreamSettings* settings) +{ + ZarrDataType dtype = ZarrDataType_uint16; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_data_type(settings, dtype), + ZarrStatus_Success); + EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint16); +} + +void +set_dimensions(ZarrStreamSettings* settings) +{ + ZarrStreamSettings_reserve_dimensions(settings, 3); + EXPECT_EQ(int, "%d", settings->dimensions.size(), 3); + + ZarrDimensionProperties dim{ + .name = " time ", + .bytes_of_name = sizeof(" time "), + .type = ZarrDimensionType_Time, + .array_size_px = 100, + .chunk_size_px = 13, + .shard_size_chunks = 7, + }; + + // can't set a dimension that is out of bounds + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 3, &dim), + ZarrStatus_InvalidIndex); + + // set the first dimension + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 0, &dim), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); + EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); + EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); + EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); + + // other dimensions should still be unset + EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), ""); + EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 0); + + EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), ""); + EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 0); + + // set the 3rd dimension before the 2nd + dim.name = "width "; + dim.bytes_of_name = sizeof("width "); + dim.type = ZarrDimensionType_Space; + dim.array_size_px = 200; + dim.chunk_size_px = 17; + dim.shard_size_chunks = 11; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 2, &dim), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); + EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); + EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); + EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); + + // 2nd dimension should still be unset + EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), ""); + EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 0); + + EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), "width"); + EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 200); + EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 17); + EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 11); + + // set the 2nd dimension + dim.name = "height"; + dim.bytes_of_name = sizeof("height"); + dim.type = ZarrDimensionType_Space; + dim.array_size_px = 300; + dim.chunk_size_px = 19; + dim.shard_size_chunks = 13; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 1, &dim), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); + EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); + EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); + EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); + + EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), "height"); + EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 300); + EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 19); + EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 13); + + EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), "width"); + EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 200); + EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 17); + EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 11); +} + +void +set_multiscale(ZarrStreamSettings* settings) +{ + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_multiscale(settings, true), + ZarrStatus_Success); + CHECK(settings->multiscale); + + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_multiscale(settings, false), + ZarrStatus_Success); + CHECK(!settings->multiscale); +} + +void +set_custom_metadata(ZarrStreamSettings* settings) +{ + // fails when not JSON formatted + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_custom_metadata( + settings, "this is not json", sizeof("this is not json")), + ZarrStatus_InvalidArgument); + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + // succeeds when JSON formatted + EXPECT_EQ( + int, + "%d", + ZarrStreamSettings_set_custom_metadata( + settings, "{\"key\": \"value\"}", sizeof("{\"key\": \"value\"}")), + ZarrStatus_Success); + // whitespace is removed + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{\"key\":\"value\"}"); +} + +int +main() +{ + int retval = 1; + + ZarrStreamSettings* settings = ZarrStreamSettings_create(); + try { + CHECK(settings); + check_preliminaries(settings); + set_store(settings); + set_compression(settings); + set_data_type(settings); + set_dimensions(settings); + set_multiscale(settings); + set_custom_metadata(settings); + + retval = 0; + } catch (const std::exception& exception) { + LOG_ERROR("%s", exception.what()); + } + ZarrStreamSettings_destroy(settings); + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/unit.test.macros.hh b/tests/unit-tests/unit.test.macros.hh new file mode 100644 index 00000000..0100c112 --- /dev/null +++ b/tests/unit-tests/unit.test.macros.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "logger.hh" + +#define EXPECT(e, ...) \ + do { \ + if (!(e)) { \ + const std::string __err = LOG_ERROR(__VA_ARGS__); \ + throw std::runtime_error(__err); \ + } \ + } while (0) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) + +/// Check that a==b +/// example: `ASSERT_EQ(int,"%d",42,meaning_of_life())` +#define EXPECT_EQ(T, fmt, a, b) \ + do { \ + T a_ = (T)(a); \ + T b_ = (T)(b); \ + EXPECT(a_ == b_, "Expected %s==%s but " fmt "!=" fmt, #a, #b, a_, b_); \ + } while (0) + +#define EXPECT_STR_EQ(a, b) \ + do { \ + std::string a_ = (a) ? (a) : ""; \ + std::string b_ = (b) ? (b) : ""; \ + EXPECT(a_ == b_, \ + "Expected %s==%s but \"%s\"!=\"%s\"", \ + #a, \ + #b, \ + a_.c_str(), \ + b_.c_str()); \ + } while (0) + From 3cd2bb33ff2b08766df6e018d6430a82d7440785 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:10:43 -0400 Subject: [PATCH 07/39] Wrap C API functions in extern "C" {} --- src/streaming/stream.settings.cpp | 518 +++++++++++++++--------------- 1 file changed, 257 insertions(+), 261 deletions(-) diff --git a/src/streaming/stream.settings.cpp b/src/streaming/stream.settings.cpp index 09c8bdb8..ad36f7ba 100644 --- a/src/streaming/stream.settings.cpp +++ b/src/streaming/stream.settings.cpp @@ -196,318 +196,314 @@ ZarrStreamSettings_s::ZarrStreamSettings_s() { } -/* Lifecycle */ -ZarrStreamSettings* -ZarrStreamSettings_create() +extern "C" { - try { - return new ZarrStreamSettings(); - } catch (const std::bad_alloc&) { - return nullptr; + ZarrStreamSettings* ZarrStreamSettings_create() + { + try { + return new ZarrStreamSettings(); + } catch (const std::bad_alloc&) { + return nullptr; + } } -} - -void -ZarrStreamSettings_destroy(ZarrStreamSettings* settings) -{ - delete settings; -} -ZarrStreamSettings* -ZarrStreamSettings_copy(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_ERROR("Null pointer: settings"); - return nullptr; + void ZarrStreamSettings_destroy(ZarrStreamSettings* settings) + { + delete settings; } - ZarrStreamSettings* copy = ZarrStreamSettings_create(); - if (!copy) { - LOG_ERROR("Failed to allocate memory for copy"); - return nullptr; - } + ZarrStreamSettings* ZarrStreamSettings_copy( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return nullptr; + } - *copy = *settings; + ZarrStreamSettings* copy = ZarrStreamSettings_create(); + if (!copy) { + LOG_ERROR("Failed to allocate memory for copy"); + return nullptr; + } - return copy; -} + *copy = *settings; -/* Setters */ -ZarrStatus -ZarrStreamSettings_set_store(ZarrStreamSettings* settings, - const char* store_path, - size_t bytes_of_store_path, - const ZarrS3Settings* s3_settings) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); + return copy; + } - bytes_of_store_path = strnlen(store_path, bytes_of_store_path); - EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, - "Invalid store path. Must not be empty"); + /* Setters */ + ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); - std::string_view store_path_sv(store_path, bytes_of_store_path); - if (store_path_sv.empty()) { - LOG_ERROR("Invalid store path. Must not be empty"); - return ZarrStatus_InvalidArgument; - } + bytes_of_store_path = strnlen(store_path, bytes_of_store_path); + EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, + "Invalid store path. Must not be empty"); - if (nullptr != s3_settings) { - if (!validate_s3_settings(s3_settings)) { + std::string_view store_path_sv(store_path, bytes_of_store_path); + if (store_path_sv.empty()) { + LOG_ERROR("Invalid store path. Must not be empty"); return ZarrStatus_InvalidArgument; } - } else if (!validate_filesystem_store_path(store_path)) { - return ZarrStatus_InvalidArgument; - } - - if (nullptr != s3_settings) { - settings->s3_endpoint = s3_settings->endpoint; - settings->s3_bucket_name = s3_settings->bucket_name; - settings->s3_access_key_id = s3_settings->access_key_id; - settings->s3_secret_access_key = s3_settings->secret_access_key; - } - settings->store_path = store_path; + if (nullptr != s3_settings) { + if (!validate_s3_settings(s3_settings)) { + return ZarrStatus_InvalidArgument; + } + } else if (!validate_filesystem_store_path(store_path)) { + return ZarrStatus_InvalidArgument; + } - return ZarrStatus_Success; -} + if (nullptr != s3_settings) { + settings->s3_endpoint = s3_settings->endpoint; + settings->s3_bucket_name = s3_settings->bucket_name; + settings->s3_access_key_id = s3_settings->access_key_id; + settings->s3_secret_access_key = s3_settings->secret_access_key; + } -ZarrStatus -ZarrStreamSettings_set_compression( - ZarrStreamSettings* settings, - const ZarrCompressionSettings* compression_settings) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(compression_settings, - "Null pointer: compression_settings"); + settings->store_path = store_path; - if (!validate_compression_settings(compression_settings)) { - return ZarrStatus_InvalidArgument; + return ZarrStatus_Success; } - settings->compressor = compression_settings->compressor; - settings->compression_codec = compression_settings->codec; - settings->compression_level = compression_settings->level; - settings->compression_shuffle = compression_settings->shuffle; - - return ZarrStatus_Success; -} - -ZarrStatus -ZarrStreamSettings_set_custom_metadata(ZarrStreamSettings* settings, - const char* external_metadata, - size_t bytes_of_external_metadata) -{ - if (!settings) { - LOG_ERROR("Null pointer: settings"); - return ZarrStatus_InvalidArgument; - } + ZarrStatus ZarrStreamSettings_set_compression( + ZarrStreamSettings* settings, + const ZarrCompressionSettings* compression_settings) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(compression_settings, + "Null pointer: compression_settings"); - if (!external_metadata) { - LOG_ERROR("Null pointer: custom_metadata"); - return ZarrStatus_InvalidArgument; - } + if (!validate_compression_settings(compression_settings)) { + return ZarrStatus_InvalidArgument; + } - if (bytes_of_external_metadata == 0) { - LOG_ERROR("Invalid length: %zu. Must be greater than 0", - bytes_of_external_metadata); - return ZarrStatus_InvalidArgument; - } + settings->compressor = compression_settings->compressor; + settings->compression_codec = compression_settings->codec; + settings->compression_level = compression_settings->level; + settings->compression_shuffle = compression_settings->shuffle; - size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); - if (nbytes < 2) { - settings->custom_metadata = "{}"; return ZarrStatus_Success; } - auto val = nlohmann::json::parse(external_metadata, - external_metadata + nbytes, - nullptr, // callback - false, // allow exceptions - true // ignore comments - ); - - if (val.is_discarded()) { - LOG_ERROR("Invalid JSON: %s", external_metadata); - return ZarrStatus_InvalidArgument; - } - settings->custom_metadata = val.dump(); + ZarrStatus ZarrStreamSettings_set_custom_metadata( + ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata) + { + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return ZarrStatus_InvalidArgument; + } - return ZarrStatus_Success; -} + if (!external_metadata) { + LOG_ERROR("Null pointer: custom_metadata"); + return ZarrStatus_InvalidArgument; + } -ZarrStatus -ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, - ZarrDataType data_type) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT( - data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); + if (bytes_of_external_metadata == 0) { + LOG_ERROR("Invalid length: %zu. Must be greater than 0", + bytes_of_external_metadata); + return ZarrStatus_InvalidArgument; + } - settings->dtype = data_type; - return ZarrStatus_Success; -} + size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); + if (nbytes < 2) { + settings->custom_metadata = "{}"; + return ZarrStatus_Success; + } -ZarrStatus -ZarrStreamSettings_reserve_dimensions(ZarrStreamSettings* settings, - size_t count) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(count >= zarr_dimension_min && - count <= zarr_dimension_max, - "Invalid count: %zu. Count must be between %d and %d", - count, - zarr_dimension_min, - zarr_dimension_max); - - settings->dimensions.resize(count); - return ZarrStatus_Success; -} + auto val = nlohmann::json::parse(external_metadata, + external_metadata + nbytes, + nullptr, // callback + false, // allow exceptions + true // ignore comments + ); -ZarrStatus -ZarrStreamSettings_set_dimension(ZarrStreamSettings* settings, - size_t index, - const ZarrDimensionProperties* dimension) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); - EXPECT_VALID_INDEX(index < settings->dimensions.size(), - "Invalid index: %zu. Must be less than %zu", - index, - settings->dimensions.size()); + if (val.is_discarded()) { + LOG_ERROR("Invalid JSON: %s", external_metadata); + return ZarrStatus_InvalidArgument; + } + settings->custom_metadata = val.dump(); - if (!validate_dimension(dimension)) { - return ZarrStatus_InvalidArgument; + return ZarrStatus_Success; } - struct ZarrDimension_s& dim = settings->dimensions[index]; + ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); - dim.name = trim(dimension->name, dimension->bytes_of_name - 1); - dim.type = dimension->type; - dim.array_size_px = dimension->array_size_px; - dim.chunk_size_px = dimension->chunk_size_px; - dim.shard_size_chunks = dimension->shard_size_chunks; + settings->dtype = data_type; + return ZarrStatus_Success; + } - return ZarrStatus_Success; -} + ZarrStatus ZarrStreamSettings_reserve_dimensions( + ZarrStreamSettings* settings, + size_t count) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + count >= zarr_dimension_min && count <= zarr_dimension_max, + "Invalid count: %zu. Count must be between %d and %d", + count, + zarr_dimension_min, + zarr_dimension_max); -ZarrStatus -ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, - uint8_t multiscale) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + settings->dimensions.resize(count); + return ZarrStatus_Success; + } - settings->multiscale = multiscale > 0; - return ZarrStatus_Success; -} + ZarrStatus ZarrStreamSettings_set_dimension( + ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); + EXPECT_VALID_INDEX(index < settings->dimensions.size(), + "Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); -/* Getters */ -const char* -ZarrStreamSettings_get_store_path(const ZarrStreamSettings* settings) -{ - SETTINGS_GET_STRING(settings, store_path); -} + if (!validate_dimension(dimension)) { + return ZarrStatus_InvalidArgument; + } -ZarrS3Settings -ZarrStreamSettings_get_s3_settings(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); - return {}; - } - - ZarrS3Settings s3_settings = { - settings->s3_endpoint.c_str(), - settings->s3_endpoint.length() + 1, - settings->s3_bucket_name.c_str(), - settings->s3_bucket_name.length() + 1, - settings->s3_access_key_id.c_str(), - settings->s3_access_key_id.length() + 1, - settings->s3_secret_access_key.c_str(), - settings->s3_secret_access_key.length() + 1, - }; - return s3_settings; -} + struct ZarrDimension_s& dim = settings->dimensions[index]; -const char* -ZarrStreamSettings_get_custom_metadata(const ZarrStreamSettings* settings) -{ - SETTINGS_GET_STRING(settings, custom_metadata); -} + dim.name = trim(dimension->name, dimension->bytes_of_name - 1); + dim.type = dimension->type; + dim.array_size_px = dimension->array_size_px; + dim.chunk_size_px = dimension->chunk_size_px; + dim.shard_size_chunks = dimension->shard_size_chunks; -ZarrDataType -ZarrStreamSettings_get_data_type(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); - return ZarrDataType_uint8; + return ZarrStatus_Success; } - return static_cast(settings->dtype); -} -ZarrCompressionSettings -ZarrStreamSettings_get_compression(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty compression."); - return {}; - } - - ZarrCompressionSettings compression = { - .compressor = settings->compressor, - .codec = static_cast(settings->compression_codec), - .level = settings->compression_level, - .shuffle = settings->compression_shuffle, - }; - return compression; -} + ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); -size_t -ZarrStreamSettings_get_dimension_count(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning 0."); - return 0; + settings->multiscale = multiscale > 0; + return ZarrStatus_Success; } - return settings->dimensions.size(); -} -ZarrDimensionProperties -ZarrStreamSettings_get_dimension(const ZarrStreamSettings* settings, - size_t index) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty dimension."); - return {}; + /* Getters */ + const char* ZarrStreamSettings_get_store_path( + const ZarrStreamSettings* settings) + { + SETTINGS_GET_STRING(settings, store_path); } - if (index >= settings->dimensions.size()) { - LOG_ERROR("Invalid index: %zu. Must be less than %zu", - index, - settings->dimensions.size()); - return {}; + ZarrS3Settings ZarrStreamSettings_get_s3_settings( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); + return {}; + } + + ZarrS3Settings s3_settings = { + settings->s3_endpoint.c_str(), + settings->s3_endpoint.length() + 1, + settings->s3_bucket_name.c_str(), + settings->s3_bucket_name.length() + 1, + settings->s3_access_key_id.c_str(), + settings->s3_access_key_id.length() + 1, + settings->s3_secret_access_key.c_str(), + settings->s3_secret_access_key.length() + 1, + }; + return s3_settings; + } + + const char* ZarrStreamSettings_get_custom_metadata( + const ZarrStreamSettings* settings) + { + SETTINGS_GET_STRING(settings, custom_metadata); + } + + ZarrDataType ZarrStreamSettings_get_data_type( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); + return ZarrDataType_uint8; + } + return static_cast(settings->dtype); } - const auto& dim = settings->dimensions[index]; + ZarrCompressionSettings ZarrStreamSettings_get_compression( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty compression."); + return {}; + } - ZarrDimensionProperties dimension = { - .name = dim.name.c_str(), - .bytes_of_name = dim.name.size() + 1, - .type = dim.type, - .array_size_px = dim.array_size_px, - .chunk_size_px = dim.chunk_size_px, - .shard_size_chunks = dim.shard_size_chunks, - }; + ZarrCompressionSettings compression = { + .compressor = settings->compressor, + .codec = + static_cast(settings->compression_codec), + .level = settings->compression_level, + .shuffle = settings->compression_shuffle, + }; + return compression; + } + + size_t ZarrStreamSettings_get_dimension_count( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return settings->dimensions.size(); + } - return dimension; -} + ZarrDimensionProperties ZarrStreamSettings_get_dimension( + const ZarrStreamSettings* settings, + size_t index) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty dimension."); + return {}; + } -bool -ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning false."); - return false; + if (index >= settings->dimensions.size()) { + LOG_ERROR("Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + return {}; + } + + const auto& dim = settings->dimensions[index]; + + ZarrDimensionProperties dimension = { + .name = dim.name.c_str(), + .bytes_of_name = dim.name.size() + 1, + .type = dim.type, + .array_size_px = dim.array_size_px, + .chunk_size_px = dim.chunk_size_px, + .shard_size_chunks = dim.shard_size_chunks, + }; + + return dimension; } - return settings->multiscale; -} + + bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning false."); + return false; + } + return settings->multiscale; + } +} \ No newline at end of file From d099795ef23bb538bbb31b8e82c6ab9f4e9d30da Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:38:19 -0400 Subject: [PATCH 08/39] Document the StreamSettings getters. --- include/acquire.zarr.h | 62 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index eac50f35..e2a33051 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -17,6 +17,19 @@ extern "C" */ uint32_t Zarr_get_api_version(); + /** + * @brief Set the log level for the Zarr API. + * @param level The log level. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + + /** + * @brief Get the log level for the Zarr API. + * @return The log level for the Zarr API. + */ + ZarrLogLevel Zarr_get_log_level(); + /** * @brief Get the message for the given status code. * @param status The status code. @@ -138,27 +151,73 @@ extern "C" const char* external_metadata, size_t bytes_of_external_metadata); + /** + * @brief Get the store path for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The store path for the Zarr stream, or NULL on failure. + */ const char* ZarrStreamSettings_get_store_path( const ZarrStreamSettings* settings); + /** + * @brief Get the S3 settings for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The S3 settings for the Zarr stream, or an uninitialized struct on + * failure. + */ ZarrS3Settings ZarrStreamSettings_get_s3_settings( const ZarrStreamSettings* settings); + /** + * @brief Get the compression settings for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The compression settings for the Zarr stream, or an uninitialized + * struct on failure. + */ ZarrCompressionSettings ZarrStreamSettings_get_compression( const ZarrStreamSettings* settings); + /** + * @brief Get the data type for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The data type for the Zarr stream, or ZarrDataType_uint8 on failure. + */ ZarrDataType ZarrStreamSettings_get_data_type( const ZarrStreamSettings* settings); + /** + * @brief Get the number of dimensions in the Zarr stream settings struct. + * @param settings The Zarr stream settings struct. + * @return The number of dimensions in the Zarr stream settings struct, or 0 on + * failure. + */ size_t ZarrStreamSettings_get_dimension_count( const ZarrStreamSettings* settings); + /** + * @brief Get the properties for an acquisition dimension. + * @param settings The Zarr stream settings struct. + * @param index The index of the dimension to get. + * @return The properties for the @p index th dimension, or an uninitialized struct on + * failure. + */ ZarrDimensionProperties ZarrStreamSettings_get_dimension( const ZarrStreamSettings* settings, size_t index); + /** + * @brief Get the multiscale flag for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The multiscale flag for the Zarr stream, or false on failure. + */ bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); + /** + * @brief Get the JSON-formatted custom metadata for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The JSON-formatted custom metadata for the Zarr stream, or NULL on + * failure. + */ const char* ZarrStreamSettings_get_custom_metadata( const ZarrStreamSettings* settings); @@ -206,9 +265,6 @@ extern "C" */ ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); - ZarrStatus Zarr_set_log_level(ZarrLogLevel level); - ZarrLogLevel Zarr_get_log_level(); - #ifdef __cplusplus } #endif From da98e57ab3b359ccb56cdcd9a911f42a9bcb403f Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:41:40 -0400 Subject: [PATCH 09/39] call it 'type' --- include/zarr.types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/zarr.types.h b/include/zarr.types.h index fdb6c715..56a54a56 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -117,7 +117,7 @@ extern "C" { const char* name; /**< Name of the dimension */ size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ - ZarrDimensionType kind; /**< Type of the dimension */ + ZarrDimensionType type; /**< Type of the dimension */ uint32_t array_size_px; /**< Size of the array along this dimension in pixels */ uint32_t chunk_size_px; /**< Size of the chunks along this dimension in From fdf5b086ee86e493fcdd128a44d5bef2aaf9b9da Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:49:15 -0400 Subject: [PATCH 10/39] No need to double CHECK the settings pointer. --- tests/unit-tests/get-stream-parameters.cpp | 1 - tests/unit-tests/set-stream-parameters.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/unit-tests/get-stream-parameters.cpp b/tests/unit-tests/get-stream-parameters.cpp index c3d287a4..5dea2cf6 100644 --- a/tests/unit-tests/get-stream-parameters.cpp +++ b/tests/unit-tests/get-stream-parameters.cpp @@ -217,7 +217,6 @@ main() ZarrStreamSettings* settings = ZarrStreamSettings_create(); try { - CHECK(settings); check_preliminaries(settings); get_store_path(settings); get_s3_settings(settings); diff --git a/tests/unit-tests/set-stream-parameters.cpp b/tests/unit-tests/set-stream-parameters.cpp index d7ee16b7..f8dedcda 100644 --- a/tests/unit-tests/set-stream-parameters.cpp +++ b/tests/unit-tests/set-stream-parameters.cpp @@ -256,7 +256,6 @@ main() ZarrStreamSettings* settings = ZarrStreamSettings_create(); try { - CHECK(settings); check_preliminaries(settings); set_store(settings); set_compression(settings); From cbbc87d4d75fcc9bbf3ced6894be5bae31ffda87 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:31:01 -0400 Subject: [PATCH 11/39] Implement ZarrStream_s. --- src/streaming/CMakeLists.txt | 2 + src/streaming/zarr.stream.cpp | 423 ++++++++++++++++++++++++++++++++++ src/streaming/zarr.stream.hh | 63 +++++ 3 files changed, 488 insertions(+) create mode 100644 src/streaming/zarr.stream.cpp create mode 100644 src/streaming/zarr.stream.hh diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 2f9fb4fd..c0401aa7 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -4,6 +4,8 @@ add_library(${tgt} macros.hh stream.settings.hh stream.settings.cpp + zarr.stream.hh + zarr.stream.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp new file mode 100644 index 00000000..58f9284e --- /dev/null +++ b/src/streaming/zarr.stream.cpp @@ -0,0 +1,423 @@ +#include "macros.hh" +#include "zarr.stream.hh" +#include "acquire.zarr.h" + +#include + +#include + +namespace fs = std::filesystem; + +namespace { +bool +is_s3_acquisition(const struct ZarrStreamSettings_s* settings) +{ + return !settings->s3_endpoint.empty() && + !settings->s3_bucket_name.empty() && + !settings->s3_access_key_id.empty() && + !settings->s3_secret_access_key.empty(); +} + +bool +is_compressed_acquisition(const struct ZarrStreamSettings_s* settings) +{ + return settings->compressor != ZarrCompressor_None; +} + +[[nodiscard]] +bool +validate_s3_settings(const struct ZarrStreamSettings_s* settings) +{ + if (settings->s3_endpoint.empty()) { + LOG_ERROR("S3 endpoint is empty"); + return false; + } + if (settings->s3_bucket_name.length() < 3 || + settings->s3_bucket_name.length() > 63) { + LOG_ERROR("Invalid length for S3 bucket name: %zu. Must be between 3 " + "and 63 characters", + settings->s3_bucket_name.length()); + return false; + } + if (settings->s3_access_key_id.empty()) { + LOG_ERROR("S3 access key ID is empty"); + return false; + } + if (settings->s3_secret_access_key.empty()) { + LOG_ERROR("S3 secret access key is empty"); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_filesystem_store_path(std::string_view data_root) +{ + fs::path path(data_root); + fs::path parent_path = path.parent_path(); + if (parent_path.empty()) { + parent_path = "."; + } + + // parent path must exist and be a directory + if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { + LOG_ERROR("Parent path '%s' does not exist or is not a directory", + parent_path.c_str()); + return false; + } + + // parent path must be writable + const auto perms = fs::status(parent_path).permissions(); + const bool is_writable = + (perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none; + + if (!is_writable) { + LOG_ERROR("Parent path '%s' is not writable", parent_path.c_str()); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_compression_settings(const ZarrStreamSettings_s* settings) +{ + if (settings->compressor >= ZarrCompressorCount) { + LOG_ERROR("Invalid compressor: %d", settings->compressor); + return false; + } + + if (settings->compression_codec >= ZarrCompressionCodecCount) { + LOG_ERROR("Invalid compression codec: %d", settings->compression_codec); + return false; + } + + // we know the compressor is not None, so the codec must be set + if (settings->compression_codec == ZarrCompressionCodec_None) { + LOG_ERROR("Compression codec must be set when using a compressor"); + return false; + } + + if (settings->compression_level == 0 || settings->compression_level > 9) { + LOG_ERROR("Invalid compression level: %d. Must be between 1 and 9", + settings->compression_level); + return false; + } + + if (settings->compression_shuffle != BLOSC_NOSHUFFLE && + settings->compression_shuffle != BLOSC_SHUFFLE && + settings->compression_shuffle != BLOSC_BITSHUFFLE) { + LOG_ERROR("Invalid shuffle: %d. Must be %d (no shuffle), %d (byte " + "shuffle), or %d (bit shuffle)", + settings->compression_shuffle, + BLOSC_NOSHUFFLE, + BLOSC_SHUFFLE, + BLOSC_BITSHUFFLE); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_dimension(const struct ZarrDimension_s& dimension, + ZarrVersion version, + bool is_append) +{ + if (dimension.name.empty()) { + LOG_ERROR("Invalid name. Must not be empty"); + return false; + } + + if (dimension.type >= ZarrDimensionTypeCount) { + LOG_ERROR("Invalid dimension type: %d", dimension.type); + return false; + } + + if (!is_append && dimension.array_size_px == 0) { + LOG_ERROR("Array size must be nonzero"); + return false; + } + + if (dimension.chunk_size_px == 0) { + LOG_ERROR("Chunk size must be nonzero"); + return false; + } + + if (version == ZarrVersion_3 && dimension.shard_size_chunks == 0) { + LOG_ERROR("Shard size must be nonzero"); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_settings(const struct ZarrStreamSettings_s* settings, + ZarrVersion version) +{ + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return false; + } + if (version < ZarrVersion_2 || version >= ZarrVersionCount) { + LOG_ERROR("Invalid Zarr version: %d", version); + return false; + } + + std::string_view store_path(settings->store_path); + + // we require the store path (root of the dataset) to be nonempty + if (store_path.empty()) { + LOG_ERROR("Store path is empty"); + return false; + } + + if (is_s3_acquisition(settings) && !validate_s3_settings(settings)) { + return false; + } else if (!is_s3_acquisition(settings) && + !validate_filesystem_store_path(store_path)) { + return false; + } + + if (settings->dtype >= ZarrDataTypeCount) { + LOG_ERROR("Invalid data type: %d", settings->dtype); + return false; + } + + if (is_compressed_acquisition(settings) && + !validate_compression_settings(settings)) { + return false; + } + + // validate the dimensions individually + for (size_t i = 0; i < settings->dimensions.size(); ++i) { + if (!validate_dimension(settings->dimensions[i], version, i == 0)) { + return false; + } + } + + return true; +} +} // namespace + +/* ZarrStream_s implementation */ + +ZarrStream::ZarrStream_s(struct ZarrStreamSettings_s* settings, + ZarrVersion version) + : settings_(*settings) + , version_(version) + , error_() +{ + // create the data store + EXPECT(create_store_(), "%s", error_.c_str()); + + // allocate writers + EXPECT(create_writers_(), "%s", error_.c_str()); + + // allocate multiscale frame placeholders + create_scaled_frames_(); + + // allocate metadata sinks + EXPECT(create_metadata_sinks_(), "%s", error_.c_str()); + + // write base metadata + EXPECT(write_base_metadata_(), "%s", error_.c_str()); + + // write group metadata + EXPECT(write_group_metadata_(), "%s", error_.c_str()); + + // write external metadata + EXPECT(write_external_metadata_(), "%s", error_.c_str()); +} + +ZarrStream_s::~ZarrStream_s() +{ + try { + // must precede close of chunk file + write_group_metadata_(); + } catch (const std::exception& e) { + LOG_ERROR("Error finalizing Zarr stream: %s", e.what()); + } +} + +size_t +ZarrStream::append(const void* data, size_t nbytes) +{ + // TODO (aliddell): implement this + return 0; +} + +void +ZarrStream_s::set_error_(const std::string& msg) +{ + error_ = msg; +} + +bool +ZarrStream_s::create_store_() +{ + if (is_s3_acquisition(&settings_)) { + // TODO (aliddell): implement this + } else { + if (fs::exists(settings_.store_path)) { + // remove everything inside the store path + std::error_code ec; + fs::remove_all(settings_.store_path, ec); + + if (ec) { + set_error_("Failed to remove existing store path '" + + settings_.store_path + "': " + ec.message()); + return false; + } + } + + // create the store path + { + std::error_code ec; + if (!fs::create_directories(settings_.store_path, ec)) { + set_error_("Failed to create store path '" + + settings_.store_path + "': " + ec.message()); + return false; + } + } + } + + return true; +} + +bool +ZarrStream_s::create_writers_() +{ + // TODO (aliddell): implement this + return true; +} + +void +ZarrStream_s::create_scaled_frames_() +{ + if (settings_.multiscale) { + // TODO (aliddell): implement this + } +} + +bool +ZarrStream_s::create_metadata_sinks_() +{ + // TODO (aliddell): implement this + return true; +} + +bool +ZarrStream_s::write_base_metadata_() +{ + // TODO (aliddell): implement this + return true; +} + +bool +ZarrStream_s::write_group_metadata_() +{ + // TODO (aliddell): implement this + return true; +} + +bool +ZarrStream_s::write_external_metadata_() +{ + // TODO (aliddell): implement this + return true; +} + +nlohmann::json +ZarrStream_s::make_multiscale_metadata_() const +{ + // TODO (aliddell): implement this + return {}; +} + +void +ZarrStream_s::write_multiscale_frames_(const uint8_t* data, + size_t bytes_of_data) +{ + if (!settings_.multiscale) { + return; + } + + // TODO (aliddell): implement this +} + +// C API +extern "C" +{ + ZarrStream* ZarrStream_create(struct ZarrStreamSettings_s* settings, + ZarrVersion version) + { + if (!validate_settings(settings, version)) { + return nullptr; + } + + // initialize the stream + ZarrStream_s* stream = nullptr; + + try { + stream = new ZarrStream(settings, version); + } catch (const std::bad_alloc&) { + LOG_ERROR("Failed to allocate memory for Zarr stream"); + } catch (const std::exception& e) { + LOG_ERROR("Error creating Zarr stream: %s", e.what()); + } + + return stream; + } + + void ZarrStream_destroy(ZarrStream* stream) + { + delete stream; + } + + ZarrStatus ZarrStream_append(ZarrStream* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out) + { + EXPECT_VALID_ARGUMENT(stream, "Null pointer: stream"); + EXPECT_VALID_ARGUMENT(data, "Null pointer: data"); + EXPECT_VALID_ARGUMENT(bytes_out, "Null pointer: bytes_out"); + + try { + *bytes_out = stream->append(data, bytes_in); + } catch (const std::exception& e) { + LOG_ERROR("Error appending data: %s", e.what()); + return ZarrStatus_InternalError; + } + + return ZarrStatus_Success; + } + + ZarrVersion ZarrStream_get_version(const ZarrStream* stream) + { + if (!stream) { + LOG_WARNING("Null pointer: stream. Returning ZarrVersion_2"); + return ZarrVersion_2; + } + + return stream->version(); + } + + ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream) + { + if (!stream) { + LOG_WARNING("Null pointer: stream. Returning nullptr"); + return nullptr; + } + + return ZarrStreamSettings_copy(&stream->settings()); + } +} \ No newline at end of file diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh new file mode 100644 index 00000000..0300bb90 --- /dev/null +++ b/src/streaming/zarr.stream.hh @@ -0,0 +1,63 @@ +#pragma once + +#include "stream.settings.hh" + +#include + +#include // size_t +#include // unique_ptr + +struct ZarrStream_s +{ + public: + ZarrStream_s(struct ZarrStreamSettings_s* settings, ZarrVersion version); + ~ZarrStream_s(); + + /** + * @brief Append data to the stream. + * @param data The data to append. + * @param nbytes The number of bytes to append. + * @return The number of bytes appended. + */ + size_t append(const void* data, size_t nbytes); + + ZarrVersion version() const { return version_; } + const ZarrStreamSettings_s& settings() const { return settings_; } + + private: + struct ZarrStreamSettings_s settings_; + ZarrVersion version_; // Zarr version. 2 or 3. + std::string error_; // error message. If nonempty, an error occurred. + + /** + * @brief Set an error message. + * @param msg The error message to set. + */ + void set_error_(const std::string& msg); + + /** @brief Create the data store. */ + [[nodiscard]] bool create_store_(); + + /** @brief Create the writers. */ + [[nodiscard]] bool create_writers_(); + + /** @brief Create placeholders for multiscale frames. */ + void create_scaled_frames_(); + + /** @brief Create the metadata sinks. */ + [[nodiscard]] bool create_metadata_sinks_(); + + /** @brief Write per-acquisition metadata. */ + [[nodiscard]] bool write_base_metadata_(); + + /** @brief Write Zarr group metadata. */ + bool write_group_metadata_(); + + /** @brief Write external metadata. */ + [[nodiscard]] bool write_external_metadata_(); + + /** @brief Construct OME metadata pertaining to the multiscale pyramid. */ + nlohmann::json make_multiscale_metadata_() const; + + void write_multiscale_frames_(const uint8_t* data, size_t bytes_of_data); +}; From 0f28cfc01cb197b7d8a7167fc906ec600e287193 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 15:31:54 -0400 Subject: [PATCH 12/39] Test ZarrStream_s. --- src/streaming/zarr.stream.cpp | 20 ++++ tests/unit-tests/CMakeLists.txt | 6 +- tests/unit-tests/create-stream.cpp | 86 ++++++++++++++++ tests/unit-tests/stream-get-settings.cpp | 124 +++++++++++++++++++++++ 4 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 tests/unit-tests/create-stream.cpp create mode 100644 tests/unit-tests/stream-get-settings.cpp diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp index 58f9284e..15f4128b 100644 --- a/src/streaming/zarr.stream.cpp +++ b/src/streaming/zarr.stream.cpp @@ -196,6 +196,26 @@ validate_settings(const struct ZarrStreamSettings_s* settings, return false; } + // we must have at least 3 dimensions + if (settings->dimensions.size() < 3) { + LOG_ERROR("Invalid number of dimensions: %zu. Must be at least 3", + settings->dimensions.size()); + return false; + } + + // check the final dimension (width), must be space + if (settings->dimensions.back().type != ZarrDimensionType_Space) { + LOG_ERROR("Last dimension must be of type Space"); + return false; + } + + // check the penultimate dimension (height), must be space + if (settings->dimensions[settings->dimensions.size() - 2].type != + ZarrDimensionType_Space) { + LOG_ERROR("Second to last dimension must be of type Space"); + return false; + } + // validate the dimensions individually for (size_t i = 0; i < settings->dimensions.size(); ++i) { if (!validate_dimension(settings->dimensions[i], version, i == 0)) { diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index f5d9264b..9e8090de 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -1,8 +1,10 @@ set(project acquire-zarr) set(tests - set-stream-parameters - get-stream-parameters + set-stream-parameters + get-stream-parameters + create-stream + stream-get-settings ) foreach (name ${tests}) diff --git a/tests/unit-tests/create-stream.cpp b/tests/unit-tests/create-stream.cpp new file mode 100644 index 00000000..1decf818 --- /dev/null +++ b/tests/unit-tests/create-stream.cpp @@ -0,0 +1,86 @@ +#include "acquire.zarr.h" +#include "zarr.stream.hh" +#include "unit.test.macros.hh" + +#include + +namespace fs = std::filesystem; + +void +configure_stream_dimensions(ZarrStreamSettings* settings) +{ + ZarrStreamSettings_reserve_dimensions(settings, 3); + + ZarrDimensionProperties dim = { + .name = "t", + .bytes_of_name = sizeof("t"), + .type = ZarrDimensionType_Time, + .array_size_px = 100, + .chunk_size_px = 10, + }; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 0, &dim), + ZarrStatus_Success); + + dim.name = "y"; + dim.type = ZarrDimensionType_Space; + dim.array_size_px = 200; + dim.chunk_size_px = 20; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 1, &dim), + ZarrStatus_Success); + + dim.name = "x"; + dim.array_size_px = 300; + dim.chunk_size_px = 30; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 2, &dim), + ZarrStatus_Success); +} + +int +main() +{ + int retval = 1; + + ZarrStream* stream; + ZarrStreamSettings* settings = ZarrStreamSettings_create(); + if (!settings) { + LOG_ERROR("Failed to create ZarrStreamSettings"); + return retval; + } + + try { + // try to create a stream with no store path + stream = ZarrStream_create(settings, ZarrVersion_2); + CHECK(nullptr == stream); + + // try to create a stream with no dimensions + settings->store_path = TEST ".zarr"; + stream = ZarrStream_create(settings, ZarrVersion_2); + CHECK(nullptr == stream); + CHECK(!fs::exists(settings->store_path)); + + configure_stream_dimensions(settings); + stream = ZarrStream_create(settings, ZarrVersion_2); + CHECK(nullptr != stream); + CHECK(fs::is_directory(settings->store_path)); + + EXPECT_EQ(int, "%d", ZarrStream_get_version(stream), ZarrVersion_2); + + retval = 0; + } catch (const std::exception& exception) { + LOG_ERROR("%s", exception.what()); + } + + // cleanup + if (fs::is_directory(settings->store_path)) { + fs::remove_all(settings->store_path); + } + ZarrStreamSettings_destroy(settings); + ZarrStream_destroy(stream); + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/stream-get-settings.cpp b/tests/unit-tests/stream-get-settings.cpp new file mode 100644 index 00000000..22f3d662 --- /dev/null +++ b/tests/unit-tests/stream-get-settings.cpp @@ -0,0 +1,124 @@ +#include "acquire.zarr.h" +#include "zarr.stream.hh" +#include "unit.test.macros.hh" + +#include + +namespace fs = std::filesystem; + +void +configure_stream_dimensions(ZarrStreamSettings* settings) +{ + ZarrStreamSettings_reserve_dimensions(settings, 3); + + ZarrDimensionProperties dim = { + .name = "t", + .bytes_of_name = sizeof("t"), + .type = ZarrDimensionType_Time, + .array_size_px = 100, + .chunk_size_px = 10, + }; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 0, &dim), + ZarrStatus_Success); + + dim.name = "y"; + dim.type = ZarrDimensionType_Space; + dim.array_size_px = 200; + dim.chunk_size_px = 20; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 1, &dim), + ZarrStatus_Success); + + dim.name = "x"; + dim.array_size_px = 300; + dim.chunk_size_px = 30; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 2, &dim), + ZarrStatus_Success); +} + +void +compare_settings(const ZarrStreamSettings* settings, + const ZarrStreamSettings* settings_copy) +{ + EXPECT_STR_EQ(settings->store_path.c_str(), + settings_copy->store_path.c_str()); + + EXPECT_STR_EQ(settings->s3_endpoint.c_str(), + settings_copy->s3_endpoint.c_str()); + EXPECT_STR_EQ(settings->s3_bucket_name.c_str(), + settings_copy->s3_bucket_name.c_str()); + EXPECT_STR_EQ(settings->s3_access_key_id.c_str(), + settings_copy->s3_access_key_id.c_str()); + EXPECT_STR_EQ(settings->s3_secret_access_key.c_str(), + settings_copy->s3_secret_access_key.c_str()); + + EXPECT_STR_EQ(settings->custom_metadata.c_str(), + settings_copy->custom_metadata.c_str()); + + EXPECT_EQ(int, "%d", settings->dtype, settings_copy->dtype); + + EXPECT_EQ(int, "%d", settings->compressor, settings_copy->compressor); + EXPECT_EQ(int, "%d", settings->compression_codec, settings_copy->compression_codec); + EXPECT_EQ(int, "%d", settings->compression_level, settings_copy->compression_level); + EXPECT_EQ(int, "%d", settings->compression_shuffle, settings_copy->compression_shuffle); + + EXPECT_EQ(int, "%d", settings->dimensions.size(), 3); + for (auto i = 0; i < 3; ++i) { + const auto& dim = &settings->dimensions[i]; + const auto& dim_copy = &settings_copy->dimensions[i]; + EXPECT_STR_EQ(dim->name.c_str(), dim_copy->name.c_str()); + EXPECT_EQ(int, "%d", dim->type, dim_copy->type); + EXPECT_EQ(int, "%d", dim->array_size_px, dim_copy->array_size_px); + EXPECT_EQ(int, "%d", dim->chunk_size_px, dim_copy->chunk_size_px); + EXPECT_EQ(int, "%d", dim->shard_size_chunks, dim_copy->shard_size_chunks); + } + + EXPECT_EQ(bool, "%d", settings->multiscale, settings_copy->multiscale); +} + +int +main() +{ + int retval = 1; + + ZarrStream* stream; + ZarrStreamSettings *settings = ZarrStreamSettings_create(), *settings_copy; + if (!settings) { + LOG_ERROR("Failed to create ZarrStreamSettings"); + return retval; + } + settings->store_path = TEST ".zarr"; + + try { + configure_stream_dimensions(settings); + stream = ZarrStream_create(settings, ZarrVersion_2); + CHECK(nullptr != stream); + CHECK(fs::is_directory(settings->store_path)); + + EXPECT_EQ(int, "%d", ZarrStream_get_version(stream), ZarrVersion_2); + + settings_copy = ZarrStream_get_settings(stream); + CHECK(nullptr != settings_copy); + CHECK(settings != settings_copy); + compare_settings(settings, settings_copy); + + retval = 0; + } catch (const std::exception& exception) { + LOG_ERROR("%s", exception.what()); + } + + // cleanup + if (fs::is_directory(settings->store_path)) { + fs::remove_all(settings->store_path); + } + ZarrStreamSettings_destroy(settings); + ZarrStreamSettings_destroy(settings_copy); + ZarrStream_destroy(stream); + + return retval; +} \ No newline at end of file From 1ca2a1defe2081e79ed26669cf82fb723cec9c86 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:32:10 -0400 Subject: [PATCH 13/39] Implement the rest of the Zarr API functions. --- src/streaming/CMakeLists.txt | 1 + src/streaming/zarr.api.cpp | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 src/streaming/zarr.api.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index c0401aa7..8feacc77 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -2,6 +2,7 @@ set(tgt acquire-zarr) add_library(${tgt} macros.hh + zarr.api.cpp stream.settings.hh stream.settings.cpp zarr.stream.hh diff --git a/src/streaming/zarr.api.cpp b/src/streaming/zarr.api.cpp new file mode 100644 index 00000000..e399d9a9 --- /dev/null +++ b/src/streaming/zarr.api.cpp @@ -0,0 +1,56 @@ +#include "zarr.types.h" +#include "macros.hh" + +#include // uint32_t + +#define ACQUIRE_ZARR_API_VERSION 0 + +extern "C" +{ + uint32_t Zarr_get_api_version() + { + return ACQUIRE_ZARR_API_VERSION; + } + + const char* Zarr_get_error_message(ZarrStatus error) + { + switch (error) { + case ZarrStatus_Success: + return "Success"; + case ZarrStatus_InvalidArgument: + return "Invalid argument"; + case ZarrStatus_Overflow: + return "Buffer overflow"; + case ZarrStatus_InvalidIndex: + return "Invalid index"; + case ZarrStatus_NotYetImplemented: + return "Not yet implemented"; + case ZarrStatus_InternalError: + return "Internal error"; + case ZarrStatus_OutOfMemory: + return "Out of memory"; + case ZarrStatus_IOError: + return "I/O error"; + case ZarrStatus_CompressionError: + return "Compression error"; + case ZarrStatus_InvalidSettings: + return "Invalid settings"; + default: + return "Unknown error"; + } + } + + ZarrStatus Zarr_set_log_level(ZarrLogLevel level) + { + EXPECT_VALID_ARGUMENT( + level < ZarrLogLevelCount, "Invalid log level: %d", level); + + Logger::set_log_level(level); + return ZarrStatus_Success; + } + + ZarrLogLevel Zarr_get_log_level() + { + return Logger::get_log_level(); + } +} \ No newline at end of file From d370e1718a8db25a5989f635dedfcddeaf7075b9 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 15:54:57 -0400 Subject: [PATCH 14/39] Implement and test Zarr common functions. --- src/streaming/CMakeLists.txt | 2 + src/streaming/zarr.common.cpp | 297 ++++++++++++++++++ src/streaming/zarr.common.hh | 134 ++++++++ tests/unit-tests/CMakeLists.txt | 5 + .../common-chunk-internal-offset.cpp | 184 +++++++++++ .../unit-tests/common-chunk-lattice-index.cpp | 83 +++++ .../common-shard-index-for-chunk.cpp | 195 ++++++++++++ .../common-shard-internal-index.cpp | 75 +++++ tests/unit-tests/common-tile-group-offset.cpp | 108 +++++++ 9 files changed, 1083 insertions(+) create mode 100644 src/streaming/zarr.common.cpp create mode 100644 src/streaming/zarr.common.hh create mode 100644 tests/unit-tests/common-chunk-internal-offset.cpp create mode 100644 tests/unit-tests/common-chunk-lattice-index.cpp create mode 100644 tests/unit-tests/common-shard-index-for-chunk.cpp create mode 100644 tests/unit-tests/common-shard-internal-index.cpp create mode 100644 tests/unit-tests/common-tile-group-offset.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 8feacc77..f1d63c31 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -7,6 +7,8 @@ add_library(${tgt} stream.settings.cpp zarr.stream.hh zarr.stream.cpp + zarr.common.hh + zarr.common.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/zarr.common.cpp b/src/streaming/zarr.common.cpp new file mode 100644 index 00000000..8cb903af --- /dev/null +++ b/src/streaming/zarr.common.cpp @@ -0,0 +1,297 @@ +#include "macros.hh" +#include "zarr.common.hh" + +#include + +size_t +zarr::bytes_of_type(ZarrDataType data_type) +{ + switch (data_type) { + case ZarrDataType_int8: + case ZarrDataType_uint8: + return 1; + case ZarrDataType_int16: + case ZarrDataType_uint16: + return 2; + case ZarrDataType_int32: + case ZarrDataType_uint32: + case ZarrDataType_float32: + return 4; + case ZarrDataType_int64: + case ZarrDataType_uint64: + case ZarrDataType_float64: + return 8; + default: + throw std::runtime_error("Invalid data type: " + + std::to_string(data_type)); + } +} + +size_t +zarr::bytes_of_frame(const std::vector& dims, ZarrDataType type) +{ + return bytes_of_type(type) * dims.back().array_size_px * + dims[dims.size() - 2].array_size_px; +} + +size_t +zarr::chunks_along_dimension(const Dimension& dimension) +{ + EXPECT(dimension.chunk_size_px > 0, "Invalid chunk size."); + + return (dimension.array_size_px + dimension.chunk_size_px - 1) / + dimension.chunk_size_px; +} + +size_t +zarr::shards_along_dimension(const Dimension& dimension) +{ + const size_t shard_size = dimension.shard_size_chunks; + if (shard_size == 0) { + return 0; + } + + const size_t n_chunks = chunks_along_dimension(dimension); + return (n_chunks + shard_size - 1) / shard_size; +} + +size_t +zarr::chunk_lattice_index(size_t frame_id, + size_t dimension_idx, + const std::vector& dims) +{ + // the last two dimensions are special cases + EXPECT(dimension_idx < dims.size() - 2, + "Invalid dimension index: %zu", + dimension_idx); + + // the first dimension is a special case + if (dimension_idx == 0) { + size_t divisor = dims.front().chunk_size_px; + for (auto i = 1; i < dims.size() - 2; ++i) { + const auto& dim = dims[i]; + divisor *= dim.array_size_px; + } + + CHECK(divisor); + return frame_id / divisor; + } + + size_t mod_divisor = 1, div_divisor = 1; + for (auto i = dimension_idx; i < dims.size() - 2; ++i) { + const auto& dim = dims[i]; + mod_divisor *= dim.array_size_px; + div_divisor *= + (i == dimension_idx ? dim.chunk_size_px : dim.array_size_px); + } + + CHECK(mod_divisor); + CHECK(div_divisor); + + return (frame_id % mod_divisor) / div_divisor; +} + +size_t +zarr::tile_group_offset(size_t frame_id, const std::vector& dims) +{ + std::vector strides; + strides.push_back(1); + for (auto i = dims.size() - 1; i > 0; --i) { + const auto& dim = dims[i]; + CHECK(dim.chunk_size_px); + const auto a = dim.array_size_px, c = dim.chunk_size_px; + strides.insert(strides.begin(), strides.front() * ((a + c - 1) / c)); + } + + size_t offset = 0; + for (auto i = dims.size() - 3; i > 0; --i) { + const auto idx = chunk_lattice_index(frame_id, i, dims); + const auto stride = strides[i]; + offset += idx * stride; + } + + return offset; +} + +size_t +zarr::chunk_internal_offset(size_t frame_id, + const std::vector& dims, + ZarrDataType type) +{ + const Dimension& x_dim = dims.back(); + const Dimension& y_dim = dims[dims.size() - 2]; + const auto tile_size = + bytes_of_type(type) * x_dim.chunk_size_px * y_dim.chunk_size_px; + + size_t offset = 0; + std::vector array_strides, chunk_strides; + array_strides.push_back(1); + chunk_strides.push_back(1); + + for (auto i = (int)dims.size() - 3; i >= 0; --i) { + const auto& dim = dims[i]; + + if (i > 0) { + CHECK(dim.array_size_px); + } + + CHECK(dim.chunk_size_px); + CHECK(array_strides.front()); + + const auto internal_idx = + i == 0 ? (frame_id / array_strides.front()) % dim.chunk_size_px + : (frame_id / array_strides.front()) % dim.array_size_px % + dim.chunk_size_px; + offset += internal_idx * chunk_strides.front(); + + array_strides.insert(array_strides.begin(), + array_strides.front() * dim.array_size_px); + chunk_strides.insert(chunk_strides.begin(), + chunk_strides.front() * dim.chunk_size_px); + } + + return offset * tile_size; +} + +size_t +zarr::number_of_chunks_in_memory(const std::vector& dimensions) +{ + size_t n_chunks = 1; + for (auto i = 1; i < dimensions.size(); ++i) { + n_chunks *= chunks_along_dimension(dimensions[i]); + } + + return n_chunks; +} + +size_t +zarr::bytes_per_chunk(const std::vector& dimensions, + ZarrDataType type) +{ + auto n_bytes = bytes_of_type(type); + for (const auto& d : dimensions) { + n_bytes *= d.chunk_size_px; + } + + return n_bytes; +} + +size_t +zarr::number_of_shards(const std::vector& dimensions) +{ + size_t n_shards = 1; + for (auto i = 1; i < dimensions.size(); ++i) { + const auto& dim = dimensions[i]; + n_shards *= shards_along_dimension(dim); + } + + return n_shards; +} + +size_t +zarr::chunks_per_shard(const std::vector& dimensions) +{ + size_t n_chunks = 1; + for (const auto& dim : dimensions) { + n_chunks *= dim.shard_size_chunks; + } + + return n_chunks; +} + +size_t +zarr::shard_index_for_chunk(size_t chunk_index, + const std::vector& dimensions) +{ + // make chunk strides + std::vector chunk_strides(1, 1); + for (auto i = dimensions.size() - 1; i > 0; --i) { + const auto& dim = dimensions[i]; + chunk_strides.insert(chunk_strides.begin(), + chunk_strides.front() * + chunks_along_dimension(dim)); + CHECK(chunk_strides.front()); + } + + // get chunk indices + std::vector chunk_lattice_indices; + for (auto i = dimensions.size() - 1; i > 0; --i) { + chunk_lattice_indices.insert(chunk_lattice_indices.begin(), + chunk_index % chunk_strides[i - 1] / + chunk_strides[i]); + } + chunk_lattice_indices.insert(chunk_lattice_indices.begin(), + chunk_index / chunk_strides.front()); + + // make shard strides + std::vector shard_strides(1, 1); + for (auto i = dimensions.size() - 1; i > 0; --i) { + const auto& dim = dimensions[i]; + shard_strides.insert(shard_strides.begin(), + shard_strides.front() * + shards_along_dimension(dim)); + CHECK(shard_strides.front()); + } + + std::vector shard_lattice_indices; + for (auto i = 0; i < dimensions.size(); ++i) { + shard_lattice_indices.push_back(chunk_lattice_indices[i] / + dimensions[i].shard_size_chunks); + } + + size_t index = 0; + for (auto i = 0; i < dimensions.size(); ++i) { + index += shard_lattice_indices[i] * shard_strides[i]; + } + + return index; +} + +size_t +zarr::shard_internal_index(size_t chunk_idx, + const std::vector& dimensions) +{ + // make chunk strides + std::vector chunk_strides(1, 1); + for (auto i = dimensions.size() - 1; i > 0; --i) { + const auto& dim = dimensions[i]; + chunk_strides.insert(chunk_strides.begin(), + chunk_strides.front() * + chunks_along_dimension(dim)); + CHECK(chunk_strides.front()); + } + + // get chunk indices + std::vector chunk_lattice_indices; + for (auto i = dimensions.size() - 1; i > 0; --i) { + chunk_lattice_indices.insert(chunk_lattice_indices.begin(), + chunk_idx % chunk_strides.at(i - 1) / + chunk_strides[i]); + } + chunk_lattice_indices.insert(chunk_lattice_indices.begin(), + chunk_idx / chunk_strides.front()); + + // make shard lattice indices + std::vector shard_lattice_indices; + for (auto i = 0; i < dimensions.size(); ++i) { + shard_lattice_indices.push_back(chunk_lattice_indices[i] / + dimensions[i].shard_size_chunks); + } + + std::vector chunk_internal_strides(1, 1); + for (auto i = dimensions.size() - 1; i > 0; --i) { + const auto& dim = dimensions[i]; + chunk_internal_strides.insert(chunk_internal_strides.begin(), + chunk_internal_strides.front() * + dim.shard_size_chunks); + } + + size_t index = 0; + + for (auto i = 0; i < dimensions.size(); ++i) { + index += (chunk_lattice_indices[i] % dimensions[i].shard_size_chunks) * + chunk_internal_strides[i]; + } + + return index; +} diff --git a/src/streaming/zarr.common.hh b/src/streaming/zarr.common.hh new file mode 100644 index 00000000..93f2c3e9 --- /dev/null +++ b/src/streaming/zarr.common.hh @@ -0,0 +1,134 @@ +#pragma once + +#include "stream.settings.hh" +#include "zarr.stream.hh" + +#include "acquire.zarr.h" + +namespace zarr { +using Dimension = ZarrDimension_s; + +/** + * @brief Get the number of bytes for a given data type. + * @param data_type The data type. + * @return The number of bytes for the data type. + */ +size_t +bytes_of_type(ZarrDataType data_type); + +/** + * @brief Get the number of bytes for a frame with the given dimensions and + * data type. + * @param dims The dimensions of the full array. + * @param type The data type of the array. + * @return The number of bytes for a single frame. + */ +size_t +bytes_of_frame(const std::vector& dims, ZarrDataType type); + +/** + * @brief Get the number of chunks along a dimension. + * @param dimension A dimension. + * @return The number of, possibly ragged, chunks along the dimension, given + * the dimension's array and chunk sizes. + */ +size_t +chunks_along_dimension(const Dimension& dimension); + +/** + * @brief Get the number of shards along a dimension. + * @param dimension A dimension. + * @return The number of shards along the dimension, given the dimension's + * array, chunk, and shard sizes. + */ +size_t +shards_along_dimension(const Dimension& dimension); + +/** + * @brief Get the index of a chunk in the chunk lattice for a given frame and + * dimension. + * @param frame_id The frame ID. + * @param dimension_idx The index of the dimension in the dimension vector. + * @param dims The dimensions. + * @return The index of the chunk in the chunk lattice. + */ +size_t +chunk_lattice_index(size_t frame_id, + size_t dimension_idx, + const std::vector& dims); + +/** + * @brief Find the offset in the array of chunk buffers for the given frame. + * @param frame_id The frame ID. + * @param dims The dimensions of the array. + * @return The offset in the array of chunk buffers. + */ +size_t +tile_group_offset(size_t frame_id, const std::vector& dims); + +/** + * @brief Find the byte offset inside a chunk for a given frame and data type. + * @param frame_id The frame ID. + * @param dims The dimensions of the array. + * @param type The data type of the array. + * @return The byte offset inside a chunk. + */ +size_t +chunk_internal_offset(size_t frame_id, + const std::vector& dims, + ZarrDataType type); + +/** + * @brief Get the number of chunks to hold in memory. + * @param dimensions The dimensions of the array. + * @return The number of chunks to buffer before writing out. + */ +size_t +number_of_chunks_in_memory(const std::vector& dimensions); + +/** + * @brief Get the size, in bytes, of a single raw chunk. + * @param dimensions The dimensions of the array. + * @param type The data type of the array. + * @return The number of bytes to allocate for a chunk. + */ +size_t +bytes_per_chunk(const std::vector& dimensions, + ZarrDataType type); + +/** + * @brief Get the number of shards to write at one time. + * @param dimensions The dimensions of the array. + * @return The number of shards to buffer and write out. + */ +size_t +number_of_shards(const std::vector& dimensions); + +/** + * @brief Get the number of chunks in a single shard. + * @param dimensions The dimensions of the array. + * @return The number of chunks in a shard. + */ +size_t +chunks_per_shard(const std::vector& dimensions); + +/** + * @brief Get the shard index for a given chunk index, given array dimensions. + * @param chunk_index The index of the chunk. + * @param dimensions The dimensions of the array. + * @return The index of the shard containing the chunk. + */ +size_t +shard_index_for_chunk(size_t chunk_index, + const std::vector& dimensions); + +/** + * @brief Get the streaming index of a chunk within a shard. + * @param chunk_index The index of the chunk. + * @param dimensions The dimensions of the array. + * @return The index of the chunk within the shard. + */ +size_t +shard_internal_index(size_t chunk_index, + const std::vector& dimensions); +} // namespace zarr \ No newline at end of file diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index 9e8090de..ab18184b 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -5,6 +5,11 @@ set(tests get-stream-parameters create-stream stream-get-settings + common-chunk-lattice-index + common-tile-group-offset + common-chunk-internal-offset + common-shard-index-for-chunk + common-shard-internal-index ) foreach (name ${tests}) diff --git a/tests/unit-tests/common-chunk-internal-offset.cpp b/tests/unit-tests/common-chunk-internal-offset.cpp new file mode 100644 index 00000000..ff45eb01 --- /dev/null +++ b/tests/unit-tests/common-chunk-internal-offset.cpp @@ -0,0 +1,184 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk + dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks + dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks + dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks + dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks + + try { + EXPECT_INT_EQ(zarr::chunk_internal_offset(0, dims, ZarrDataType_uint16), + 0); + EXPECT_INT_EQ(zarr::chunk_internal_offset(1, dims, ZarrDataType_uint16), + 512); + EXPECT_INT_EQ(zarr::chunk_internal_offset(2, dims, ZarrDataType_uint16), + 0); + EXPECT_INT_EQ(zarr::chunk_internal_offset(3, dims, ZarrDataType_uint16), + 512); + EXPECT_INT_EQ(zarr::chunk_internal_offset(4, dims, ZarrDataType_uint16), + 0); + EXPECT_INT_EQ(zarr::chunk_internal_offset(5, dims, ZarrDataType_uint16), + 1024); + EXPECT_INT_EQ(zarr::chunk_internal_offset(6, dims, ZarrDataType_uint16), + 1536); + EXPECT_INT_EQ(zarr::chunk_internal_offset(7, dims, ZarrDataType_uint16), + 1024); + EXPECT_INT_EQ(zarr::chunk_internal_offset(8, dims, ZarrDataType_uint16), + 1536); + EXPECT_INT_EQ(zarr::chunk_internal_offset(9, dims, ZarrDataType_uint16), + 1024); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(10, dims, ZarrDataType_uint16), 0); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(11, dims, ZarrDataType_uint16), 512); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(12, dims, ZarrDataType_uint16), 0); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(13, dims, ZarrDataType_uint16), 512); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(14, dims, ZarrDataType_uint16), 0); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(15, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(16, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(17, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(18, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(19, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(20, dims, ZarrDataType_uint16), 3072); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(21, dims, ZarrDataType_uint16), 3584); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(22, dims, ZarrDataType_uint16), 3072); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(23, dims, ZarrDataType_uint16), 3584); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(24, dims, ZarrDataType_uint16), 3072); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(25, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(26, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(27, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(28, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(29, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(30, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(31, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(32, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(33, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(34, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(35, dims, ZarrDataType_uint16), 5120); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(36, dims, ZarrDataType_uint16), 5632); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(37, dims, ZarrDataType_uint16), 5120); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(38, dims, ZarrDataType_uint16), 5632); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(39, dims, ZarrDataType_uint16), 5120); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(40, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(41, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(42, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(43, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(44, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(45, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(46, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(47, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(48, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(49, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(50, dims, ZarrDataType_uint16), 7168); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(51, dims, ZarrDataType_uint16), 7680); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(52, dims, ZarrDataType_uint16), 7168); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(53, dims, ZarrDataType_uint16), 7680); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(54, dims, ZarrDataType_uint16), 7168); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(55, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(56, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(57, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(58, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(59, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(60, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(61, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(62, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(63, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(64, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(65, dims, ZarrDataType_uint16), 9216); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(66, dims, ZarrDataType_uint16), 9728); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(67, dims, ZarrDataType_uint16), 9216); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(68, dims, ZarrDataType_uint16), 9728); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(69, dims, ZarrDataType_uint16), 9216); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(70, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(71, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(72, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(73, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(74, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(75, dims, ZarrDataType_uint16), 0); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-chunk-lattice-index.cpp b/tests/unit-tests/common-chunk-lattice-index.cpp new file mode 100644 index 00000000..3dc94d44 --- /dev/null +++ b/tests/unit-tests/common-chunk-lattice-index.cpp @@ -0,0 +1,83 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + try { + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk + dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks + dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks + dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks + dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks + + EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 0, dims), 1); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-index-for-chunk.cpp b/tests/unit-tests/common-shard-index-for-chunk.cpp new file mode 100644 index 00000000..6c012b43 --- /dev/null +++ b/tests/unit-tests/common-shard-index-for-chunk.cpp @@ -0,0 +1,195 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + try { + std::vector dims; + dims.emplace_back("t", + ZarrDimensionType_Time, + 0, + 5, // 5 timepoints / chunk + 2); // 2 chunks / shard + dims.emplace_back("c", + ZarrDimensionType_Channel, + 8, + 4, // 8 / 4 = 2 chunks + 2); // 4 / 2 = 2 shards + dims.emplace_back("z", + ZarrDimensionType_Space, + 6, + 2, // 6 / 2 = 3 chunks + 1); // 3 / 1 = 3 shards + dims.emplace_back("y", + ZarrDimensionType_Space, + 48, + 16, // 48 / 16 = 3 chunks + 1); // 3 / 1 = 3 shards + dims.emplace_back("x", + ZarrDimensionType_Space, + 64, + 16, // 64 / 16 = 4 chunks + 2); // 4 / 2 = 2 shards + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(12, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(13, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(14, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(15, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(16, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(17, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(18, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(19, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(20, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(21, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(22, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(23, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(24, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(25, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(26, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(27, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(28, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(29, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(30, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(31, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(32, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(33, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(34, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(35, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(36, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(37, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(38, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(39, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(40, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(41, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(42, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(43, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(44, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(45, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(46, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(47, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(48, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(49, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(50, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(51, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(52, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(53, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(54, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(55, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(56, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(57, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(58, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(59, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(60, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(61, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(62, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(63, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(64, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(65, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(66, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(67, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(68, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(69, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(70, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(71, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(72, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(73, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(74, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(75, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(76, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(77, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(78, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(79, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(80, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(81, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(82, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(83, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(84, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(85, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(86, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(87, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(88, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(89, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(90, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(91, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(92, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(93, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(94, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(95, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(96, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(97, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(98, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(99, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(100, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(101, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(102, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(103, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(104, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(105, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(106, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(107, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(108, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(109, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(110, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(111, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(112, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(113, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(114, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(115, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(116, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(117, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(118, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(119, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(120, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(121, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(122, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(123, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(124, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(125, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(126, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(127, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(128, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(129, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(130, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(131, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(132, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(133, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(134, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(135, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(136, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(137, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(138, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(139, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(140, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(141, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(142, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(143, dims), 17); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-internal-index.cpp b/tests/unit-tests/common-shard-internal-index.cpp new file mode 100644 index 00000000..346c13bf --- /dev/null +++ b/tests/unit-tests/common-shard-internal-index.cpp @@ -0,0 +1,75 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + std::vector dims; + dims.emplace_back("t", + ZarrDimensionType_Time, + 0, + 32, // 32 timepoints / chunk + 1); // 1 shard + dims.emplace_back("y", + ZarrDimensionType_Space, + 960, + 320, // 3 chunks + 2); // 2 ragged shards + dims.emplace_back("x", + ZarrDimensionType_Space, + 1080, + 270, // 4 chunks + 3); // 2 ragged shards + + try { + EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(0, dims), 0); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(1, dims), 1); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(2, dims), 2); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); + EXPECT_INT_EQ(zarr::shard_internal_index(3, dims), 0); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(4, dims), 3); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(5, dims), 4); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(6, dims), 5); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 1); + EXPECT_INT_EQ(zarr::shard_internal_index(7, dims), 3); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 2); + EXPECT_INT_EQ(zarr::shard_internal_index(8, dims), 0); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 2); + EXPECT_INT_EQ(zarr::shard_internal_index(9, dims), 1); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 2); + EXPECT_INT_EQ(zarr::shard_internal_index(10, dims), 2); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 3); + EXPECT_INT_EQ(zarr::shard_internal_index(11, dims), 0); + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-tile-group-offset.cpp b/tests/unit-tests/common-tile-group-offset.cpp new file mode 100644 index 00000000..be9efecc --- /dev/null +++ b/tests/unit-tests/common-tile-group-offset.cpp @@ -0,0 +1,108 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk + dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks + dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks + dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks + dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks + + try { + EXPECT_INT_EQ(zarr::tile_group_offset(0, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(1, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(2, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(3, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(4, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(5, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(6, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(7, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(8, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(9, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(10, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(11, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(12, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(13, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(14, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(15, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(16, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(17, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(18, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(19, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(20, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(21, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(22, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(23, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(24, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(25, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(26, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(27, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(28, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(29, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(30, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(31, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(32, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(33, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(34, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(35, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(36, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(37, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(38, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(39, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(40, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(41, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(42, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(43, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(44, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(45, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(46, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(47, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(48, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(49, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(50, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(51, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(52, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(53, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(54, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(55, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(56, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(57, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(58, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(59, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(60, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(61, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(62, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(63, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(64, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(65, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(66, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(67, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(68, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(69, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(70, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(71, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(72, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(73, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(74, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(75, dims), 0); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file From caf295b92a3c4bbc15056f55f4bcd7528c85134a Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 16:10:07 -0400 Subject: [PATCH 15/39] Implement and test ThreadPool, S3Connection{Pool}. Also implement Blosc compression parameters. --- src/streaming/CMakeLists.txt | 6 + src/streaming/blosc.compression.params.cpp | 29 ++ src/streaming/blosc.compression.params.hh | 25 ++ src/streaming/s3.connection.cpp | 254 ++++++++++++++++++ src/streaming/s3.connection.hh | 141 ++++++++++ src/streaming/thread.pool.cpp | 96 +++++++ src/streaming/thread.pool.hh | 56 ++++ tests/unit-tests/CMakeLists.txt | 3 + tests/unit-tests/s3-connection-put-object.cpp | 88 ++++++ .../s3-connection-upload-multipart-object.cpp | 127 +++++++++ .../thread-pool-push-to-job-queue.cpp | 60 +++++ 11 files changed, 885 insertions(+) create mode 100644 src/streaming/blosc.compression.params.cpp create mode 100644 src/streaming/blosc.compression.params.hh create mode 100644 src/streaming/s3.connection.cpp create mode 100644 src/streaming/s3.connection.hh create mode 100644 src/streaming/thread.pool.cpp create mode 100644 src/streaming/thread.pool.hh create mode 100644 tests/unit-tests/s3-connection-put-object.cpp create mode 100644 tests/unit-tests/s3-connection-upload-multipart-object.cpp create mode 100644 tests/unit-tests/thread-pool-push-to-job-queue.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index f1d63c31..ba22b948 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -9,6 +9,12 @@ add_library(${tgt} zarr.stream.cpp zarr.common.hh zarr.common.cpp + blosc.compression.params.hh + blosc.compression.params.cpp + thread.pool.hh + thread.pool.cpp + s3.connection.hh + s3.connection.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/blosc.compression.params.cpp b/src/streaming/blosc.compression.params.cpp new file mode 100644 index 00000000..80a77a35 --- /dev/null +++ b/src/streaming/blosc.compression.params.cpp @@ -0,0 +1,29 @@ +#include "blosc.compression.params.hh" + +const char* +zarr::blosc_codec_to_string(ZarrCompressionCodec codec) +{ + switch (codec) { + case ZarrCompressionCodec_BloscZstd: + return "zstd"; + case ZarrCompressionCodec_BloscLZ4: + return "lz4"; + default: + return "unrecognized codec"; + } +} + +zarr::BloscCompressionParams::BloscCompressionParams() + : clevel{ 1 } + , shuffle{ 1 } +{ +} + +zarr::BloscCompressionParams::BloscCompressionParams(std::string_view codec_id, + uint8_t clevel, + uint8_t shuffle) + : codec_id{ codec_id } + , clevel{ clevel } + , shuffle{ shuffle } +{ +} diff --git a/src/streaming/blosc.compression.params.hh b/src/streaming/blosc.compression.params.hh new file mode 100644 index 00000000..6f4165d8 --- /dev/null +++ b/src/streaming/blosc.compression.params.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "acquire.zarr.h" + +#include + +#include +#include + +namespace zarr { +const char* +blosc_codec_to_string(ZarrCompressionCodec codec); + +struct BloscCompressionParams +{ + std::string codec_id; + uint8_t clevel; + uint8_t shuffle; + + BloscCompressionParams(); + BloscCompressionParams(std::string_view codec_id, + uint8_t clevel, + uint8_t shuffle); +}; +} // namespace zarr diff --git a/src/streaming/s3.connection.cpp b/src/streaming/s3.connection.cpp new file mode 100644 index 00000000..89ee96ca --- /dev/null +++ b/src/streaming/s3.connection.cpp @@ -0,0 +1,254 @@ +#include "macros.hh" +#include "s3.connection.hh" + +#include + +#include +#include +#include + +zarr::S3Connection::S3Connection(const std::string& endpoint, + const std::string& access_key_id, + const std::string& secret_access_key) +{ + minio::s3::BaseUrl url(endpoint); + url.https = endpoint.starts_with("https"); + + provider_ = std::make_unique( + access_key_id, secret_access_key); + client_ = std::make_unique(url, provider_.get()); + + CHECK(client_); +} + +bool +zarr::S3Connection::check_connection() +{ + return static_cast(client_->ListBuckets()); +} + +bool +zarr::S3Connection::bucket_exists(std::string_view bucket_name) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + + minio::s3::BucketExistsArgs args; + args.bucket = bucket_name; + + auto response = client_->BucketExists(args); + return response.exist; +} + +bool +zarr::S3Connection::object_exists(std::string_view bucket_name, + std::string_view object_name) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_name.empty(), "Object name must not be empty."); + + minio::s3::StatObjectArgs args; + args.bucket = bucket_name; + args.object = object_name; + + auto response = client_->StatObject(args); + // casts to true if response code in 200 range and error message is empty + return static_cast(response); +} + +std::string +zarr::S3Connection::put_object(std::string_view bucket_name, + std::string_view object_name, + std::span data) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_name.empty(), "Object name must not be empty."); + EXPECT(!data.empty(), "Data must not be empty."); + + minio::utils::CharBuffer buffer((char*)const_cast(data.data()), + data.size()); + std::basic_istream stream(&buffer); + + LOG_DEBUG( + "Putting object %s in bucket %s", object_name.data(), bucket_name.data()); + minio::s3::PutObjectArgs args(stream, (long)data.size(), 0); + args.bucket = bucket_name; + args.object = object_name; + + auto response = client_->PutObject(args); + if (!response) { + LOG_ERROR("Failed to put object %s in bucket %s: %s", + object_name.data(), + bucket_name.data(), + response.Error().String().c_str()); + return {}; + } + + return response.etag; +} + +bool +zarr::S3Connection::delete_object(std::string_view bucket_name, + std::string_view object_name) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_name.empty(), "Object name must not be empty."); + + LOG_DEBUG("Deleting object %s from bucket %s", + object_name.data(), + bucket_name.data()); + minio::s3::RemoveObjectArgs args; + args.bucket = bucket_name; + args.object = object_name; + + auto response = client_->RemoveObject(args); + if (!response) { + LOG_ERROR("Failed to delete object %s from bucket %s: %s", + object_name.data(), + bucket_name.data(), + response.Error().String().c_str()); + return false; + } + + return true; +} + +std::string +zarr::S3Connection::create_multipart_object(std::string_view bucket_name, + std::string_view object_name) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_name.empty(), "Object name must not be empty."); + + LOG_DEBUG("Creating multipart object %s in bucket %s", + object_name.data(), + bucket_name.data()); + minio::s3::CreateMultipartUploadArgs args; + args.bucket = bucket_name; + args.object = object_name; + + auto response = client_->CreateMultipartUpload(args); + CHECK(!response.upload_id.empty()); + + return response.upload_id; +} + +std::string +zarr::S3Connection::upload_multipart_object_part(std::string_view bucket_name, + std::string_view object_name, + std::string_view upload_id, + std::span data, + unsigned int part_number) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_name.empty(), "Object name must not be empty."); + EXPECT(!data.empty(), "Number of bytes must be positive."); + EXPECT(part_number, "Part number must be positive."); + + LOG_DEBUG("Uploading multipart object part %zu for object %s in bucket %s", + part_number, + object_name.data(), + bucket_name.data()); + + std::string_view sv(reinterpret_cast(data.data()), + data.size()); + + minio::s3::UploadPartArgs args; + args.bucket = bucket_name; + args.object = object_name; + args.part_number = part_number; + args.upload_id = upload_id; + args.data = sv; + + auto response = client_->UploadPart(args); + if (!response) { + LOG_ERROR("Failed to upload part %zu for object %s in bucket %s: %s", + part_number, + object_name.data(), + bucket_name.data(), + response.Error().String().c_str()); + return {}; + } + + return response.etag; +} + +bool +zarr::S3Connection::complete_multipart_object( + std::string_view bucket_name, + std::string_view object_name, + std::string_view upload_id, + const std::list& parts) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_name.empty(), "Object name must not be empty."); + EXPECT(!upload_id.empty(), "Upload id must not be empty."); + EXPECT(!parts.empty(), "Parts list must not be empty."); + + LOG_DEBUG("Completing multipart object %s in bucket %s", + object_name.data(), + bucket_name.data()); + minio::s3::CompleteMultipartUploadArgs args; + args.bucket = bucket_name; + args.object = object_name; + args.upload_id = upload_id; + args.parts = parts; + + auto response = client_->CompleteMultipartUpload(args); + if (!response) { + LOG_ERROR("Failed to complete multipart object %s in bucket %s: %s", + object_name.data(), + bucket_name.data(), + response.Error().String().c_str()); + return false; + } + + return true; +} + +zarr::S3ConnectionPool::S3ConnectionPool(size_t n_connections, + const std::string& endpoint, + const std::string& access_key_id, + const std::string& secret_access_key) + : is_accepting_connections_{ true } +{ + for (auto i = 0; i < n_connections; ++i) { + auto connection = std::make_unique( + endpoint, access_key_id, secret_access_key); + + if (connection->check_connection()) { + connections_.push_back(std::make_unique( + endpoint, access_key_id, secret_access_key)); + } + } + + CHECK(!connections_.empty()); +} + +zarr::S3ConnectionPool::~S3ConnectionPool() noexcept +{ + is_accepting_connections_ = false; + cv_.notify_all(); +} + +std::unique_ptr +zarr::S3ConnectionPool::get_connection() +{ + std::unique_lock lock(connections_mutex_); + cv_.wait(lock, [this] { return !connections_.empty(); }); + + if (!is_accepting_connections_ || connections_.empty()) { + return nullptr; + } + + auto conn = std::move(connections_.back()); + connections_.pop_back(); + return conn; +} + +void +zarr::S3ConnectionPool::return_connection(std::unique_ptr&& conn) +{ + std::scoped_lock lock(connections_mutex_); + connections_.push_back(std::move(conn)); + cv_.notify_one(); +} diff --git a/src/streaming/s3.connection.hh b/src/streaming/s3.connection.hh new file mode 100644 index 00000000..8d92b93a --- /dev/null +++ b/src/streaming/s3.connection.hh @@ -0,0 +1,141 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace zarr { +class S3Connection +{ + public: + explicit S3Connection(const std::string& endpoint, + const std::string& access_key_id, + const std::string& secret_access_key); + + /** + * @brief Test a connection by listing all buckets at this connection's + * endpoint. + * @returns True if the connection is valid, otherwise false. + */ + bool check_connection(); + + /* Bucket operations */ + + /** + * @brief Check whether a bucket exists. + * @param bucket_name The name of the bucket. + * @returns True if the bucket exists, otherwise false. + * @throws std::runtime_error if the bucket name is empty. + */ + bool bucket_exists(std::string_view bucket_name); + + /* Object operations */ + + /** + * @brief Check whether an object exists. + * @param bucket_name The name of the bucket containing the object. + * @param object_name The name of the object. + * @returns True if the object exists, otherwise false. + * @throws std::runtime_error if the bucket name is empty or the object + * name is empty. + */ + bool object_exists(std::string_view bucket_name, + std::string_view object_name); + + /** + * @brief Put an object. + * @param bucket_name The name of the bucket to put the object in. + * @param object_name The name of the object. + * @param data The data to put in the object. + * @returns The etag of the object. + * @throws std::runtime_error if the bucket name is empty, the object name + * is empty, or @p data is empty. + */ + [[nodiscard]] std::string put_object(std::string_view bucket_name, + std::string_view object_name, + std::span data); + + /** + * @brief Delete an object. + * @param bucket_name The name of the bucket containing the object. + * @param object_name The name of the object. + * @returns True if the object was successfully deleted, otherwise false. + * @throws std::runtime_error if the bucket name is empty or the object + * name is empty. + */ + [[nodiscard]] bool delete_object(std::string_view bucket_name, + std::string_view object_name); + + /* Multipart object operations */ + + /// @brief Create a multipart object. + /// @param bucket_name The name of the bucket containing the object. + /// @param object_name The name of the object. + /// @returns The upload id of the multipart object. Nonempty if and only if + /// the operation succeeds. + /// @throws std::runtime_error if the bucket name is empty or the object + /// name is empty. + [[nodiscard]] std::string create_multipart_object( + std::string_view bucket_name, + std::string_view object_name); + + /// @brief Upload a part of a multipart object. + /// @param bucket_name The name of the bucket containing the object. + /// @param object_name The name of the object. + /// @param upload_id The upload id of the multipart object. + /// @param data The data to upload. + /// @param part_number The part number of the object. + /// @returns The etag of the uploaded part. Nonempty if and only if the + /// operation is successful. + /// @throws std::runtime_error if the bucket name is empty, the object name + /// is empty, @p data is empty, or @p part_number is 0. + [[nodiscard]] std::string upload_multipart_object_part( + std::string_view bucket_name, + std::string_view object_name, + std::string_view upload_id, + std::span data, + unsigned int part_number); + + /// @brief Complete a multipart object. + /// @param bucket_name The name of the bucket containing the object. + /// @param object_name The name of the object. + /// @param upload_id The upload id of the multipart object. + /// @param parts List of the parts making up the object. + /// @returns True if the object was successfully completed, otherwise false. + [[nodiscard]] bool complete_multipart_object( + std::string_view bucket_name, + std::string_view object_name, + std::string_view upload_id, + const std::list& parts); + + private: + std::unique_ptr client_; + std::unique_ptr provider_; +}; + +class S3ConnectionPool +{ + public: + S3ConnectionPool(size_t n_connections, + const std::string& endpoint, + const std::string& access_key_id, + const std::string& secret_access_key); + ~S3ConnectionPool() noexcept; + + std::unique_ptr get_connection(); + void return_connection(std::unique_ptr&& conn); + + private: + std::vector> connections_; + mutable std::mutex connections_mutex_; + std::condition_variable cv_; + + std::atomic is_accepting_connections_; +}; +} // namespace zarr diff --git a/src/streaming/thread.pool.cpp b/src/streaming/thread.pool.cpp new file mode 100644 index 00000000..82d91ff6 --- /dev/null +++ b/src/streaming/thread.pool.cpp @@ -0,0 +1,96 @@ +#include "thread.pool.hh" + +zarr::ThreadPool::ThreadPool(unsigned int n_threads, + std::function err) + : error_handler_{ err } + , is_accepting_jobs_{ true } +{ + n_threads = std::clamp( + n_threads, 1u, std::max(std::thread::hardware_concurrency(), 1u)); + + for (auto i = 0; i < n_threads; ++i) { + threads_.emplace_back([this] { thread_worker_(); }); + } +} + +zarr::ThreadPool::~ThreadPool() noexcept +{ + { + std::scoped_lock lock(jobs_mutex_); + while (!jobs_.empty()) { + jobs_.pop(); + } + } + + await_stop(); +} + +bool +zarr::ThreadPool::push_to_job_queue(JobT&& job) +{ + std::unique_lock lock(jobs_mutex_); + if (!is_accepting_jobs_) { + return false; + } + + jobs_.push(std::move(job)); + cv_.notify_one(); + + return true; +} + +void +zarr::ThreadPool::await_stop() noexcept +{ + { + std::scoped_lock lock(jobs_mutex_); + is_accepting_jobs_ = false; + + cv_.notify_all(); + } + + // spin down threads + for (auto& thread : threads_) { + if (thread.joinable()) { + thread.join(); + } + } +} + +std::optional +zarr::ThreadPool::pop_from_job_queue_() noexcept +{ + if (jobs_.empty()) { + return std::nullopt; + } + + auto job = std::move(jobs_.front()); + jobs_.pop(); + return job; +} + +bool +zarr::ThreadPool::should_stop_() const noexcept +{ + return !is_accepting_jobs_ && jobs_.empty(); +} + +void +zarr::ThreadPool::thread_worker_() +{ + while (true) { + std::unique_lock lock(jobs_mutex_); + cv_.wait(lock, [&] { return should_stop_() || !jobs_.empty(); }); + + if (should_stop_()) { + break; + } + + if (auto job = pop_from_job_queue_(); job.has_value()) { + lock.unlock(); + if (std::string err_msg; !job.value()(err_msg)) { + error_handler_(err_msg); + } + } + } +} \ No newline at end of file diff --git a/src/streaming/thread.pool.hh b/src/streaming/thread.pool.hh new file mode 100644 index 00000000..e8b95e95 --- /dev/null +++ b/src/streaming/thread.pool.hh @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace zarr { +class ThreadPool +{ + public: + using JobT = std::function; + + // The error handler `err` is called when a job returns false. This + // can happen when the job encounters an error, or otherwise fails. The + // std::string& argument to the error handler is a diagnostic message from + // the failing job and is logged to the error stream by the Zarr driver when + // the next call to `append()` is made. + ThreadPool(unsigned int n_threads, + std::function err); + ~ThreadPool() noexcept; + + /** + * @brief Push a job onto the job queue. + * + * @param job The job to push onto the queue. + * @return true if the job was successfully pushed onto the queue, false + * otherwise. + */ + [[nodiscard]] bool push_to_job_queue(JobT&& job); + + /** + * @brief Block until all jobs on the queue have processed, then spin down + * the threads. + * @note After calling this function, the job queue no longer accepts jobs. + */ + void await_stop() noexcept; + + private: + std::function error_handler_; + + std::vector threads_; + mutable std::mutex jobs_mutex_; + std::condition_variable cv_; + std::queue jobs_; + + std::atomic is_accepting_jobs_; + + std::optional pop_from_job_queue_() noexcept; + [[nodiscard]] bool should_stop_() const noexcept; + void thread_worker_(); +}; +} // zarr diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index ab18184b..c87823ac 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -10,6 +10,9 @@ set(tests common-chunk-internal-offset common-shard-index-for-chunk common-shard-internal-index + thread-pool-push-to-job-queue + s3-connection-put-object + s3-connection-upload-multipart-object ) foreach (name ${tests}) diff --git a/tests/unit-tests/s3-connection-put-object.cpp b/tests/unit-tests/s3-connection-put-object.cpp new file mode 100644 index 00000000..f519d020 --- /dev/null +++ b/tests/unit-tests/s3-connection-put-object.cpp @@ -0,0 +1,88 @@ +#include "s3.connection.hh" +#include "unit.test.macros.hh" + +#include + +namespace { +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +int +main() +{ + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping test."); + return 0; + } + + int retval = 1; + const std::string object_name = "test-object"; + + try { + zarr::S3Connection conn{ s3_endpoint, + s3_access_key_id, + s3_secret_access_key }; + + if (!conn.check_connection()) { + LOG_ERROR("Failed to connect to S3."); + return 1; + } + CHECK(conn.bucket_exists(bucket_name)); + CHECK(conn.delete_object(bucket_name, object_name)); + CHECK(!conn.object_exists(bucket_name, object_name)); + + std::vector data(1024, 0); + + std::string etag = + conn.put_object(bucket_name, + object_name, + std::span(data.data(), data.size())); + CHECK(!etag.empty()); + + CHECK(conn.object_exists(bucket_name, object_name)); + + // cleanup + CHECK(conn.delete_object(bucket_name, object_name)); + + retval = 0; + } catch (const std::exception& e) { + LOG_ERROR("Failed: %s", e.what()); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/s3-connection-upload-multipart-object.cpp b/tests/unit-tests/s3-connection-upload-multipart-object.cpp new file mode 100644 index 00000000..a5d467cb --- /dev/null +++ b/tests/unit-tests/s3-connection-upload-multipart-object.cpp @@ -0,0 +1,127 @@ +#include "s3.connection.hh" +#include "unit.test.macros.hh" + +#include + +namespace { +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +int +main() +{ + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping test."); + return 0; + } + + int retval = 1; + const std::string object_name = "test-object"; + + try { + zarr::S3Connection conn( + s3_endpoint, s3_access_key_id, s3_secret_access_key); + + if (!conn.check_connection()) { + LOG_ERROR("Failed to connect to S3."); + return 1; + } + CHECK(conn.bucket_exists(bucket_name)); + CHECK(conn.delete_object(bucket_name, object_name)); + CHECK(!conn.object_exists(bucket_name, object_name)); + + std::string upload_id = + conn.create_multipart_object(bucket_name, object_name); + CHECK(!upload_id.empty()); + + std::list parts; + + // parts need to be at least 5MiB, except the last part + std::vector data(5 << 20, 0); + for (auto i = 0; i < 4; ++i) { + std::string etag = conn.upload_multipart_object_part( + bucket_name, + object_name, + upload_id, + std::span(data.data(), data.size()), + i + 1); + CHECK(!etag.empty()); + + minio::s3::Part part; + part.number = i + 1; + part.etag = etag; + part.size = data.size(); + + parts.push_back(part); + } + + // last part is 1MiB + { + const unsigned int part_number = parts.size() + 1; + const size_t part_size = 1 << 20; // 1MiB + std::string etag = conn.upload_multipart_object_part( + bucket_name, + object_name, + upload_id, + std::span(data.data(), data.size()), + part_number); + CHECK(!etag.empty()); + + minio::s3::Part part; + part.number = part_number; + part.etag = etag; + part.size = part_size; + + parts.push_back(part); + } + + CHECK(conn.complete_multipart_object( + bucket_name, object_name, upload_id, parts)); + + CHECK(conn.object_exists(bucket_name, object_name)); + + // cleanup + CHECK(conn.delete_object(bucket_name, object_name)); + + retval = 0; + } catch (const std::exception& e) { + LOG_ERROR("Failed: %s", e.what()); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/thread-pool-push-to-job-queue.cpp b/tests/unit-tests/thread-pool-push-to-job-queue.cpp new file mode 100644 index 00000000..b778c4bd --- /dev/null +++ b/tests/unit-tests/thread-pool-push-to-job-queue.cpp @@ -0,0 +1,60 @@ +#include "thread.pool.hh" +#include "unit.test.macros.hh" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +int +main() +{ + int retval = 0; + + fs::path tmp_path = fs::temp_directory_path() / TEST; + CHECK(!fs::exists(tmp_path)); + + zarr::ThreadPool pool{ 1, [](const std::string&) {} }; + + CHECK(pool.push_to_job_queue([&tmp_path](std::string&) { + std::ofstream ofs(tmp_path); + ofs << "Hello, Acquire!"; + ofs.close(); + return true; + })); + pool.await_stop(); + + CHECK(fs::exists(tmp_path)); + + std::ifstream ifs(tmp_path); + CHECK(ifs.is_open()); + + std::string contents; + while (!ifs.eof()) { + std::getline(ifs, contents); + } + ifs.close(); + + if (contents != "Hello, Acquire!") { + fprintf(stderr, + "Expected 'Hello, Acquire!' but got '%s'\n", + contents.c_str()); + retval = 1; + } + + goto Cleanup; + +Finalize: + return retval; + +Cleanup: + std::error_code ec; + if (!fs::remove(tmp_path, ec)) { + fprintf(stderr, "Failed to remove file: %s\n", ec.message().c_str()); + retval = 1; + } + + goto Finalize; +} \ No newline at end of file From e9102dc74d1ffe1bc9892b84c0fb29eeeb4c1840 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 16:45:27 -0400 Subject: [PATCH 16/39] Implement and test Sink and SinkCreator types. --- src/streaming/CMakeLists.txt | 7 + src/streaming/file.sink.cpp | 26 + src/streaming/file.sink.hh | 20 + src/streaming/s3.sink.cpp | 182 +++++++ src/streaming/s3.sink.hh | 60 +++ src/streaming/sink.creator.cpp | 488 ++++++++++++++++++ src/streaming/sink.creator.hh | 183 +++++++ src/streaming/sink.hh | 24 + tests/unit-tests/CMakeLists.txt | 4 + tests/unit-tests/file-sink-write.cpp | 55 ++ tests/unit-tests/s3-sink-write.cpp | 124 +++++ .../sink-creator-make-data-sinks.cpp | 274 ++++++++++ .../sink-creator-make-metadata-sinks.cpp | 213 ++++++++ 13 files changed, 1660 insertions(+) create mode 100644 src/streaming/file.sink.cpp create mode 100644 src/streaming/file.sink.hh create mode 100644 src/streaming/s3.sink.cpp create mode 100644 src/streaming/s3.sink.hh create mode 100644 src/streaming/sink.creator.cpp create mode 100644 src/streaming/sink.creator.hh create mode 100644 src/streaming/sink.hh create mode 100644 tests/unit-tests/file-sink-write.cpp create mode 100644 tests/unit-tests/s3-sink-write.cpp create mode 100644 tests/unit-tests/sink-creator-make-data-sinks.cpp create mode 100644 tests/unit-tests/sink-creator-make-metadata-sinks.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index ba22b948..73f2b7cf 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -15,6 +15,13 @@ add_library(${tgt} thread.pool.cpp s3.connection.hh s3.connection.cpp + sink.hh + file.sink.hh + file.sink.cpp + s3.sink.hh + s3.sink.cpp + sink.creator.hh + sink.creator.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/file.sink.cpp b/src/streaming/file.sink.cpp new file mode 100644 index 00000000..8c665fa1 --- /dev/null +++ b/src/streaming/file.sink.cpp @@ -0,0 +1,26 @@ +#include "macros.hh" +#include "file.sink.hh" + +#include +#include + +namespace fs = std::filesystem; + +zarr::FileSink::FileSink(std::string_view filename) +: file_(filename.data(), std::ios::binary | std::ios::trunc) +{ +} + +bool +zarr::FileSink::write(size_t offset, const uint8_t* data, size_t bytes_of_buf) +{ + EXPECT(data, "Null pointer: data"); + if (bytes_of_buf == 0) { + return true; + } + + file_.seekp(offset); + file_.write(reinterpret_cast(data), bytes_of_buf); + file_.flush(); + return true; +} diff --git a/src/streaming/file.sink.hh b/src/streaming/file.sink.hh new file mode 100644 index 00000000..313419be --- /dev/null +++ b/src/streaming/file.sink.hh @@ -0,0 +1,20 @@ +#pragma once + +#include "sink.hh" + +#include +#include +#include + +namespace zarr { +class FileSink : public Sink +{ + public: + explicit FileSink(std::string_view filename); + + bool write(size_t offset, const uint8_t* data, size_t bytes_of_buf) override; + + private: + std::ofstream file_; +}; +} // namespace zarr diff --git a/src/streaming/s3.sink.cpp b/src/streaming/s3.sink.cpp new file mode 100644 index 00000000..dd2c78fb --- /dev/null +++ b/src/streaming/s3.sink.cpp @@ -0,0 +1,182 @@ +#include "macros.hh" +#include "s3.sink.hh" + +#include + +#ifdef min +#undef min +#endif + +zarr::S3Sink::S3Sink(std::string_view bucket_name, + std::string_view object_key, + std::shared_ptr connection_pool) + : bucket_name_{ bucket_name } + , object_key_{ object_key } + , connection_pool_{ connection_pool } +{ + EXPECT(!bucket_name_.empty(), "Bucket name must not be empty"); + EXPECT(!object_key_.empty(), "Object key must not be empty"); + EXPECT(connection_pool_, "Null pointer: connection_pool"); +} + +zarr::S3Sink::~S3Sink() +{ + if (!is_multipart_upload_() && nbytes_buffered_ > 0) { + if (!put_object_()) { + LOG_ERROR("Failed to upload object: %s", object_key_.c_str()); + } + } else if (is_multipart_upload_()) { + if (nbytes_buffered_ > 0 && !flush_part_()) { + LOG_ERROR("Failed to upload part %zu of object %s", + parts_.size() + 1, + object_key_.c_str()); + } + if (!finalize_multipart_upload_()) { + LOG_ERROR("Failed to finalize multipart upload of object %s", + object_key_.c_str()); + } + } +} + +bool +zarr::S3Sink::write(size_t offset, const uint8_t* data, size_t bytes_of_data) +{ + EXPECT(data, "Null pointer: data"); + if (bytes_of_data == 0) { + return true; + } + + if (offset < nbytes_flushed_) { + LOG_ERROR("Cannot write data at offset %zu, already flushed to %zu", + offset, + nbytes_flushed_); + return false; + } + nbytes_buffered_ = offset - nbytes_flushed_; + + while (bytes_of_data > 0) { + const auto bytes_to_write = + std::min(bytes_of_data, part_buffer_.size() - nbytes_buffered_); + + if (bytes_to_write) { + std::copy_n( + data, bytes_to_write, part_buffer_.begin() + nbytes_buffered_); + nbytes_buffered_ += bytes_to_write; + data += bytes_to_write; + bytes_of_data -= bytes_to_write; + } + + if (nbytes_buffered_ == part_buffer_.size() && !flush_part_()) { + return false; + } + } + + return true; +} + +bool +zarr::S3Sink::put_object_() +{ + if (nbytes_buffered_ == 0) { + return false; + } + + auto connection = connection_pool_->get_connection(); + + bool retval = false; + try { + std::string etag = + connection->put_object(bucket_name_, + object_key_, + { part_buffer_.data(), nbytes_buffered_ }); + EXPECT( + !etag.empty(), "Failed to upload object: %s", object_key_.c_str()); + + retval = true; + } catch (const std::exception& exc) { + LOG_ERROR("Error: %s", exc.what()); + } + + // cleanup + connection_pool_->return_connection(std::move(connection)); + nbytes_flushed_ = nbytes_buffered_; + nbytes_buffered_ = 0; + + return retval; +} + +bool +zarr::S3Sink::is_multipart_upload_() const +{ + return !upload_id_.empty() && !parts_.empty(); +} + +std::string +zarr::S3Sink::get_multipart_upload_id_() +{ + if (upload_id_.empty()) { + upload_id_ = + connection_pool_->get_connection()->create_multipart_object( + bucket_name_, object_key_); + } + + return upload_id_; +} + +bool +zarr::S3Sink::flush_part_() +{ + if (nbytes_buffered_ == 0) { + return false; + } + + auto connection = connection_pool_->get_connection(); + + bool retval = false; + try { + minio::s3::Part part; + part.number = static_cast(parts_.size()) + 1; + + std::span data(part_buffer_.data(), nbytes_buffered_); + part.etag = + connection->upload_multipart_object_part(bucket_name_, + object_key_, + get_multipart_upload_id_(), + data, + part.number); + EXPECT(!part.etag.empty(), + "Failed to upload part %u of object %s", + part.number, + object_key_.c_str()); + + parts_.push_back(part); + + retval = true; + } catch (const std::exception& exc) { + LOG_ERROR("Error: %s", exc.what()); + } + + // cleanup + connection_pool_->return_connection(std::move(connection)); + nbytes_flushed_ += nbytes_buffered_; + nbytes_buffered_ = 0; + + return retval; +} + +bool +zarr::S3Sink::finalize_multipart_upload_() +{ + if (!is_multipart_upload_()) { + return false; + } + + auto connection = connection_pool_->get_connection(); + + bool retval = connection->complete_multipart_object( + bucket_name_, object_key_, upload_id_, parts_); + + connection_pool_->return_connection(std::move(connection)); + + return retval; +} diff --git a/src/streaming/s3.sink.hh b/src/streaming/s3.sink.hh new file mode 100644 index 00000000..a5524260 --- /dev/null +++ b/src/streaming/s3.sink.hh @@ -0,0 +1,60 @@ +#pragma once + +#include "sink.hh" +#include "s3.connection.hh" + +#include + +#include +#include + +namespace zarr { +class S3Sink final : public Sink +{ + public: + S3Sink(std::string_view bucket_name, + std::string_view object_key, + std::shared_ptr connection_pool); + ~S3Sink() override; + + bool write(size_t offset, + const uint8_t* data, + size_t bytes_of_data) override; + + private: + std::string bucket_name_; + std::string object_key_; + + std::shared_ptr connection_pool_; + + // multipart upload + std::array part_buffer_; + size_t nbytes_buffered_ = 0; + size_t nbytes_flushed_ = 0; + + std::string upload_id_; + std::list parts_; + + // single-part upload + /// @brief Upload the object to S3. + /// @returns True if the object was successfully uploaded, otherwise false. + [[nodiscard]] bool put_object_(); + + // multipart upload + bool is_multipart_upload_() const; + + /// @brief Get the multipart upload ID, if it exists. Otherwise, create a + /// new multipart upload. + /// @returns The multipart upload ID. + std::string get_multipart_upload_id_(); + + /// @brief Flush the current part to S3. + /// @returns True if the part was successfully flushed, otherwise false. + [[nodiscard]] bool flush_part_(); + + /// @brief Finalize the multipart upload. + /// @returns True if a multipart upload was successfully finalized, + /// otherwise false. + [[nodiscard]] bool finalize_multipart_upload_(); +}; +} // namespace zarr diff --git a/src/streaming/sink.creator.cpp b/src/streaming/sink.creator.cpp new file mode 100644 index 00000000..bd6217f0 --- /dev/null +++ b/src/streaming/sink.creator.cpp @@ -0,0 +1,488 @@ +#include "macros.hh" +#include "sink.creator.hh" +#include "file.sink.hh" +#include "s3.sink.hh" +#include "acquire.zarr.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +zarr::SinkCreator::SinkCreator( + std::shared_ptr thread_pool_, + std::shared_ptr connection_pool) + : thread_pool_{ thread_pool_ } + , connection_pool_{ connection_pool } +{ +} + +std::unique_ptr +zarr::SinkCreator::make_sink(std::string_view file_path) +{ + if (file_path.starts_with("file://")) { + file_path = file_path.substr(7); + } + + EXPECT(!file_path.empty(), "File path must not be empty."); + + fs::path path(file_path); + EXPECT(!path.empty(), "Invalid file path: %s", file_path.data()); + + fs::path parent_path = path.parent_path(); + + if (!fs::is_directory(parent_path)) { + std::error_code ec; + if (!fs::create_directories(parent_path, ec)) { + LOG_ERROR("Failed to create directory '%s': %s", + parent_path.c_str(), + ec.message().c_str()); + return nullptr; + } + } + + return std::make_unique(file_path); +} + +std::unique_ptr +zarr::SinkCreator::make_sink(std::string_view bucket_name, + std::string_view object_key) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!object_key.empty(), "Object key must not be empty."); + EXPECT(connection_pool_, "S3 connection pool not provided."); + if (!bucket_exists_(bucket_name)) { + LOG_ERROR("Bucket '%s' does not exist.", bucket_name.data()); + return nullptr; + } + + return std::make_unique(bucket_name, object_key, connection_pool_); +} + +bool +zarr::SinkCreator::make_data_sinks( + std::string_view base_path, + const std::vector& dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks) +{ + if (base_path.starts_with("file://")) { + base_path = base_path.substr(7); + } + + EXPECT(!base_path.empty(), "Base path must not be empty."); + + std::queue paths; + try { + paths = make_data_sink_paths_( + base_path, dimensions, parts_along_dimension, true); + } catch (const std::exception& exc) { + LOG_ERROR("Failed to create dataset paths: %s", exc.what()); + return false; + } + + return make_files_(paths, part_sinks); +} + +bool +zarr::SinkCreator::make_data_sinks( + std::string_view bucket_name, + std::string_view base_path, + const std::vector& dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks) +{ + EXPECT(!base_path.empty(), "Base path must not be empty."); + + auto paths = make_data_sink_paths_( + base_path, dimensions, parts_along_dimension, false); + + return make_s3_objects_(bucket_name, paths, part_sinks); +} + +bool +zarr::SinkCreator::make_metadata_sinks( + size_t version, + std::string_view base_path, + std::unordered_map>& metadata_sinks) +{ + if (base_path.starts_with("file://")) { + base_path = base_path.substr(7); + } + EXPECT(!base_path.empty(), "Base path must not be empty."); + + std::vector file_paths = + make_metadata_sink_paths_(version, base_path, true); + + return make_files_(base_path.data(), file_paths, metadata_sinks); +} + +bool +zarr::SinkCreator::make_metadata_sinks( + size_t version, + std::string_view bucket_name, + std::string_view base_path, + std::unordered_map>& metadata_sinks) +{ + EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); + EXPECT(!base_path.empty(), "Base path must not be empty."); + if (!bucket_exists_(bucket_name)) { + LOG_ERROR("Bucket '%s' does not exist.", bucket_name.data()); + return false; + } + + auto file_paths = make_metadata_sink_paths_(version, base_path, false); + return make_s3_objects_(bucket_name, base_path, file_paths, metadata_sinks); +} + +std::queue +zarr::SinkCreator::make_data_sink_paths_( + std::string_view base_path, + const std::vector& dimensions, + const std::function& parts_along_dimension, + bool create_directories) +{ + std::queue paths; + paths.emplace(base_path); + + if (create_directories) { + EXPECT( + make_dirs_(paths), "Failed to create directory '%s'.", base_path); + } + + // create intermediate paths + for (auto i = 1; // skip the last dimension + i < dimensions.size() - 1; // skip the x dimension + ++i) { + const auto& dim = dimensions.at(i); + const auto n_parts = parts_along_dimension(dim); + CHECK(n_parts); + + auto n_paths = paths.size(); + for (auto j = 0; j < n_paths; ++j) { + const auto path = paths.front(); + paths.pop(); + + for (auto k = 0; k < n_parts; ++k) { + const auto kstr = std::to_string(k); + paths.push(path + (path.empty() ? kstr : "/" + kstr)); + } + } + + if (create_directories) { + EXPECT(make_dirs_(paths), + "Failed to create directories for dimension '%s'.", + dim.name.c_str()); + } + } + + // create final paths + { + const auto& dim = dimensions.back(); + const auto n_parts = parts_along_dimension(dim); + CHECK(n_parts); + + auto n_paths = paths.size(); + for (auto i = 0; i < n_paths; ++i) { + const auto path = paths.front(); + paths.pop(); + for (auto j = 0; j < n_parts; ++j) + paths.push(path + "/" + std::to_string(j)); + } + } + + return paths; +} + +std::vector +zarr::SinkCreator::make_metadata_sink_paths_(size_t version, + std::string_view base_path, + bool create_directories) +{ + std::vector paths; + + switch (version) { + case ZarrVersion_2: + paths.emplace_back(".zattrs"); + paths.emplace_back(".zgroup"); + paths.emplace_back("0/.zattrs"); + paths.emplace_back("acquire.json"); + break; + case ZarrVersion_3: + paths.emplace_back("zarr.json"); + paths.emplace_back("meta/root.group.json"); + paths.emplace_back("meta/acquire.json"); + break; + default: + throw std::runtime_error("Invalid Zarr version " + + std::to_string(static_cast(version))); + } + + if (create_directories) { + std::queue dir_paths; + dir_paths.emplace(base_path); + EXPECT(make_dirs_(dir_paths), + "Failed to create metadata directories."); + dir_paths.pop(); // remove the base path + + std::unordered_set parent_paths; + for (const auto& path : paths) { + fs::path parent = fs::path(path).parent_path(); + if (!parent.empty()) { + parent_paths.emplace((fs::path(base_path) / parent).string()); + } + } + + for (const auto& dir_path : parent_paths) { + dir_paths.push(dir_path); + } + + if (!dir_paths.empty()) { + EXPECT(make_dirs_(dir_paths), + "Failed to create metadata directories."); + } + } + + return paths; +} + +bool +zarr::SinkCreator::make_dirs_(std::queue& dir_paths) +{ + if (dir_paths.empty()) { + return true; + } + + std::atomic all_successful = 1; + + const auto n_dirs = dir_paths.size(); + std::latch latch(n_dirs); + + for (auto i = 0; i < n_dirs; ++i) { + const auto dirname = dir_paths.front(); + dir_paths.pop(); + + EXPECT(thread_pool_->push_to_job_queue( + [dirname, &latch, &all_successful](std::string& err) -> bool { + if (dirname.empty()) { + err = "Directory name must not be empty."; + latch.count_down(); + all_successful.fetch_and(0); + return false; + } + + if (fs::is_directory(dirname)) { + latch.count_down(); + return true; + } else if (fs::exists(dirname)) { + err = + "'" + dirname + "' exists but is not a directory"; + latch.count_down(); + all_successful.fetch_and(0); + return false; + } + + if (all_successful) { + std::error_code ec; + if (!fs::create_directories(dirname, ec)) { + err = "Failed to create directory '" + dirname + + "': " + ec.message(); + latch.count_down(); + all_successful.fetch_and(0); + return false; + } + } + + latch.count_down(); + return true; + }), + "Failed to push job to thread pool."); + + dir_paths.push(dirname); + } + + latch.wait(); + + return (bool)all_successful; +} + +bool +zarr::SinkCreator::make_files_(std::queue& file_paths, + std::vector>& sinks) +{ + if (file_paths.empty()) { + return true; + } + + std::atomic all_successful = 1; + + const auto n_files = file_paths.size(); + sinks.resize(n_files); + std::fill(sinks.begin(), sinks.end(), nullptr); + std::latch latch(n_files); + + for (auto i = 0; i < n_files; ++i) { + const auto filename = file_paths.front(); + file_paths.pop(); + + std::unique_ptr* psink = sinks.data() + i; + + EXPECT(thread_pool_->push_to_job_queue( + [filename, psink, &latch, &all_successful]( + std::string& err) -> bool { + bool success = false; + + try { + if (all_successful) { + *psink = std::make_unique(filename); + } + success = true; + } catch (const std::exception& exc) { + err = "Failed to create file '" + filename + + "': " + exc.what(); + } catch (...) { + err = "Failed to create file '" + filename + + "': (unknown)."; + } + + latch.count_down(); + all_successful.fetch_and((char)success); + + return success; + }), + "Failed to push job to thread pool."); + } + + latch.wait(); + + return (bool)all_successful; +} + +bool +zarr::SinkCreator::make_files_( + const std::string& base_dir, + const std::vector& file_paths, + std::unordered_map>& sinks) +{ + if (file_paths.empty()) { + return true; + } + + std::atomic all_successful = 1; + + const auto n_files = file_paths.size(); + std::latch latch(n_files); + + sinks.clear(); + for (const auto& filename : file_paths) { + sinks[filename] = nullptr; + std::unique_ptr* psink = &sinks[filename]; + + const std::string prefix = base_dir.empty() ? "" : base_dir + "/"; + const auto file_path = prefix + filename; + + EXPECT(thread_pool_->push_to_job_queue( + [filename = file_path, psink, &latch, &all_successful]( + std::string& err) -> bool { + bool success = false; + + try { + if (all_successful) { + *psink = std::make_unique(filename); + } + success = true; + } catch (const std::exception& exc) { + err = "Failed to create file '" + filename + + "': " + exc.what(); + } catch (...) { + err = "Failed to create file '" + filename + + "': (unknown)."; + } + + latch.count_down(); + all_successful.fetch_and((char)success); + + return success; + }), + "Failed to push job to thread pool."); + } + + latch.wait(); + + return (bool)all_successful; +} + +bool +zarr::SinkCreator::bucket_exists_(std::string_view bucket_name) +{ + CHECK(!bucket_name.empty()); + EXPECT(connection_pool_, "S3 connection pool not provided."); + + auto conn = connection_pool_->get_connection(); + bool bucket_exists = conn->bucket_exists(bucket_name); + + connection_pool_->return_connection(std::move(conn)); + + return bucket_exists; +} + +bool +zarr::SinkCreator::make_s3_objects_(std::string_view bucket_name, + std::queue& object_keys, + std::vector>& sinks) +{ + if (object_keys.empty()) { + return true; + } + + if (bucket_name.empty()) { + LOG_ERROR("Bucket name not provided."); + return false; + } + if (!connection_pool_) { + LOG_ERROR("S3 connection pool not provided."); + return false; + } + + const auto n_objects = object_keys.size(); + sinks.resize(n_objects); + for (auto i = 0; i < n_objects; ++i) { + sinks[i] = std::make_unique( + bucket_name, object_keys.front(), connection_pool_); + object_keys.pop(); + } + + return true; +} + +bool +zarr::SinkCreator::make_s3_objects_( + std::string_view bucket_name, + std::string_view base_path, + std::vector& object_keys, + std::unordered_map>& sinks) +{ + if (object_keys.empty()) { + return true; + } + + if (bucket_name.empty()) { + LOG_ERROR("Bucket name not provided."); + return false; + } + + if (!connection_pool_) { + LOG_ERROR("S3 connection pool not provided."); + return false; + } + + sinks.clear(); + for (const auto& key : object_keys) { + sinks[key] = std::make_unique( + bucket_name, std::string(base_path) + "/" + key, connection_pool_); + } + + return true; +} diff --git a/src/streaming/sink.creator.hh b/src/streaming/sink.creator.hh new file mode 100644 index 00000000..050fc179 --- /dev/null +++ b/src/streaming/sink.creator.hh @@ -0,0 +1,183 @@ +#pragma once + +#include "stream.settings.hh" // ZarrDimension_s +#include "sink.hh" +#include "thread.pool.hh" +#include "s3.connection.hh" + +#include +#include +#include + +namespace zarr { +using Dimension = ZarrDimension_s; + +class SinkCreator final +{ + public: + SinkCreator(std::shared_ptr thread_pool_, + std::shared_ptr connection_pool); + ~SinkCreator() noexcept = default; + + /** + * @brief Create a sink from a file path. + * @param file_path The path to the file. + * @return Pointer to the sink created, or nullptr if the file cannot be + * opened. + * @throws std::runtime_error if the file path is not valid. + */ + static std::unique_ptr make_sink(std::string_view file_path); + + /** + * @brief Create a sink from an S3 bucket name and object key. + * @param bucket_name The name of the bucket in which the object is stored. + * @param object_key The key of the object to write to. + * @return Pointer to the sink created, or nullptr if the bucket does not + * exist. + * @throws std::runtime_error if the bucket name or object key is not valid, + * or if there is no connection pool. + */ + std::unique_ptr make_sink(std::string_view bucket_name, + std::string_view object_key); + + /** + * @brief Create a collection of file sinks for a Zarr dataset. + * @param[in] base_path The path to the base directory for the dataset. + * @param[in] dimensions The dimensions of the dataset. + * @param[in] parts_along_dimension Function to determine the number of + * parts (i.e., shards or chunks) along a dimension. + * @param[out] part_sinks The sinks created. + * @return True iff all file sinks were created successfully. + * @throws std::runtime_error if @p base_path is not valid, or if the number + * of parts along a dimension is zero. + */ + [[nodiscard]] bool make_data_sinks( + std::string_view base_path, + const std::vector& dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks); + + /** + * @brief Create a collection of S3 sinks for a Zarr dataset. + * @param[in] bucket_name The name of the bucket in which the dataset is + * stored. + * @param[in] base_path The path to the base directory for the dataset. + * @param[in] dimensions The dimensions of the dataset. + * @param[in] parts_along_dimension Function to determine the number of + * parts (i.e., shards or chunks) along a dimension. + * @param[out] part_sinks The sinks created. + * @return True iff all file sinks were created successfully. + */ + [[nodiscard]] bool make_data_sinks( + std::string_view bucket_name, + std::string_view base_path, + const std::vector& dimensions, + const std::function& parts_along_dimension, + std::vector>& part_sinks); + + /** + * @brief Create a collection of metadata sinks for a Zarr dataset. + * @param[in] version The Zarr version. + * @param[in] base_path The base URI for the dataset. + * @param[out] metadata_sinks The sinks created, keyed by path. + * @return True iff all metadata sinks were created successfully. + * @throws std::runtime_error if @p base_uri is not valid, or if, for S3 + * sinks, the bucket does not exist. + */ + [[nodiscard]] bool make_metadata_sinks( + size_t version, + std::string_view base_path, + std::unordered_map>& metadata_sinks); + + /** + * @brief + * @param version + * @param bucket_name + * @param base_path + * @param metadata_sinks + * @return + * @throws std::runtime_error if @p version is invalid, if @p bucket_name is + * empty or does not exist, or if @p base_path is empty. + */ + [[nodiscard]] bool make_metadata_sinks( + size_t version, + std::string_view bucket_name, + std::string_view base_path, + std::unordered_map>& metadata_sinks); + + private: + std::shared_ptr thread_pool_; + std::shared_ptr connection_pool_; // could be null + + /** + * @brief Construct the paths for a Zarr dataset. + * @param base_path The base path for the dataset. + * @param dimensions The dimensions of the dataset. + * @param parts_along_dimension Function to determine the number of parts + * @param create_directories Whether to create intermediate directories. + * @return A queue of paths to the dataset components. + * @throws std::runtime_error if intermediate directories cannot be created, + * or if the number of parts along a dimension is zero. + */ + std::queue make_data_sink_paths_( + std::string_view base_path, + const std::vector& dimensions, + const std::function& parts_along_dimension, + bool create_directories); + + std::vector make_metadata_sink_paths_( + size_t version, + std::string_view base_path, + bool create_directories); + + /// @brief Parallel create a collection of directories. + /// @param[in] dir_paths The directories to create. + /// @return True iff all directories were created successfully. + [[nodiscard]] bool make_dirs_(std::queue& dir_paths); + + /// @brief Parallel create a collection of files. + /// @param[in,out] file_paths The files to create. Unlike `make_dirs_`, + /// this function drains the queue. + /// @param[out] files The files created. + /// @return True iff all files were created successfully. + [[nodiscard]] bool make_files_(std::queue& file_paths, + std::vector>& sinks); + + /// @brief Parallel create a collection of files, keyed by path. + /// @param[in] base_dir The base directory for the files. + /// @param[in] file_paths Paths to the files to create, relative to @p + /// base_dir. + /// @param[out] sinks The sinks created, keyed by path. + /// @return True iff all files were created successfully. + [[nodiscard]] bool make_files_( + const std::string& base_dir, + const std::vector& file_paths, + std::unordered_map>& sinks); + + /// @brief Check whether an S3 bucket exists. + /// @param[in] bucket_name The name of the bucket to check. + /// @return True iff the bucket exists. + bool bucket_exists_(std::string_view bucket_name); + + /// @brief Create a collection of S3 objects. + /// @param[in] bucket_name The name of the bucket. + /// @param[in,out] object_keys The keys of the objects to create. + /// @param[out] sinks The sinks created. + /// @return True iff all S3 objects were created successfully. + [[nodiscard]] bool make_s3_objects_( + std::string_view bucket_name, + std::queue& object_keys, + std::vector>& sinks); + + /// @brief Create a collection of S3 objects, keyed by object key. + /// @param[in] bucket_name The name of the bucket. + /// @param[in] object_keys The keys of the objects to create. + /// @param[out] sinks The sinks created, keyed by object key. + /// @return True iff all S3 objects were created successfully. + [[nodiscard]] bool make_s3_objects_( + std::string_view bucket_name, + std::string_view base_path, + std::vector& object_keys, + std::unordered_map>& sinks); +}; +} // namespace zarr diff --git a/src/streaming/sink.hh b/src/streaming/sink.hh new file mode 100644 index 00000000..0ec60ef5 --- /dev/null +++ b/src/streaming/sink.hh @@ -0,0 +1,24 @@ +#pragma once + +#include // uint8_t +#include // size_t + +namespace zarr { +class Sink +{ + public: + virtual ~Sink() = default; + + /** + * @brief Write data to the sink. + * @param offset The offset in the sink to write to. + * @param buf The buffer to write to the sink. + * @param bytes_of_buf The number of bytes to write from @p buf. + * @return True if the write was successful, false otherwise. + * @throws std::runtime_error if @p buf is nullptr or the write fails. + */ + [[nodiscard]] virtual bool write(size_t offset, + const uint8_t* buf, + size_t bytes_of_buf) = 0; +}; +} // namespace zarr diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index c87823ac..f5a7c5a7 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -13,6 +13,10 @@ set(tests thread-pool-push-to-job-queue s3-connection-put-object s3-connection-upload-multipart-object + file-sink-write + s3-sink-write + sink-creator-make-metadata-sinks + sink-creator-make-data-sinks ) foreach (name ${tests}) diff --git a/tests/unit-tests/file-sink-write.cpp b/tests/unit-tests/file-sink-write.cpp new file mode 100644 index 00000000..49398981 --- /dev/null +++ b/tests/unit-tests/file-sink-write.cpp @@ -0,0 +1,55 @@ +#include "file.sink.hh" +#include "unit.test.macros.hh" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +int +main() +{ + int retval = 0; + fs::path tmp_path = fs::temp_directory_path() / TEST; + + CHECK(!fs::exists(tmp_path)); + { + const uint8_t str[] = "Hello, Acquire!"; + zarr::FileSink sink(tmp_path.string()); + CHECK(sink.write(0, str, sizeof(str) - 1)); + } + + // The file tmp_path should now contain the string "Hello, world!\n". + CHECK(fs::exists(tmp_path)); + + std::ifstream ifs(tmp_path); + CHECK(ifs.is_open()); + + std::string contents; + while (!ifs.eof()) { + std::getline(ifs, contents); + } + ifs.close(); + + if (contents != "Hello, Acquire!") { + fprintf(stderr, + "Expected 'Hello, Acquire!' but got '%s'\n", + contents.c_str()); + retval = 1; + } + goto Cleanup; + +Finalize: + return retval; + +Cleanup: + std::error_code ec; + if (!fs::remove(tmp_path, ec)) { + fprintf(stderr, "Failed to remove file: %s\n", ec.message().c_str()); + retval = 1; + } + + goto Finalize; +} \ No newline at end of file diff --git a/tests/unit-tests/s3-sink-write.cpp b/tests/unit-tests/s3-sink-write.cpp new file mode 100644 index 00000000..8d80e9da --- /dev/null +++ b/tests/unit-tests/s3-sink-write.cpp @@ -0,0 +1,124 @@ +#include "s3.sink.hh" +#include "unit.test.macros.hh" + +#include +#include + +namespace { +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +int +main() +{ + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping test."); + return 0; + } + + int retval = 1; + const std::string object_name = "test-object"; + + try { + auto pool = std::make_shared( + 1, s3_endpoint, s3_access_key_id, s3_secret_access_key); + + auto conn = pool->get_connection(); + if (!conn->check_connection()) { + LOG_ERROR("Failed to connect to S3."); + return 1; + } + CHECK(conn->bucket_exists(bucket_name)); + CHECK(conn->delete_object(bucket_name, object_name)); + CHECK(!conn->object_exists(bucket_name, object_name)); + + pool->return_connection(std::move(conn)); + + { + const uint8_t str[] = "Hello, Acquire!"; + auto sink = + std::make_unique(bucket_name, object_name, pool); + CHECK(sink->write(0, str, sizeof(str) - 1)); + } + + conn = pool->get_connection(); + CHECK(conn->object_exists(bucket_name, object_name)); + pool->return_connection(std::move(conn)); + + // Verify the object contents. + { + minio::s3::BaseUrl url(s3_endpoint); + url.https = s3_endpoint.starts_with("https://"); + + minio::creds::StaticProvider provider(s3_access_key_id, + s3_secret_access_key); + + minio::s3::Client client(url, &provider); + minio::s3::GetObjectArgs args; + args.bucket = bucket_name; + args.object = object_name; + + std::string contents; + args.datafunc = + [&contents](minio::http::DataFunctionArgs args) -> bool { + contents = args.datachunk; + return true; + }; + + // Call get object. + minio::s3::GetObjectResponse resp = client.GetObject(args); + + if (contents != "Hello, Acquire!") { + LOG_ERROR("Expected 'Hello, Acquire!' but got '%s'", + contents.c_str()); + return 1; + } + } + + // cleanup + conn = pool->get_connection(); + CHECK(conn->delete_object(bucket_name, object_name)); + + retval = 0; + } catch (const std::exception& e) { + LOG_ERROR("Exception: %s", e.what()); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/sink-creator-make-data-sinks.cpp b/tests/unit-tests/sink-creator-make-data-sinks.cpp new file mode 100644 index 00000000..f01f5255 --- /dev/null +++ b/tests/unit-tests/sink-creator-make-data-sinks.cpp @@ -0,0 +1,274 @@ +#include "sink.creator.hh" +#include "s3.connection.hh" +#include "zarr.common.hh" +#include "acquire.zarr.h" +#include "unit.test.macros.hh" + +#include +#include + +namespace fs = std::filesystem; + +namespace { +const std::string test_dir = TEST "-data"; + +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +void +sink_creator_make_chunk_sinks(std::shared_ptr thread_pool, + const std::vector& dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + // create the sinks, then let them go out of scope to close the handles + { + std::vector> sinks; + CHECK(sink_creator.make_data_sinks( + test_dir, dimensions, zarr::chunks_along_dimension, sinks)); + } + + const auto chunks_in_y = zarr::chunks_along_dimension(dimensions[1]); + const auto chunks_in_x = zarr::chunks_along_dimension(dimensions[2]); + + const fs::path base_path(test_dir); + for (auto i = 0; i < chunks_in_y; ++i) { + const fs::path y_dir = base_path / std::to_string(i); + + for (auto j = 0; j < chunks_in_x; ++j) { + const fs::path x_file = y_dir / std::to_string(j); + CHECK(fs::is_regular_file(x_file)); + + // cleanup + fs::remove(x_file); + } + CHECK(!fs::is_regular_file(y_dir / std::to_string(chunks_in_x))); + fs::remove(y_dir); + } + CHECK(!fs::is_directory(base_path / std::to_string(chunks_in_y))); +} + +void +sink_creator_make_chunk_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name, + const std::vector& dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + // create the sinks, then let them go out of scope to close the handles + { + const uint8_t data[] = { 0, 0 }; + std::vector> sinks; + CHECK(sink_creator.make_data_sinks(bucket_name, + test_dir, + dimensions, + zarr::chunks_along_dimension, + sinks)); + + for (auto& sink : sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data, 2)); + } + } + + const auto chunks_in_y = zarr::chunks_along_dimension(dimensions[1]); + const auto chunks_in_x = zarr::chunks_along_dimension(dimensions[2]); + + auto conn = connection_pool->get_connection(); + + const std::string base_path(test_dir); + for (auto i = 0; i < chunks_in_y; ++i) { + const std::string y_dir = base_path + "/" + std::to_string(i); + + for (auto j = 0; j < chunks_in_x; ++j) { + const std::string x_file = y_dir + "/" + std::to_string(j); + CHECK(conn->object_exists(bucket_name, x_file)); + + // cleanup + CHECK(conn->delete_object(bucket_name, x_file)); + } + CHECK(!conn->object_exists(bucket_name, + y_dir + "/" + std::to_string(chunks_in_x))); + CHECK(conn->delete_object(bucket_name, y_dir)); + } + CHECK(!conn->object_exists(bucket_name, + base_path + "/" + std::to_string(chunks_in_y))); + CHECK(conn->delete_object(bucket_name, base_path)); +} + +void +sink_creator_make_shard_sinks(std::shared_ptr thread_pool, + const std::vector& dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + // create the sinks, then let them go out of scope to close the handles + { + std::vector> sinks; + CHECK(sink_creator.make_data_sinks( + test_dir, dimensions, zarr::shards_along_dimension, sinks)); + } + + const auto shards_in_y = zarr::shards_along_dimension(dimensions[1]); + const auto shards_in_x = zarr::shards_along_dimension(dimensions[2]); + + const fs::path base_path(test_dir); + for (auto i = 0; i < shards_in_y; ++i) { + const fs::path y_dir = base_path / std::to_string(i); + + for (auto j = 0; j < shards_in_x; ++j) { + const fs::path x_file = y_dir / std::to_string(j); + CHECK(fs::is_regular_file(x_file)); + + // cleanup + fs::remove(x_file); + } + CHECK(!fs::is_regular_file(y_dir / std::to_string(shards_in_x))); + fs::remove(y_dir); + } + CHECK(!fs::is_directory(base_path / std::to_string(shards_in_y))); +} + +void +sink_creator_make_shard_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name, + const std::vector& dimensions) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + // create the sinks, then let them go out of scope to close the handles + { + const uint8_t data[] = { 0, 0 }; + std::vector> sinks; + CHECK(sink_creator.make_data_sinks(bucket_name, + test_dir, + dimensions, + zarr::shards_along_dimension, + sinks)); + + for (auto& sink : sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data, 2)); + } + } + + const auto shards_in_y = zarr::shards_along_dimension(dimensions[1]); + const auto shards_in_x = zarr::shards_along_dimension(dimensions[2]); + + auto conn = connection_pool->get_connection(); + + const std::string base_path(test_dir); + for (auto i = 0; i < shards_in_y; ++i) { + const std::string y_dir = base_path + "/" + std::to_string(i); + + for (auto j = 0; j < shards_in_x; ++j) { + const std::string x_file = y_dir + "/" + std::to_string(j); + CHECK(conn->object_exists(bucket_name, x_file)); + + // cleanup + CHECK(conn->delete_object(bucket_name, x_file)); + } + CHECK(!conn->object_exists(bucket_name, + y_dir + "/" + std::to_string(shards_in_x))); + CHECK(conn->delete_object(bucket_name, y_dir)); + } + CHECK(!conn->object_exists(bucket_name, + base_path + "/" + std::to_string(shards_in_y))); + CHECK(conn->delete_object(bucket_name, base_path)); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + std::vector dims; + dims.emplace_back("z", + ZarrDimensionType_Space, + 0, + 3, // 3 planes per chunk + 1); // 1 chunk per shard (3 planes per shard) + dims.emplace_back("y", + ZarrDimensionType_Space, + 4, + 2, // 2 rows per chunk, 2 chunks + 2); // 2 chunks per shard (4 rows per shard, 1 shard) + dims.emplace_back("x", + ZarrDimensionType_Space, + 12, + 3, // 3 columns per chunk, 4 chunks + 2); // 2 chunks per shard (6 columns per shard, 2 shards) + + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Failed: %s", err.c_str()); }); + + try { + sink_creator_make_chunk_sinks(thread_pool, dims); + sink_creator_make_shard_sinks(thread_pool, dims); + } catch (const std::exception& e) { + LOG_ERROR("Failed: %s", e.what()); + return 1; + } + + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping S3 portion of test."); + return 0; + } + + auto connection_pool = std::make_shared( + 4, s3_endpoint, s3_access_key_id, s3_secret_access_key); + + try { + sink_creator_make_chunk_sinks( + thread_pool, connection_pool, bucket_name, dims); + sink_creator_make_shard_sinks( + thread_pool, connection_pool, bucket_name, dims); + } catch (const std::exception& e) { + LOG_ERROR("Failed: %s", e.what()); + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/tests/unit-tests/sink-creator-make-metadata-sinks.cpp b/tests/unit-tests/sink-creator-make-metadata-sinks.cpp new file mode 100644 index 00000000..e46ebd61 --- /dev/null +++ b/tests/unit-tests/sink-creator-make-metadata-sinks.cpp @@ -0,0 +1,213 @@ +#include "sink.creator.hh" +#include "s3.connection.hh" +#include "unit.test.macros.hh" + +#include +#include + +namespace fs = std::filesystem; + +namespace { +const std::string test_dir = TEST "-data"; + +bool +get_credentials(std::string& endpoint, + std::string& bucket_name, + std::string& access_key_id, + std::string& secret_access_key) +{ + char* env = nullptr; + if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { + LOG_ERROR("ZARR_S3_ENDPOINT not set."); + return false; + } + endpoint = env; + + if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { + LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); + return false; + } + bucket_name = env; + + if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { + LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); + return false; + } + access_key_id = env; + + if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { + LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); + return false; + } + secret_access_key = env; + + return true; +} +} // namespace + +void +sink_creator_make_v2_metadata_sinks( + std::shared_ptr thread_pool) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + std::unordered_map> metadata_sinks; + CHECK(sink_creator.make_metadata_sinks(2, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 4); + CHECK(metadata_sinks.contains(".zattrs")); + CHECK(metadata_sinks.contains(".zgroup")); + CHECK(metadata_sinks.contains("0/.zattrs")); + CHECK(metadata_sinks.contains("acquire.json")); + + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + sink.reset(nullptr); // close the file + + fs::path file_path(test_dir + "/" + key); + CHECK(fs::is_regular_file(file_path)); + // cleanup + fs::remove(file_path); + } + + fs::remove(test_dir + "/0"); +} + +void +sink_creator_make_v2_metadata_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + std::unordered_map> metadata_sinks; + CHECK( + sink_creator.make_metadata_sinks(2, bucket_name, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 4); + CHECK(metadata_sinks.contains(".zattrs")); + CHECK(metadata_sinks.contains(".zgroup")); + CHECK(metadata_sinks.contains("0/.zattrs")); + CHECK(metadata_sinks.contains("acquire.json")); + + auto conn = connection_pool->get_connection(); + + const uint8_t data[] = { 0, 0 }; + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data, 2)); + sink.reset(nullptr); // close the connection + + std::string path = test_dir + "/" + key; + CHECK(conn->object_exists(bucket_name, path)); + // cleanup + CHECK(conn->delete_object(bucket_name, path)); + } + + CHECK(conn->delete_object(bucket_name, "0")); +} + +void +sink_creator_make_v3_metadata_sinks( + std::shared_ptr thread_pool) +{ + zarr::SinkCreator sink_creator(thread_pool, nullptr); + + std::unordered_map> metadata_sinks; + CHECK(sink_creator.make_metadata_sinks(3, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 3); + CHECK(metadata_sinks.contains("zarr.json")); + CHECK(metadata_sinks.contains("meta/root.group.json")); + CHECK(metadata_sinks.contains("meta/acquire.json")); + + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + sink.reset(nullptr); // close the file + + fs::path file_path(test_dir + "/" + key); + CHECK(fs::is_regular_file(file_path)); + // cleanup + fs::remove(file_path); + } + + fs::remove(test_dir + "/meta"); +} + +void +sink_creator_make_v3_metadata_sinks( + std::shared_ptr thread_pool, + std::shared_ptr connection_pool, + const std::string& bucket_name) +{ + zarr::SinkCreator sink_creator(thread_pool, connection_pool); + + std::unordered_map> metadata_sinks; + CHECK( + sink_creator.make_metadata_sinks(3, bucket_name, test_dir, metadata_sinks)); + + CHECK(metadata_sinks.size() == 3); + CHECK(metadata_sinks.contains("zarr.json")); + CHECK(metadata_sinks.contains("meta/root.group.json")); + CHECK(metadata_sinks.contains("meta/acquire.json")); + + auto conn = connection_pool->get_connection(); + + const uint8_t data[] = { 0, 0 }; + for (auto& [key, sink] : metadata_sinks) { + CHECK(sink); + // we need to write some data to the sink to ensure it is created + CHECK(sink->write(0, data, 2)); + sink.reset(nullptr); // close the connection + + std::string path = test_dir + "/" + key; + CHECK(conn->object_exists(bucket_name, path)); + // cleanup + CHECK(conn->delete_object(bucket_name, path)); + } + + CHECK(conn->delete_object(bucket_name, "meta")); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Failed: %s", err.c_str()); }); + + try { + sink_creator_make_v2_metadata_sinks(thread_pool); + sink_creator_make_v3_metadata_sinks(thread_pool); + } catch (const std::exception& e) { + LOG_ERROR("Failed: %s", e.what()); + return 1; + } + + std::string s3_endpoint, bucket_name, s3_access_key_id, + s3_secret_access_key; + if (!get_credentials( + s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { + LOG_WARNING("Failed to get credentials. Skipping S3 portion of test."); + return 0; + } + + auto connection_pool = std::make_shared( + 4, s3_endpoint, s3_access_key_id, s3_secret_access_key); + + try { + sink_creator_make_v2_metadata_sinks( + thread_pool, connection_pool, bucket_name); + sink_creator_make_v3_metadata_sinks( + thread_pool, connection_pool, bucket_name); + } catch (const std::exception& e) { + LOG_ERROR("Failed: %s", e.what()); + return 1; + } + + return 0; +} \ No newline at end of file From 95222969c8301f6f6ce2fb7e5395293d1ce09c81 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 16:51:05 -0400 Subject: [PATCH 17/39] Implement and test base ArrayWriter. --- src/streaming/CMakeLists.txt | 2 + src/streaming/array.writer.cpp | 420 ++++++++++++++++++ src/streaming/array.writer.hh | 96 ++++ tests/unit-tests/CMakeLists.txt | 2 + .../array-writer-downsample-writer-config.cpp | 130 ++++++ .../array-writer-write-frame-to-chunks.cpp | 86 ++++ 6 files changed, 736 insertions(+) create mode 100644 src/streaming/array.writer.cpp create mode 100644 src/streaming/array.writer.hh create mode 100644 tests/unit-tests/array-writer-downsample-writer-config.cpp create mode 100644 tests/unit-tests/array-writer-write-frame-to-chunks.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 73f2b7cf..3c95a5b5 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -22,6 +22,8 @@ add_library(${tgt} s3.sink.cpp sink.creator.hh sink.creator.cpp + array.writer.hh + array.writer.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/array.writer.cpp b/src/streaming/array.writer.cpp new file mode 100644 index 00000000..529be751 --- /dev/null +++ b/src/streaming/array.writer.cpp @@ -0,0 +1,420 @@ +#include "macros.hh" +#include "array.writer.hh" +#include "zarr.common.hh" +#include "zarr.stream.hh" +#include "sink.creator.hh" + +#include +#include +#include +#include + +#ifdef min +#undef min +#endif + +bool +zarr::downsample(const ArrayWriterConfig& config, + ArrayWriterConfig& downsampled_config) +{ + // downsample dimensions + downsampled_config.dimensions.clear(); + for (const auto& dim : config.dimensions) { + if (dim.type == + ZarrDimensionType_Channel) { // don't downsample channels + downsampled_config.dimensions.push_back(dim); + } else { + const uint32_t array_size_px = + (dim.array_size_px + (dim.array_size_px % 2)) / 2; + + const uint32_t chunk_size_px = + dim.array_size_px == 0 + ? dim.chunk_size_px + : std::min(dim.chunk_size_px, array_size_px); + + CHECK(chunk_size_px); + const uint32_t n_chunks = + (array_size_px + chunk_size_px - 1) / chunk_size_px; + + const uint32_t shard_size_chunks = + dim.array_size_px == 0 + ? 1 + : std::min(n_chunks, dim.shard_size_chunks); + + downsampled_config.dimensions.emplace_back(dim.name, + dim.type, + array_size_px, + chunk_size_px, + shard_size_chunks); + } + } + + downsampled_config.level_of_detail = config.level_of_detail + 1; + downsampled_config.bucket_name = config.bucket_name; + downsampled_config.store_path = config.store_path; + + downsampled_config.dtype = config.dtype; + + // copy the Blosc compression parameters + downsampled_config.compression_params = config.compression_params; + + // can we downsample downsampled_config? + for (auto i = 0; i < config.dimensions.size(); ++i) { + // downsampling made the chunk size strictly smaller + const auto& dim = config.dimensions[i]; + const auto& downsampled_dim = downsampled_config.dimensions[i]; + + if (dim.chunk_size_px > downsampled_dim.chunk_size_px) { + return false; + } + } + + return true; +} + +/// Writer +zarr::ArrayWriter::ArrayWriter( + const ArrayWriterConfig& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool) + : config_{ config } + , thread_pool_{ thread_pool } + , s3_connection_pool_{ s3_connection_pool } + , bytes_to_flush_{ 0 } + , frames_written_{ 0 } + , append_chunk_index_{ 0 } + , is_finalizing_{ false } +{ +} + +size_t +zarr::ArrayWriter::write_frame(const uint8_t* data, size_t nbytes) +{ + const size_t nbytes_frame = + bytes_of_frame(config_.dimensions, config_.dtype); + + if (nbytes_frame != nbytes) { + LOG_WARNING("Frame size mismatch: expected %zu, got %zu. Skipping", + nbytes_frame, + nbytes); + return 0; + } + + if (chunk_buffers_.empty()) { + make_buffers_(); + } + + // split the incoming frame into tiles and write_frame them to the chunk + // buffers + const auto bytes_written = write_frame_to_chunks_(data, nbytes); + EXPECT(bytes_written == nbytes, "Failed to write_frame frame to chunks"); + + LOG_DEBUG("Wrote %zu bytes of frame %zu", bytes_written, frames_written_); + bytes_to_flush_ += bytes_written; + ++frames_written_; + + if (should_flush_()) { + flush_(); + } + + return bytes_written; +} + +bool +zarr::ArrayWriter::make_data_sinks_() +{ + std::string data_root; + std::function parts_along_dimension; + switch (version_()) { + case ZarrVersion_2: + parts_along_dimension = chunks_along_dimension; + data_root = config_.store_path + "/" + + std::to_string(config_.level_of_detail) + "/" + + std::to_string(append_chunk_index_); + break; + case ZarrVersion_3: + parts_along_dimension = shards_along_dimension; + data_root = config_.store_path + "/data/root/" + + std::to_string(config_.level_of_detail) + "/c" + + std::to_string(append_chunk_index_); + break; + default: + LOG_ERROR("Unsupported Zarr version"); + return false; + } + + SinkCreator creator(thread_pool_, s3_connection_pool_); + + if (config_.bucket_name && !creator.make_data_sinks(*config_.bucket_name, + data_root, + config_.dimensions, + parts_along_dimension, + data_sinks_)) { + LOG_ERROR("Failed to create data sinks in %s for bucket %s", + data_root.c_str(), + config_.bucket_name->c_str()); + return false; + } else if (!config_.bucket_name && + !creator.make_data_sinks(data_root, + config_.dimensions, + parts_along_dimension, + data_sinks_)) { + LOG_ERROR("Failed to create data sinks in %s", data_root.c_str()); + return false; + } + + return true; +} + +bool +zarr::ArrayWriter::make_metadata_sink_() +{ + if (metadata_sink_) { + return true; + } + + std::string metadata_path; + switch (version_()) { + case ZarrVersion_2: + metadata_path = config_.store_path + "/" + + std::to_string(config_.level_of_detail) + + "/.zarray"; + break; + case ZarrVersion_3: + metadata_path = config_.store_path + "/meta/root/" + + std::to_string(config_.level_of_detail) + + ".array.json"; + break; + default: + LOG_ERROR("Unsupported Zarr version"); + return false; + } + + if (config_.bucket_name) { + SinkCreator creator(thread_pool_, s3_connection_pool_); + metadata_sink_ = + creator.make_sink(*config_.bucket_name, metadata_path); + } else { + metadata_sink_ = zarr::SinkCreator::make_sink(metadata_path); + } + + if (!metadata_sink_) { + LOG_ERROR("Failed to create metadata sink: %s", + metadata_path.c_str()); + return false; + } + + return true; +} + +void +zarr::ArrayWriter::make_buffers_() noexcept +{ + LOG_DEBUG("Creating chunk buffers"); + + const size_t n_chunks = number_of_chunks_in_memory(config_.dimensions); + chunk_buffers_.resize(n_chunks); // no-op if already the correct size + + const auto nbytes = bytes_per_chunk(config_.dimensions, config_.dtype); + + for (auto& buf : chunk_buffers_) { + buf.resize(nbytes); + std::fill_n(buf.begin(), nbytes, 0); + } +} + +size_t +zarr::ArrayWriter::write_frame_to_chunks_(const uint8_t* buf, size_t buf_size) +{ + // break the frame into tiles and write_frame them to the chunk buffers + const auto bytes_per_px = bytes_of_type(config_.dtype); + + const auto& dimensions = config_.dimensions; + + const auto& x_dim = dimensions.back(); + const auto frame_cols = x_dim.array_size_px; + const auto tile_cols = x_dim.chunk_size_px; + + const auto& y_dim = dimensions[dimensions.size() - 2]; + const auto frame_rows = y_dim.array_size_px; + const auto tile_rows = y_dim.chunk_size_px; + + if (tile_cols == 0 || tile_rows == 0) { + return 0; + } + + const auto bytes_per_row = tile_cols * bytes_per_px; + + size_t bytes_written = 0; + + const auto n_tiles_x = (frame_cols + tile_cols - 1) / tile_cols; + const auto n_tiles_y = (frame_rows + tile_rows - 1) / tile_rows; + + // don't take the frame id from the incoming frame, as the camera may have + // dropped frames + const auto frame_id = frames_written_; + + // offset among the chunks in the lattice + const auto group_offset = tile_group_offset(frame_id, dimensions); + // offset within the chunk + const auto chunk_offset = + chunk_internal_offset(frame_id, dimensions, config_.dtype); + + for (auto i = 0; i < n_tiles_y; ++i) { + // TODO (aliddell): we can optimize this when tiles_per_frame_x_ is 1 + for (auto j = 0; j < n_tiles_x; ++j) { + const auto c = group_offset + i * n_tiles_x + j; + auto& chunk = chunk_buffers_[c]; + auto chunk_it = + chunk.begin() + static_cast(chunk_offset); + + for (auto k = 0; k < tile_rows; ++k) { + const auto frame_row = i * tile_rows + k; + if (frame_row < frame_rows) { + const auto frame_col = j * tile_cols; + + const auto region_width = + std::min(frame_col + tile_cols, frame_cols) - frame_col; + + const auto region_start = + bytes_per_px * (frame_row * frame_cols + frame_col); + const auto nbytes = region_width * bytes_per_px; + const auto region_stop = region_start + nbytes; + if (region_stop > buf_size) { + LOG_ERROR("Buffer overflow"); + return bytes_written; + } + + // copy region + if (nbytes > std::distance(chunk_it, chunk.end())) { + LOG_ERROR("Buffer overflow"); + return bytes_written; + } + std::copy(buf + region_start, buf + region_stop, chunk_it); + + bytes_written += (region_stop - region_start); + } + chunk_it += static_cast(bytes_per_row); + } + } + } + + return bytes_written; +} + +bool +zarr::ArrayWriter::should_flush_() const +{ + const auto& dims = config_.dimensions; + size_t frames_before_flush = dims.front().chunk_size_px; + for (auto i = 1; i < dims.size() - 2; ++i) { + frames_before_flush *= dims[i].array_size_px; + } + + CHECK(frames_before_flush > 0); + return frames_written_ % frames_before_flush == 0; +} + +void +zarr::ArrayWriter::compress_buffers_() +{ + if (!config_.compression_params.has_value()) { + return; + } + + LOG_DEBUG("Compressing"); + + BloscCompressionParams params = config_.compression_params.value(); + const auto bytes_per_px = bytes_of_type(config_.dtype); + + std::scoped_lock lock(buffers_mutex_); + std::latch latch(chunk_buffers_.size()); + for (auto& chunk : chunk_buffers_) { + EXPECT(thread_pool_->push_to_job_queue( + [¶ms, buf = &chunk, bytes_per_px, &latch]( + std::string& err) -> bool { + bool success = false; + const size_t bytes_of_chunk = buf->size(); + + try { + const auto tmp_size = + bytes_of_chunk + BLOSC_MAX_OVERHEAD; + std::vector tmp(tmp_size); + const auto nb = + blosc_compress_ctx(params.clevel, + params.shuffle, + bytes_per_px, + bytes_of_chunk, + buf->data(), + tmp.data(), + tmp_size, + params.codec_id.c_str(), + 0 /* blocksize - 0:automatic */, + 1); + + tmp.resize(nb); + buf->swap(tmp); + + success = true; + } catch (const std::exception& exc) { + char msg[128]; + snprintf(msg, + sizeof(msg), + "Failed to compress chunk: %s", + exc.what()); + err = msg; + } catch (...) { + err = "Failed to compress chunk (unknown)"; + } + latch.count_down(); + + return success; + }), + "Failed to push to job queue"); + } + + // wait for all threads to finish + latch.wait(); +} + +void +zarr::ArrayWriter::flush_() +{ + if (bytes_to_flush_ == 0) { + return; + } + + // compress buffers and write_frame out + compress_buffers_(); + CHECK(flush_impl_()); + + const auto should_rollover = should_rollover_(); + if (should_rollover) { + rollover_(); + } + + if (should_rollover || is_finalizing_) { + CHECK(write_array_metadata_()); + } + + // reset buffers + make_buffers_(); + + // reset state + bytes_to_flush_ = 0; +} + +void +zarr::ArrayWriter::close_sinks_() +{ + data_sinks_.clear(); +} + +void +zarr::ArrayWriter::rollover_() +{ + LOG_DEBUG("Rolling over"); + + close_sinks_(); + ++append_chunk_index_; +} diff --git a/src/streaming/array.writer.hh b/src/streaming/array.writer.hh new file mode 100644 index 00000000..e04eac61 --- /dev/null +++ b/src/streaming/array.writer.hh @@ -0,0 +1,96 @@ +#pragma once + +#include "stream.settings.hh" +#include "thread.pool.hh" +#include "s3.connection.hh" +#include "blosc.compression.params.hh" +#include "file.sink.hh" + +#include +#include + +namespace fs = std::filesystem; + +namespace zarr { + +struct ArrayWriterConfig final +{ + std::vector dimensions; + ZarrDataType dtype; + int level_of_detail; + std::optional bucket_name; + std::string store_path; + std::optional compression_params; +}; + +/// @brief Downsample the array writer configuration to a lower resolution. +/// @param[in] config The original array writer configuration. +/// @param[out] downsampled_config The downsampled array writer configuration. +/// @return True if @p downsampled_config can be downsampled further. +/// This is determined by the chunk size in @p config. This function will return +/// false if and only if downsampling brings one or more dimensions lower than +/// the chunk size along that dimension. +[[nodiscard]] bool +downsample(const ArrayWriterConfig& config, + ArrayWriterConfig& downsampled_config); + +class ArrayWriter +{ + public: + ArrayWriter(const ArrayWriterConfig& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool); + + virtual ~ArrayWriter() = default; + + /** + * @brief Write a frame to the array. + * @param data The frame data. + * @param nbytes The number of bytes in the frame. + * @return The number of bytes written. + */ + [[nodiscard]] size_t write_frame(const uint8_t* data, size_t nbytes); + + protected: + ArrayWriterConfig config_; + + /// Chunking + std::vector> chunk_buffers_; + + /// Filesystem + std::vector> data_sinks_; + std::unique_ptr metadata_sink_; + + /// Multithreading + std::shared_ptr thread_pool_; + std::mutex buffers_mutex_; + + /// Bookkeeping + uint64_t bytes_to_flush_; + uint32_t frames_written_; + uint32_t append_chunk_index_; + bool is_finalizing_; + + std::shared_ptr s3_connection_pool_; + + virtual ZarrVersion version_() const = 0; + + [[nodiscard]] bool make_data_sinks_(); + [[nodiscard]] bool make_metadata_sink_(); + void make_buffers_() noexcept; + + bool should_flush_() const; + virtual bool should_rollover_() const = 0; + + size_t write_frame_to_chunks_(const uint8_t* buf, size_t buf_size); + void compress_buffers_(); + + void flush_(); + [[nodiscard]] virtual bool flush_impl_() = 0; + void rollover_(); + + [[nodiscard]] virtual bool write_array_metadata_() = 0; + + void close_sinks_(); +}; +} // namespace zarr diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index f5a7c5a7..0f1fdad7 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -17,6 +17,8 @@ set(tests s3-sink-write sink-creator-make-metadata-sinks sink-creator-make-data-sinks + array-writer-downsample-writer-config + array-writer-write-frame-to-chunks ) foreach (name ${tests}) diff --git a/tests/unit-tests/array-writer-downsample-writer-config.cpp b/tests/unit-tests/array-writer-downsample-writer-config.cpp new file mode 100644 index 00000000..79f32434 --- /dev/null +++ b/tests/unit-tests/array-writer-downsample-writer-config.cpp @@ -0,0 +1,130 @@ +#include "array.writer.hh" +#include "unit.test.macros.hh" + +#include + +namespace fs = std::filesystem; + +int +main() +{ + int retval = 1; + + try { + const fs::path base_dir = "acquire"; + + zarr::ArrayWriterConfig config{ .dimensions = {}, + .dtype = ZarrDataType_uint8, + .level_of_detail = 0, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt }; + + config.dimensions.emplace_back("t", + ZarrDimensionType_Time, + 0, + 5, + 1); // 5 timepoints / chunk, 1 shard + config.dimensions.emplace_back( + "c", ZarrDimensionType_Channel, 2, 1, 1); // 2 chunks, 2 shards + config.dimensions.emplace_back( + "z", ZarrDimensionType_Space, 7, 3, 3); // 3 chunks, 3 shards + config.dimensions.emplace_back( + "y", ZarrDimensionType_Space, 48, 16, 3); // 3 chunks, 1 shard + config.dimensions.emplace_back( + "x", ZarrDimensionType_Space, 64, 16, 2); // 4 chunks, 2 shards + + zarr::ArrayWriterConfig downsampled_config; + CHECK(zarr::downsample(config, downsampled_config)); + + // check dimensions + CHECK(downsampled_config.dimensions.size() == 5); + + CHECK(downsampled_config.dimensions[0].name == "t"); + CHECK(downsampled_config.dimensions[0].array_size_px == 0); + CHECK(downsampled_config.dimensions[0].chunk_size_px == 5); + CHECK(downsampled_config.dimensions[0].shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions[1].name == "c"); + // we don't downsample channels + CHECK(downsampled_config.dimensions[1].array_size_px == 2); + CHECK(downsampled_config.dimensions[1].chunk_size_px == 1); + CHECK(downsampled_config.dimensions[1].shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions[2].name == "z"); + CHECK(downsampled_config.dimensions[2].array_size_px == 4); + CHECK(downsampled_config.dimensions[2].chunk_size_px == 3); + CHECK(downsampled_config.dimensions[2].shard_size_chunks == 2); + + CHECK(downsampled_config.dimensions[3].name == "y"); + CHECK(downsampled_config.dimensions[3].array_size_px == 24); + CHECK(downsampled_config.dimensions[3].chunk_size_px == 16); + CHECK(downsampled_config.dimensions[3].shard_size_chunks == 2); + + CHECK(downsampled_config.dimensions[4].name == "x"); + CHECK(downsampled_config.dimensions[4].array_size_px == 32); + CHECK(downsampled_config.dimensions[4].chunk_size_px == 16); + CHECK(downsampled_config.dimensions[4].shard_size_chunks == 2); + + // check level of detail + CHECK(downsampled_config.level_of_detail == 1); + + // check store path + CHECK(downsampled_config.store_path == config.store_path); + + // check compression params + CHECK(!downsampled_config.compression_params.has_value()); + + // downsample again + config = std::move(downsampled_config); + + // can't downsample anymore + CHECK(!zarr::downsample(config, downsampled_config)); + + // check dimensions + CHECK(downsampled_config.dimensions.size() == 5); + + CHECK(downsampled_config.dimensions[0].name == "t"); + CHECK(downsampled_config.dimensions[0].array_size_px == 0); + CHECK(downsampled_config.dimensions[0].chunk_size_px == 5); + CHECK(downsampled_config.dimensions[0].shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions[1].name == "c"); + // we don't downsample channels + CHECK(downsampled_config.dimensions[1].array_size_px == 2); + CHECK(downsampled_config.dimensions[1].chunk_size_px == 1); + CHECK(downsampled_config.dimensions[1].shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions[2].name == "z"); + CHECK(downsampled_config.dimensions[2].array_size_px == 2); + CHECK(downsampled_config.dimensions[2].chunk_size_px == 2); + CHECK(downsampled_config.dimensions[2].shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions[3].name == "y"); + CHECK(downsampled_config.dimensions[3].array_size_px == 12); + CHECK(downsampled_config.dimensions[3].chunk_size_px == 12); + CHECK(downsampled_config.dimensions[3].shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions[4].name == "x"); + CHECK(downsampled_config.dimensions[4].array_size_px == 16); + CHECK(downsampled_config.dimensions[4].chunk_size_px == 16); + CHECK(downsampled_config.dimensions[4].shard_size_chunks == 1); + + // check level of detail + CHECK(downsampled_config.level_of_detail == 2); + + // check data root + CHECK(downsampled_config.store_path == config.store_path); + + // check compression params + CHECK(!downsampled_config.compression_params.has_value()); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/array-writer-write-frame-to-chunks.cpp b/tests/unit-tests/array-writer-write-frame-to-chunks.cpp new file mode 100644 index 00000000..3d890bc6 --- /dev/null +++ b/tests/unit-tests/array-writer-write-frame-to-chunks.cpp @@ -0,0 +1,86 @@ +#include "array.writer.hh" +#include "unit.test.macros.hh" +#include "zarr.common.hh" + +namespace { +class TestWriter : public zarr::ArrayWriter +{ + public: + TestWriter(const zarr::ArrayWriterConfig& array_spec, + std::shared_ptr thread_pool) + : zarr::ArrayWriter(array_spec, thread_pool, nullptr) + { + } + + private: + ZarrVersion version_() const override { return ZarrVersionCount; } + bool should_rollover_() const override { return false; } + bool flush_impl_() override { return true; } + bool write_array_metadata_() override { return true; } +}; +} // namespace + +int +main() +{ + const auto base_dir = fs::temp_directory_path() / TEST; + int retval = 1; + + const unsigned int array_width = 64, array_height = 48, array_planes = 2, + array_channels = 1, array_timepoints = 2; + const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 1, + chunk_channels = 1, chunk_timepoints = 1; + + const unsigned int n_frames = + array_planes * array_channels * array_timepoints; + + const ZarrDataType dtype = ZarrDataType_uint16; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); + dims.emplace_back( + "c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); + dims.emplace_back( + "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); + dims.emplace_back( + "y", ZarrDimensionType_Space, array_height, chunk_height, 0); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, 0); + + zarr::ArrayWriterConfig config = { + .dimensions = dims, + .dtype = dtype, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + TestWriter writer(config, thread_pool); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data(frame_size, 0); + + for (auto i = 0; i < n_frames; ++i) { + CHECK(writer.write_frame(data.data(), frame_size) == frame_size); + } + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file From d291c537c6f6bb8e1aef013a6995ef728673fb33 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 16:59:15 -0400 Subject: [PATCH 18/39] Implement and test ZarrV2ArrayWriter. --- src/streaming/CMakeLists.txt | 2 + src/streaming/zarrv2.array.writer.cpp | 177 +++++++++++++++++ src/streaming/zarrv2.array.writer.hh | 22 +++ tests/unit-tests/CMakeLists.txt | 3 + tests/unit-tests/zarrv2-writer-write-even.cpp | 182 ++++++++++++++++++ .../zarrv2-writer-write-ragged-append-dim.cpp | 151 +++++++++++++++ ...arrv2-writer-write-ragged-internal-dim.cpp | 169 ++++++++++++++++ 7 files changed, 706 insertions(+) create mode 100644 src/streaming/zarrv2.array.writer.cpp create mode 100644 src/streaming/zarrv2.array.writer.hh create mode 100644 tests/unit-tests/zarrv2-writer-write-even.cpp create mode 100644 tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp create mode 100644 tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 3c95a5b5..bf81bbbd 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -24,6 +24,8 @@ add_library(${tgt} sink.creator.cpp array.writer.hh array.writer.cpp + zarrv2.array.writer.hh + zarrv2.array.writer.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/zarrv2.array.writer.cpp b/src/streaming/zarrv2.array.writer.cpp new file mode 100644 index 00000000..56f21509 --- /dev/null +++ b/src/streaming/zarrv2.array.writer.cpp @@ -0,0 +1,177 @@ +#include "macros.hh" +#include "zarrv2.array.writer.hh" +#include "sink.creator.hh" +#include "zarr.common.hh" + +#include + +#include +#include + +namespace { +std::string +sample_type_to_dtype(ZarrDataType t) + +{ + const std::string dtype_prefix = + std::endian::native == std::endian::big ? ">" : "<"; + + switch (t) { + case ZarrDataType_uint8: + return dtype_prefix + "u1"; + case ZarrDataType_uint16: + return dtype_prefix + "u2"; + case ZarrDataType_uint32: + return dtype_prefix + "u4"; + case ZarrDataType_uint64: + return dtype_prefix + "u8"; + case ZarrDataType_int8: + return dtype_prefix + "i1"; + case ZarrDataType_int16: + return dtype_prefix + "i2"; + case ZarrDataType_int32: + return dtype_prefix + "i4"; + case ZarrDataType_int64: + return dtype_prefix + "i8"; + case ZarrDataType_float32: + return dtype_prefix + "f4"; + case ZarrDataType_float64: + return dtype_prefix + "f8"; + default: + throw std::runtime_error("Invalid data type: " + + std::to_string(static_cast(t))); + } +} +} // namespace + +zarr::ZarrV2ArrayWriter::ZarrV2ArrayWriter( + const ArrayWriterConfig& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool) + : ArrayWriter(config, thread_pool, s3_connection_pool) +{ +} + +zarr::ZarrV2ArrayWriter::~ZarrV2ArrayWriter() +{ + is_finalizing_ = true; + try { + flush_(); + } catch (const std::exception& exc) { + LOG_ERROR("Failed to finalize array writer: %s", exc.what()); + } catch (...) { + LOG_ERROR("Failed to finalize array writer: (unknown)"); + } +} + +ZarrVersion +zarr::ZarrV2ArrayWriter::version_() const +{ + return ZarrVersion_2; +} + +bool +zarr::ZarrV2ArrayWriter::flush_impl_() +{ + // create chunk files + CHECK(data_sinks_.empty()); + if (!make_data_sinks_()) { + return false; + } + + CHECK(data_sinks_.size() == chunk_buffers_.size()); + + std::latch latch(chunk_buffers_.size()); + { + std::scoped_lock lock(buffers_mutex_); + for (auto i = 0; i < data_sinks_.size(); ++i) { + auto& chunk = chunk_buffers_.at(i); + EXPECT(thread_pool_->push_to_job_queue( + std::move([&sink = data_sinks_.at(i), + data = chunk.data(), + size = chunk.size(), + &latch](std::string& err) -> bool { + bool success = false; + try { + CHECK(sink->write(0, data, size)); + success = true; + } catch (const std::exception& exc) { + err = "Failed to write chunk: " + + std::string(exc.what()); + } catch (...) { + err = "Failed to write chunk: (unknown)"; + } + + latch.count_down(); + return success; + })), + "Failed to push job to thread pool"); + } + } + + // wait for all threads to finish + latch.wait(); + + return true; +} + +bool +zarr::ZarrV2ArrayWriter::write_array_metadata_() +{ + if (!make_metadata_sink_()) { + return false; + } + + using json = nlohmann::json; + + std::vector array_shape, chunk_shape; + + size_t append_size = frames_written_; + for (auto dim = config_.dimensions.rbegin() + 2; + dim < config_.dimensions.rend() - 1; + ++dim) { + CHECK(dim->array_size_px); + append_size = + (append_size + dim->array_size_px - 1) / dim->array_size_px; + } + array_shape.push_back(append_size); + + chunk_shape.push_back(config_.dimensions.front().chunk_size_px); + for (auto dim = config_.dimensions.begin() + 1; + dim != config_.dimensions.end(); + ++dim) { + array_shape.push_back(dim->array_size_px); + chunk_shape.push_back(dim->chunk_size_px); + } + + json metadata; + metadata["zarr_format"] = 2; + metadata["shape"] = array_shape; + metadata["chunks"] = chunk_shape; + metadata["dtype"] = sample_type_to_dtype(config_.dtype); + metadata["fill_value"] = 0; + metadata["order"] = "C"; + metadata["filters"] = nullptr; + metadata["dimension_separator"] = "/"; + + if (config_.compression_params) { + const BloscCompressionParams bcp = *config_.compression_params; + metadata["compressor"] = json{ { "id", "blosc" }, + { "cname", bcp.codec_id }, + { "clevel", bcp.clevel }, + { "shuffle", bcp.shuffle } }; + } else { + metadata["compressor"] = nullptr; + } + + const std::string metadata_str = metadata.dump(4); + const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); + + return metadata_sink_->write(0, metadata_bytes, metadata_str.size()); +} + +bool +zarr::ZarrV2ArrayWriter::should_rollover_() const +{ + return true; +} diff --git a/src/streaming/zarrv2.array.writer.hh b/src/streaming/zarrv2.array.writer.hh new file mode 100644 index 00000000..c756ad75 --- /dev/null +++ b/src/streaming/zarrv2.array.writer.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "array.writer.hh" + +namespace zarr { +class ZarrV2ArrayWriter final : public ArrayWriter +{ + public: + ZarrV2ArrayWriter( + const ArrayWriterConfig& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool); + + ~ZarrV2ArrayWriter() override; + + private: + ZarrVersion version_() const override; + bool flush_impl_() override; + bool write_array_metadata_() override; + bool should_rollover_() const override; +}; +} // namespace zarr diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index 0f1fdad7..52adb1bc 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -19,6 +19,9 @@ set(tests sink-creator-make-data-sinks array-writer-downsample-writer-config array-writer-write-frame-to-chunks + zarrv2-writer-write-even + zarrv2-writer-write-ragged-append-dim + zarrv2-writer-write-ragged-internal-dim ) foreach (name ${tests}) diff --git a/tests/unit-tests/zarrv2-writer-write-even.cpp b/tests/unit-tests/zarrv2-writer-write-even.cpp new file mode 100644 index 00000000..9f24abab --- /dev/null +++ b/tests/unit-tests/zarrv2-writer-write-even.cpp @@ -0,0 +1,182 @@ +#include "zarrv2.array.writer.hh" +#include "macros.hh" +#include "zarr.common.hh" + +#include + +#include + +#define EXPECT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +namespace fs = std::filesystem; + +namespace { +const fs::path base_dir = fs::temp_directory_path() / TEST; + +const unsigned int array_width = 64, array_height = 48, array_planes = 6, + array_channels = 8, array_timepoints = 10; +const unsigned int n_frames = array_planes * array_channels * array_timepoints; + +const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, + chunk_channels = 4, chunk_timepoints = 5; + +const unsigned int chunks_in_x = + (array_width + chunk_width - 1) / chunk_width; // 4 chunks +const unsigned int chunks_in_y = + (array_height + chunk_height - 1) / chunk_height; // 3 chunks + +const unsigned int chunks_in_z = + (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks +const unsigned int chunks_in_c = + (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks +const unsigned int chunks_in_t = + (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; // 2 chunks + +const int level_of_detail = 0; +} // namespace + +void +check_json() +{ + fs::path meta_path = + base_dir / std::to_string(level_of_detail) / ".zarray"; + CHECK(fs::is_regular_file(meta_path)); + + std::ifstream f(meta_path); + nlohmann::json meta = nlohmann::json::parse(f); + + EXPECT(meta["dtype"].get() == "().c_str()); + + EXPECT_EQ(meta["zarr_format"].get(), 2); + + const auto& array_shape = meta["shape"]; + const auto& chunk_shape = meta["chunks"]; + + EXPECT_EQ(array_shape.size(), 5); + EXPECT_EQ(array_shape[0].get(), array_timepoints); + EXPECT_EQ(array_shape[1].get(), array_channels); + EXPECT_EQ(array_shape[2].get(), array_planes); + EXPECT_EQ(array_shape[3].get(), array_height); + EXPECT_EQ(array_shape[4].get(), array_width); + + EXPECT_EQ(chunk_shape.size(), 5); + EXPECT_EQ(chunk_shape[0].get(), chunk_timepoints); + EXPECT_EQ(chunk_shape[1].get(), chunk_channels); + EXPECT_EQ(chunk_shape[2].get(), chunk_planes); + EXPECT_EQ(chunk_shape[3].get(), chunk_height); + EXPECT_EQ(chunk_shape[4].get(), chunk_width); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + int retval = 1; + + const ZarrDataType dtype = ZarrDataType_uint16; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), [](const std::string& err) { + LOG_ERROR("Error: %s\n", err.c_str()); + }); + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); + dims.emplace_back( + "c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); + dims.emplace_back( + "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); + dims.emplace_back( + "y", ZarrDimensionType_Space, array_height, chunk_height, 0); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, 0); + + zarr::ArrayWriterConfig config = { + .dimensions = dims, + .dtype = dtype, + .level_of_detail = level_of_detail, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + { + zarr::ZarrV2ArrayWriter writer(config, thread_pool, nullptr); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data(frame_size, 0); + + for (auto i = 0; i < n_frames; ++i) { // 2 time points + CHECK(writer.write_frame(data.data(), frame_size)); + } + } + + check_json(); + + const auto expected_file_size = chunk_width * chunk_height * + chunk_planes * chunk_channels * + chunk_timepoints * nbytes_px; + + const fs::path data_root = + base_dir / std::to_string(config.level_of_detail); + + CHECK(fs::is_directory(data_root)); + for (auto t = 0; t < chunks_in_t; ++t) { + const auto t_dir = data_root / std::to_string(t); + CHECK(fs::is_directory(t_dir)); + + for (auto c = 0; c < chunks_in_c; ++c) { + const auto c_dir = t_dir / std::to_string(c); + CHECK(fs::is_directory(c_dir)); + + for (auto z = 0; z < chunks_in_z; ++z) { + const auto z_dir = c_dir / std::to_string(z); + CHECK(fs::is_directory(z_dir)); + + for (auto y = 0; y < chunks_in_y; ++y) { + const auto y_dir = z_dir / std::to_string(y); + CHECK(fs::is_directory(y_dir)); + + for (auto x = 0; x < chunks_in_x; ++x) { + const auto x_file = y_dir / std::to_string(x); + CHECK(fs::is_regular_file(x_file)); + const auto file_size = fs::file_size(x_file); + EXPECT_EQ(file_size, expected_file_size); + } + + CHECK(!fs::is_regular_file( + y_dir / std::to_string(chunks_in_x))); + } + + CHECK( + !fs::is_directory(z_dir / std::to_string(chunks_in_y))); + } + + CHECK(!fs::is_directory(c_dir / std::to_string(chunks_in_z))); + } + + CHECK(!fs::is_directory(t_dir / std::to_string(chunks_in_c))); + } + + CHECK(!fs::is_directory(data_root / std::to_string(chunks_in_t))); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp new file mode 100644 index 00000000..fc37b1cc --- /dev/null +++ b/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp @@ -0,0 +1,151 @@ +#include "zarrv2.array.writer.hh" +#include "macros.hh" +#include "zarr.common.hh" + +#include +#include + +#define EXPECT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +namespace fs = std::filesystem; + +namespace { +const fs::path base_dir = fs::temp_directory_path() / TEST; + +const unsigned int array_width = 64, array_height = 48, array_planes = 5; +const unsigned int n_frames = array_planes; + +const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2; + +const unsigned int chunks_in_x = + (array_width + chunk_width - 1) / chunk_width; // 4 chunks +const unsigned int chunks_in_y = + (array_height + chunk_height - 1) / chunk_height; // 3 chunks + +const unsigned int chunks_in_z = + (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged + +const int level_of_detail = 1; +} // namespace + +void +check_json() +{ + fs::path zarray_path = + base_dir / std::to_string(level_of_detail) / ".zarray"; + CHECK(fs::is_regular_file(zarray_path)); + + std::ifstream f(zarray_path); + nlohmann::json zarray = nlohmann::json::parse(f); + + EXPECT(zarray["dtype"].get() == "().c_str()); + + EXPECT_EQ(zarray["zarr_format"].get(), 2); + + const auto& chunks = zarray["chunks"]; + EXPECT_EQ(chunks.size(), 3); + EXPECT_EQ(chunks[0].get(), chunk_planes); + EXPECT_EQ(chunks[1].get(), chunk_height); + EXPECT_EQ(chunks[2].get(), chunk_width); + + const auto& shape = zarray["shape"]; + EXPECT_EQ(shape.size(), 3); + EXPECT_EQ(shape[0].get(), array_planes); + EXPECT_EQ(shape[1].get(), array_height); + EXPECT_EQ(shape[2].get(), array_width); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + int retval = 1; + + const ZarrDataType dtype = ZarrDataType_uint8; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), [](const std::string& err) { + LOG_ERROR("Error: %s\n", err.c_str()); + }); + + std::vector dims; + dims.emplace_back( + "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); + dims.emplace_back( + "y", ZarrDimensionType_Space, array_height, chunk_height, 0); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, 0); + + zarr::ArrayWriterConfig config = { + .dimensions = dims, + .dtype = dtype, + .level_of_detail = level_of_detail, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + { + zarr::ZarrV2ArrayWriter writer(config, thread_pool, nullptr); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data(frame_size, 0); + + for (auto i = 0; i < n_frames; ++i) { // 2 time points + CHECK(writer.write_frame(data.data(), frame_size)); + } + } + + check_json(); + + const auto expected_file_size = + chunk_width * chunk_height * chunk_planes * nbytes_px; + + const fs::path data_root = + base_dir / std::to_string(config.level_of_detail); + + CHECK(fs::is_directory(data_root)); + + for (auto z = 0; z < chunks_in_z; ++z) { + const auto z_dir = data_root / std::to_string(z); + CHECK(fs::is_directory(z_dir)); + + for (auto y = 0; y < chunks_in_y; ++y) { + const auto y_dir = z_dir / std::to_string(y); + CHECK(fs::is_directory(y_dir)); + + for (auto x = 0; x < chunks_in_x; ++x) { + const auto x_file = y_dir / std::to_string(x); + CHECK(fs::is_regular_file(x_file)); + const auto file_size = fs::file_size(x_file); + EXPECT_EQ(file_size, expected_file_size); + } + + CHECK( + !fs::is_regular_file(y_dir / std::to_string(chunks_in_x))); + } + + CHECK(!fs::is_directory(z_dir / std::to_string(chunks_in_y))); + } + + CHECK(!fs::is_directory(data_root / std::to_string(chunks_in_z))); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp new file mode 100644 index 00000000..e78ef981 --- /dev/null +++ b/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp @@ -0,0 +1,169 @@ +#include "zarrv2.array.writer.hh" +#include "macros.hh" +#include "zarr.common.hh" + +#include +#include + +#define EXPECT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +namespace fs = std::filesystem; + +namespace { +const fs::path base_dir = fs::temp_directory_path() / TEST; + +const unsigned int array_width = 64, array_height = 48, array_planes = 5, + array_timepoints = 5; +const unsigned int n_frames = array_planes * array_timepoints; + +const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, + chunk_timepoints = 5; + +const unsigned int chunks_in_x = + (array_width + chunk_width - 1) / chunk_width; // 4 chunks +const unsigned int chunks_in_y = + (array_height + chunk_height - 1) / chunk_height; // 3 chunks +const unsigned int chunks_in_z = + (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged +const unsigned int chunks_in_t = + (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; // 1 chunk + +const int level_of_detail = 2; +} // namespace + +void +check_json() +{ + fs::path zarray_path = + base_dir / std::to_string(level_of_detail) / ".zarray"; + CHECK(fs::is_regular_file(zarray_path)); + + std::ifstream f(zarray_path); + nlohmann::json zarray = nlohmann::json::parse(f); + + EXPECT(zarray["dtype"].get() == "().c_str()); + + EXPECT_EQ(zarray["zarr_format"].get(), 2); + + const auto& chunks = zarray["chunks"]; + EXPECT_EQ(chunks.size(), 4); + EXPECT_EQ(chunks[0].get(), chunk_timepoints); + EXPECT_EQ(chunks[1].get(), chunk_planes); + EXPECT_EQ(chunks[2].get(), chunk_height); + EXPECT_EQ(chunks[3].get(), chunk_width); + + const auto& shape = zarray["shape"]; + EXPECT_EQ(shape.size(), 4); + EXPECT_EQ(shape[0].get(), array_timepoints); + EXPECT_EQ(shape[1].get(), array_planes); + EXPECT_EQ(shape[2].get(), array_height); + EXPECT_EQ(shape[3].get(), array_width); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + int retval = 1; + + const ZarrDataType dtype = ZarrDataType_float64; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), [](const std::string& err) { + LOG_ERROR("Error: %s\n", err.c_str()); + }); + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); + dims.emplace_back( + "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); + dims.emplace_back( + "y", ZarrDimensionType_Space, array_height, chunk_height, 0); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, 0); + + zarr::ArrayWriterConfig config = { + .dimensions = dims, + .dtype = dtype, + .level_of_detail = level_of_detail, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + { + zarr::ZarrV2ArrayWriter writer(config, thread_pool, nullptr); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data(frame_size, 0); + + for (auto i = 0; i < n_frames; ++i) { // 2 time points + CHECK(writer.write_frame(data.data(), frame_size)); + } + } + + check_json(); + + const auto expected_file_size = chunk_width * chunk_height * + chunk_planes * chunk_timepoints * + nbytes_px; + + const fs::path data_root = + base_dir / std::to_string(config.level_of_detail); + + CHECK(fs::is_directory(data_root)); + + for (auto t = 0; t < chunks_in_t; ++t) { + const auto t_dir = data_root / std::to_string(t); + CHECK(fs::is_directory(t_dir)); + { + for (auto z = 0; z < chunks_in_z; ++z) { + const auto z_dir = t_dir / std::to_string(z); + CHECK(fs::is_directory(z_dir)); + + for (auto y = 0; y < chunks_in_y; ++y) { + const auto y_dir = z_dir / std::to_string(y); + CHECK(fs::is_directory(y_dir)); + + for (auto x = 0; x < chunks_in_x; ++x) { + const auto x_file = y_dir / std::to_string(x); + CHECK(fs::is_regular_file(x_file)); + const auto file_size = fs::file_size(x_file); + EXPECT_EQ(file_size, expected_file_size); + } + + CHECK(!fs::is_regular_file( + y_dir / std::to_string(chunks_in_x))); + } + + CHECK( + !fs::is_directory(z_dir / std::to_string(chunks_in_y))); + } + + CHECK( + !fs::is_directory(t_dir / std::to_string(chunks_in_z))); + } + + CHECK(!fs::is_directory(data_root / std::to_string(chunks_in_t))); + } + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file From 082dcd0a02e70e951de431b919df7042cf8a15a4 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 17:04:24 -0400 Subject: [PATCH 19/39] Implement and test ZarrV3ArrayWriter. --- src/streaming/CMakeLists.txt | 2 + src/streaming/zarrv3.array.writer.cpp | 267 ++++++++++++++++++ src/streaming/zarrv3.array.writer.hh | 25 ++ tests/unit-tests/CMakeLists.txt | 3 + tests/unit-tests/zarrv3-writer-write-even.cpp | 221 +++++++++++++++ .../zarrv3-writer-write-ragged-append-dim.cpp | 177 ++++++++++++ ...arrv3-writer-write-ragged-internal-dim.cpp | 200 +++++++++++++ 7 files changed, 895 insertions(+) create mode 100644 src/streaming/zarrv3.array.writer.cpp create mode 100644 src/streaming/zarrv3.array.writer.hh create mode 100644 tests/unit-tests/zarrv3-writer-write-even.cpp create mode 100644 tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp create mode 100644 tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index bf81bbbd..08093c37 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -26,6 +26,8 @@ add_library(${tgt} array.writer.cpp zarrv2.array.writer.hh zarrv2.array.writer.cpp + zarrv3.array.writer.hh + zarrv3.array.writer.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/zarrv3.array.writer.cpp b/src/streaming/zarrv3.array.writer.cpp new file mode 100644 index 00000000..84f3314d --- /dev/null +++ b/src/streaming/zarrv3.array.writer.cpp @@ -0,0 +1,267 @@ +#include "macros.hh" +#include "zarrv3.array.writer.hh" +#include "sink.creator.hh" +#include "zarr.common.hh" + +#include + +#include // std::fill_n +#include +#include + +#ifdef max +#undef max +#endif + +namespace { +std::string +sample_type_to_dtype(ZarrDataType t) + +{ + switch (t) { + case ZarrDataType_uint8: + return "uint8"; + case ZarrDataType_uint16: + return "uint16"; + case ZarrDataType_uint32: + return "uint32"; + case ZarrDataType_uint64: + return "uint64"; + case ZarrDataType_int8: + return "int8"; + case ZarrDataType_int16: + return "int16"; + case ZarrDataType_int32: + return "int32"; + case ZarrDataType_int64: + return "int64"; + case ZarrDataType_float32: + return "float32"; + case ZarrDataType_float64: + return "float64"; + default: + throw std::runtime_error("Invalid ZarrDataType: " + + std::to_string(static_cast(t))); + } +} +} // namespace + +zarr::ZarrV3ArrayWriter::ZarrV3ArrayWriter( + const ArrayWriterConfig& array_spec, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool) + : ArrayWriter(array_spec, thread_pool, s3_connection_pool) + , shard_file_offsets_(number_of_shards(array_spec.dimensions), 0) + , shard_tables_{ number_of_shards(array_spec.dimensions) } +{ + const auto cps = chunks_per_shard(array_spec.dimensions); + + for (auto& table : shard_tables_) { + table.resize(2 * cps); + std::fill_n( + table.begin(), table.size(), std::numeric_limits::max()); + } +} + +zarr::ZarrV3ArrayWriter::~ZarrV3ArrayWriter() +{ + is_finalizing_ = true; + try { + flush_(); + } catch (const std::exception& exc) { + LOG_ERROR("Failed to finalize array writer: %s", exc.what()); + } catch (...) { + LOG_ERROR("Failed to finalize array writer: (unknown)"); + } +} + +ZarrVersion +zarr::ZarrV3ArrayWriter::version_() const +{ + return ZarrVersion_3; +} + +bool +zarr::ZarrV3ArrayWriter::flush_impl_() +{ + // create shard files if they don't exist + if (data_sinks_.empty() && !make_data_sinks_()) { + return false; + } + + const auto n_shards = number_of_shards(config_.dimensions); + CHECK(data_sinks_.size() == n_shards); + + // get shard indices for each chunk + std::vector> chunk_in_shards(n_shards); + for (auto i = 0; i < chunk_buffers_.size(); ++i) { + const auto index = shard_index_for_chunk(i, config_.dimensions); + chunk_in_shards.at(index).push_back(i); + } + + // write out chunks to shards + bool write_table = is_finalizing_ || should_rollover_(); + std::latch latch(n_shards); + for (auto i = 0; i < n_shards; ++i) { + const auto& chunks = chunk_in_shards.at(i); + auto& chunk_table = shard_tables_.at(i); + size_t* file_offset = &shard_file_offsets_.at(i); + + EXPECT(thread_pool_->push_to_job_queue([&sink = data_sinks_.at(i), + &chunks, + &chunk_table, + file_offset, + write_table, + &latch, + this]( + std::string& err) mutable { + bool success = false; + + try { + for (const auto& chunk_idx : chunks) { + auto& chunk = chunk_buffers_.at(chunk_idx); + success = + sink->write(*file_offset, chunk.data(), chunk.size()); + if (!success) { + break; + } + + const auto internal_idx = + shard_internal_index(chunk_idx, config_.dimensions); + chunk_table.at(2 * internal_idx) = *file_offset; + chunk_table.at(2 * internal_idx + 1) = chunk.size(); + + *file_offset += chunk.size(); + } + + if (success && write_table) { + const auto* table = + reinterpret_cast(chunk_table.data()); + success = + sink->write(*file_offset, + table, + chunk_table.size() * sizeof(uint64_t)); + } + } catch (const std::exception& exc) { + err = "Failed to write chunk: " + std::string(exc.what()); + } catch (...) { + err = "Failed to write chunk: (unknown)"; + } + + latch.count_down(); + return success; + }), + "Failed to push job to thread pool"); + } + + // wait for all threads to finish + latch.wait(); + + // reset shard tables and file offsets + if (write_table) { + for (auto& table : shard_tables_) { + std::fill_n(table.begin(), + table.size(), + std::numeric_limits::max()); + } + + std::fill_n(shard_file_offsets_.begin(), shard_file_offsets_.size(), 0); + } + + return true; +} + +bool +zarr::ZarrV3ArrayWriter::write_array_metadata_() +{ + if (!make_metadata_sink_()) { + return false; + } + + using json = nlohmann::json; + + std::vector array_shape, chunk_shape, shard_shape; + + size_t append_size = frames_written_; + for (auto dim = config_.dimensions.rbegin() + 2; + dim < config_.dimensions.rend() - 1; + ++dim) { + CHECK(dim->array_size_px); + append_size = (append_size + dim->array_size_px - 1) / dim->array_size_px; + } + array_shape.push_back(append_size); + + chunk_shape.push_back(config_.dimensions.front().chunk_size_px); + shard_shape.push_back(config_.dimensions.front().shard_size_chunks); + for (auto dim = config_.dimensions.begin() + 1; + dim != config_.dimensions.end(); + ++dim) { + array_shape.push_back(dim->array_size_px); + chunk_shape.push_back(dim->chunk_size_px); + shard_shape.push_back(dim->shard_size_chunks); + } + + json metadata; + metadata["attributes"] = json::object(); + metadata["chunk_grid"] = json::object({ + { "chunk_shape", chunk_shape }, + { "separator", "/" }, + { "type", "regular" }, + }); + + metadata["chunk_memory_layout"] = "C"; + metadata["data_type"] = sample_type_to_dtype(config_.dtype); + metadata["extensions"] = json::array(); + metadata["fill_value"] = 0; + metadata["shape"] = array_shape; + + if (config_.compression_params) { + const auto params = *config_.compression_params; + metadata["compressor"] = json::object({ + { "codec", "https://purl.org/zarr/spec/codec/blosc/1.0" }, + { "configuration", + json::object({ + { "blocksize", 0 }, + { "clevel", params.clevel }, + { "cname", params.codec_id }, + { "shuffle", params.shuffle }, + }) }, + }); + } else { + metadata["compressor"] = nullptr; + } + + // sharding storage transformer + // TODO (aliddell): + // https://github.com/zarr-developers/zarr-python/issues/877 + metadata["storage_transformers"] = json::array(); + metadata["storage_transformers"][0] = json::object({ + { "type", "indexed" }, + { "extension", + "https://purl.org/zarr/spec/storage_transformers/sharding/1.0" }, + { "configuration", + json::object({ + { "chunks_per_shard", shard_shape }, + }) }, + }); + + const std::string metadata_str = metadata.dump(4); + const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); + + return metadata_sink_->write(0, metadata_bytes, metadata_str.size()); +} + +bool +zarr::ZarrV3ArrayWriter::should_rollover_() const +{ + const auto& dims = config_.dimensions; + const auto& append_dim = dims.front(); + size_t frames_before_flush = + append_dim.chunk_size_px * append_dim.shard_size_chunks; + for (auto i = 1; i < dims.size() - 2; ++i) { + frames_before_flush *= dims[i].array_size_px; + } + + CHECK(frames_before_flush > 0); + return frames_written_ % frames_before_flush == 0; +} diff --git a/src/streaming/zarrv3.array.writer.hh b/src/streaming/zarrv3.array.writer.hh new file mode 100644 index 00000000..9a123050 --- /dev/null +++ b/src/streaming/zarrv3.array.writer.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "array.writer.hh" + +namespace zarr { +struct ZarrV3ArrayWriter final : public ArrayWriter +{ + public: + ZarrV3ArrayWriter( + const ArrayWriterConfig& array_spec, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool); + + ~ZarrV3ArrayWriter() override; + + private: + std::vector shard_file_offsets_; + std::vector> shard_tables_; + + ZarrVersion version_() const override; + bool flush_impl_() override; + bool write_array_metadata_() override; + bool should_rollover_() const override; +}; +} // namespace zarr diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index 52adb1bc..f195f222 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -22,6 +22,9 @@ set(tests zarrv2-writer-write-even zarrv2-writer-write-ragged-append-dim zarrv2-writer-write-ragged-internal-dim + zarrv3-writer-write-even + zarrv3-writer-write-ragged-append-dim + zarrv3-writer-write-ragged-internal-dim ) foreach (name ${tests}) diff --git a/tests/unit-tests/zarrv3-writer-write-even.cpp b/tests/unit-tests/zarrv3-writer-write-even.cpp new file mode 100644 index 00000000..d3414780 --- /dev/null +++ b/tests/unit-tests/zarrv3-writer-write-even.cpp @@ -0,0 +1,221 @@ +#include "zarrv3.array.writer.hh" +#include "macros.hh" +#include "zarr.common.hh" + +#include + +#include + +#define EXPECT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +namespace fs = std::filesystem; + +namespace { +const fs::path base_dir = fs::temp_directory_path() / TEST; + +const unsigned int array_width = 64, array_height = 48, array_planes = 6, + array_channels = 8, array_timepoints = 10; +const unsigned int n_frames = array_planes * array_channels * array_timepoints; + +const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, + chunk_channels = 4, chunk_timepoints = 5; + +const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, + shard_channels = 2, shard_timepoints = 2; +const unsigned int chunks_per_shard = + shard_width * shard_height * shard_planes * shard_channels * shard_timepoints; + +const unsigned int chunks_in_x = + (array_width + chunk_width - 1) / chunk_width; // 4 chunks +const unsigned int chunks_in_y = + (array_height + chunk_height - 1) / chunk_height; // 3 chunks +const unsigned int chunks_in_z = + (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks +const unsigned int chunks_in_c = + (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks +const unsigned int chunks_in_t = + (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; + +const unsigned int shards_in_x = + (chunks_in_x + shard_width - 1) / shard_width; // 2 shards +const unsigned int shards_in_y = + (chunks_in_y + shard_height - 1) / shard_height; // 3 shards +const unsigned int shards_in_z = + (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards +const unsigned int shards_in_c = + (chunks_in_c + shard_channels - 1) / shard_channels; // 1 shard +const unsigned int shards_in_t = + (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard + +const int level_of_detail = 3; +} // namespace + +void +check_json() +{ + fs::path meta_path = base_dir / "meta" / "root" / + (std::to_string(level_of_detail) + ".array.json"); + CHECK(fs::is_regular_file(meta_path)); + + std::ifstream f(meta_path); + nlohmann::json meta = nlohmann::json::parse(f); + + EXPECT(meta["data_type"].get() == "uint16", + "Expected dtype to be 'uint16', but got '%s'", + meta["data_type"].get().c_str()); + + const auto& array_shape = meta["shape"]; + const auto& chunk_shape = meta["chunk_grid"]["chunk_shape"]; + const auto& shard_shape = + meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; + + EXPECT_EQ(array_shape.size(), 5); + EXPECT_EQ(array_shape[0].get(), array_timepoints); + EXPECT_EQ(array_shape[1].get(), array_channels); + EXPECT_EQ(array_shape[2].get(), array_planes); + EXPECT_EQ(array_shape[3].get(), array_height); + EXPECT_EQ(array_shape[4].get(), array_width); + + EXPECT_EQ(chunk_shape.size(), 5); + EXPECT_EQ(chunk_shape[0].get(), chunk_timepoints); + EXPECT_EQ(chunk_shape[1].get(), chunk_channels); + EXPECT_EQ(chunk_shape[2].get(), chunk_planes); + EXPECT_EQ(chunk_shape[3].get(), chunk_height); + EXPECT_EQ(chunk_shape[4].get(), chunk_width); + + EXPECT_EQ(shard_shape.size(), 5); + EXPECT_EQ(shard_shape[0].get(), shard_timepoints); + EXPECT_EQ(shard_shape[1].get(), shard_channels); + EXPECT_EQ(shard_shape[2].get(), shard_planes); + EXPECT_EQ(shard_shape[3].get(), shard_height); + EXPECT_EQ(shard_shape[4].get(), shard_width); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + int retval = 1; + + const ZarrDataType dtype = ZarrDataType_uint16; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); + + std::vector dims; + dims.emplace_back("t", + ZarrDimensionType_Time, + array_timepoints, + chunk_timepoints, + shard_timepoints); + dims.emplace_back("c", + ZarrDimensionType_Channel, + array_channels, + chunk_channels, + shard_channels); + dims.emplace_back("z", + ZarrDimensionType_Space, + array_planes, + chunk_planes, + shard_planes); + dims.emplace_back("y", + ZarrDimensionType_Space, + array_height, + chunk_height, + shard_height); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); + + zarr::ArrayWriterConfig config = { + .dimensions = dims, + .dtype = dtype, + .level_of_detail = level_of_detail, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + { + zarr::ZarrV3ArrayWriter writer(config, thread_pool, nullptr); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data(frame_size, 0); + + for (auto i = 0; i < n_frames; ++i) { + CHECK(writer.write_frame(data.data(), frame_size)); + } + } + + check_json(); + + const auto chunk_size = chunk_width * chunk_height * chunk_planes * + chunk_channels * chunk_timepoints * nbytes_px; + const auto index_size = chunks_per_shard * + sizeof(uint64_t) * // indices are 64 bits + 2; // 2 indices per chunk + const auto expected_file_size = shard_width * shard_height * + shard_planes * shard_channels * + shard_timepoints * chunk_size + + index_size; + + const fs::path data_root = + base_dir / "data/root" / std::to_string(config.level_of_detail); + CHECK(fs::is_directory(data_root)); + for (auto t = 0; t < shards_in_t; ++t) { + const auto t_dir = data_root / ("c" + std::to_string(t)); + CHECK(fs::is_directory(t_dir)); + + for (auto c = 0; c < shards_in_c; ++c) { + const auto c_dir = t_dir / std::to_string(c); + CHECK(fs::is_directory(c_dir)); + + for (auto z = 0; z < shards_in_z; ++z) { + const auto z_dir = c_dir / std::to_string(z); + CHECK(fs::is_directory(z_dir)); + + for (auto y = 0; y < shards_in_y; ++y) { + const auto y_dir = z_dir / std::to_string(y); + CHECK(fs::is_directory(y_dir)); + + for (auto x = 0; x < shards_in_x; ++x) { + const auto x_file = y_dir / std::to_string(x); + CHECK(fs::is_regular_file(x_file)); + const auto file_size = fs::file_size(x_file); + EXPECT_EQ(file_size, expected_file_size); + } + + CHECK(!fs::is_regular_file( + y_dir / std::to_string(shards_in_x))); + } + + CHECK( + !fs::is_directory(z_dir / std::to_string(shards_in_y))); + } + + CHECK(!fs::is_directory(c_dir / std::to_string(shards_in_z))); + } + + CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_c))); + } + + CHECK( + !fs::is_directory(data_root / ("c" + std::to_string(shards_in_t)))); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp new file mode 100644 index 00000000..45959819 --- /dev/null +++ b/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp @@ -0,0 +1,177 @@ +#include "zarrv3.array.writer.hh" +#include "macros.hh" +#include "zarr.common.hh" + +#include + +#include + +#define EXPECT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +namespace fs = std::filesystem; + +namespace { +const fs::path base_dir = fs::temp_directory_path() / TEST; + +const unsigned int array_width = 64, array_height = 48, array_planes = 5; +const unsigned int n_frames = array_planes; + +const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2; + +const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1; +const unsigned int chunks_per_shard = shard_width * shard_height * shard_planes; + +const unsigned int chunks_in_x = + (array_width + chunk_width - 1) / chunk_width; // 4 chunks +const unsigned int chunks_in_y = + (array_height + chunk_height - 1) / chunk_height; // 3 chunks +const unsigned int chunks_in_z = + (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks + +const unsigned int shards_in_x = + (chunks_in_x + shard_width - 1) / shard_width; // 2 shards +const unsigned int shards_in_y = + (chunks_in_y + shard_height - 1) / shard_height; // 3 shards +const unsigned int shards_in_z = + (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards + +const int level_of_detail = 4; +} // namespace + +void +check_json() +{ + fs::path meta_path = base_dir / "meta" / "root" / + (std::to_string(level_of_detail) + ".array.json"); + CHECK(fs::is_regular_file(meta_path)); + + std::ifstream f(meta_path); + nlohmann::json meta = nlohmann::json::parse(f); + + EXPECT(meta["data_type"].get() == "int32", + "Expected dtype to be 'int32', but got '%s'", + meta["data_type"].get().c_str()); + + const auto& array_shape = meta["shape"]; + const auto& chunk_shape = meta["chunk_grid"]["chunk_shape"]; + const auto& shard_shape = + meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; + + EXPECT_EQ(array_shape.size(), 3); + EXPECT_EQ(array_shape[0].get(), array_planes); + EXPECT_EQ(array_shape[1].get(), array_height); + EXPECT_EQ(array_shape[2].get(), array_width); + + EXPECT_EQ(chunk_shape.size(), 3); + EXPECT_EQ(chunk_shape[0].get(), chunk_planes); + EXPECT_EQ(chunk_shape[1].get(), chunk_height); + EXPECT_EQ(chunk_shape[2].get(), chunk_width); + + EXPECT_EQ(shard_shape.size(), 3); + EXPECT_EQ(shard_shape[0].get(), shard_planes); + EXPECT_EQ(shard_shape[1].get(), shard_height); + EXPECT_EQ(shard_shape[2].get(), shard_width); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + int retval = 1; + + const ZarrDataType dtype = ZarrDataType_int32; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); + + std::vector dims; + dims.emplace_back("z", + ZarrDimensionType_Space, + array_planes, + chunk_planes, + shard_planes); + dims.emplace_back("y", + ZarrDimensionType_Space, + array_height, + chunk_height, + shard_height); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); + + zarr::ArrayWriterConfig config = { + .dimensions = dims, + .dtype = dtype, + .level_of_detail = 4, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + { + zarr::ZarrV3ArrayWriter writer(config, thread_pool, nullptr); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data(frame_size, 0); + + for (auto i = 0; i < n_frames; ++i) { + CHECK(writer.write_frame(data.data(), frame_size) == + frame_size); + } + } + + check_json(); + + const auto chunk_size = + chunk_width * chunk_height * chunk_planes * nbytes_px; + const auto index_size = chunks_per_shard * + sizeof(uint64_t) * // indices are 64 bits + 2; // 2 indices per chunk + const auto expected_file_size = + shard_width * shard_height * shard_planes * chunk_size + index_size; + + const fs::path data_root = + base_dir / "data/root" / std::to_string(config.level_of_detail); + CHECK(fs::is_directory(data_root)); + for (auto z = 0; z < shards_in_z; ++z) { + const auto z_dir = data_root / ("c" + std::to_string(z)); + CHECK(fs::is_directory(z_dir)); + + for (auto y = 0; y < shards_in_y; ++y) { + const auto y_dir = z_dir / std::to_string(y); + CHECK(fs::is_directory(y_dir)); + + for (auto x = 0; x < shards_in_x; ++x) { + const auto x_file = y_dir / std::to_string(x); + CHECK(fs::is_regular_file(x_file)); + const auto file_size = fs::file_size(x_file); + EXPECT_EQ(file_size, expected_file_size); + } + + CHECK( + !fs::is_regular_file(y_dir / std::to_string(shards_in_x))); + } + + CHECK(!fs::is_directory(z_dir / std::to_string(shards_in_y))); + } + + CHECK( + !fs::is_directory(data_root / ("c" + std::to_string(shards_in_z)))); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp new file mode 100644 index 00000000..2083775f --- /dev/null +++ b/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp @@ -0,0 +1,200 @@ +#include "zarrv3.array.writer.hh" +#include "macros.hh" +#include "zarr.common.hh" + +#include + +#include + +#define EXPECT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +namespace fs = std::filesystem; + +namespace { +const fs::path base_dir = fs::temp_directory_path() / TEST; + +const unsigned int array_width = 64, array_height = 48, array_planes = 5, + array_timepoints = 10; +const unsigned int n_frames = array_planes * array_timepoints; + +const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, + chunk_timepoints = 5; + +const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, + shard_timepoints = 2; +const unsigned int chunks_per_shard = + shard_width * shard_height * shard_planes * shard_timepoints; + +const unsigned int chunks_in_x = + (array_width + chunk_width - 1) / chunk_width; // 4 chunks +const unsigned int chunks_in_y = + (array_height + chunk_height - 1) / chunk_height; // 3 chunks +const unsigned int chunks_in_z = + (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged +const unsigned int chunks_in_t = + (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; + +const unsigned int shards_in_x = + (chunks_in_x + shard_width - 1) / shard_width; // 2 shards +const unsigned int shards_in_y = + (chunks_in_y + shard_height - 1) / shard_height; // 3 shards +const unsigned int shards_in_z = + (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards +const unsigned int shards_in_t = + (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard + +const int level_of_detail = 5; +} // namespace + +void +check_json() +{ + fs::path meta_path = base_dir / "meta" / "root" / + (std::to_string(level_of_detail) + ".array.json"); + CHECK(fs::is_regular_file(meta_path)); + + std::ifstream f(meta_path); + nlohmann::json meta = nlohmann::json::parse(f); + + EXPECT(meta["data_type"].get() == "float64", + "Expected dtype to be 'uint16', but got '%s'", + meta["data_type"].get().c_str()); + + const auto& array_shape = meta["shape"]; + const auto& chunk_shape = meta["chunk_grid"]["chunk_shape"]; + const auto& shard_shape = + meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; + + EXPECT_EQ(array_shape.size(), 4); + EXPECT_EQ(array_shape[0].get(), array_timepoints); + EXPECT_EQ(array_shape[1].get(), array_planes); + EXPECT_EQ(array_shape[2].get(), array_height); + EXPECT_EQ(array_shape[3].get(), array_width); + + EXPECT_EQ(chunk_shape.size(), 4); + EXPECT_EQ(chunk_shape[0].get(), chunk_timepoints); + EXPECT_EQ(chunk_shape[1].get(), chunk_planes); + EXPECT_EQ(chunk_shape[2].get(), chunk_height); + EXPECT_EQ(chunk_shape[3].get(), chunk_width); + + EXPECT_EQ(shard_shape.size(), 4); + EXPECT_EQ(shard_shape[0].get(), shard_timepoints); + EXPECT_EQ(shard_shape[1].get(), shard_planes); + EXPECT_EQ(shard_shape[2].get(), shard_height); + EXPECT_EQ(shard_shape[3].get(), shard_width); +} + +int +main() +{ + Logger::set_log_level(ZarrLogLevel_Debug); + + int retval = 1; + + const ZarrDataType dtype = ZarrDataType_float64; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); + + std::vector dims; + dims.emplace_back("t", + ZarrDimensionType_Time, + array_timepoints, + chunk_timepoints, + shard_timepoints); + dims.emplace_back("z", + ZarrDimensionType_Space, + array_planes, + chunk_planes, + shard_planes); + dims.emplace_back("y", + ZarrDimensionType_Space, + array_height, + chunk_height, + shard_height); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); + + zarr::ArrayWriterConfig config = { + .dimensions = dims, + .dtype = dtype, + .level_of_detail = 5, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + { + zarr::ZarrV3ArrayWriter writer(config, thread_pool, nullptr); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data(frame_size, 0); + + for (auto i = 0; i < n_frames; ++i) { + CHECK(writer.write_frame(data.data(), frame_size) == + frame_size); + } + } + + const auto chunk_size = chunk_width * chunk_height * chunk_planes * + chunk_timepoints * nbytes_px; + const auto index_size = chunks_per_shard * + sizeof(uint64_t) * // indices are 64 bits + 2; // 2 indices per chunk + const auto expected_file_size = shard_width * shard_height * + shard_planes * shard_timepoints * + chunk_size + + index_size; + + const fs::path data_root = + base_dir / "data/root" / std::to_string(config.level_of_detail); + CHECK(fs::is_directory(data_root)); + for (auto t = 0; t < shards_in_t; ++t) { + const auto t_dir = data_root / ("c" + std::to_string(t)); + CHECK(fs::is_directory(t_dir)); + + for (auto z = 0; z < shards_in_z; ++z) { + const auto z_dir = t_dir / std::to_string(z); + CHECK(fs::is_directory(z_dir)); + + for (auto y = 0; y < shards_in_y; ++y) { + const auto y_dir = z_dir / std::to_string(y); + CHECK(fs::is_directory(y_dir)); + + for (auto x = 0; x < shards_in_x; ++x) { + const auto x_file = y_dir / std::to_string(x); + CHECK(fs::is_regular_file(x_file)); + const auto file_size = fs::file_size(x_file); + CHECK(file_size == expected_file_size); + } + + CHECK(!fs::is_regular_file(y_dir / + std::to_string(shards_in_x))); + } + + CHECK(!fs::is_directory(z_dir / std::to_string(shards_in_y))); + } + + CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_z))); + } + + CHECK( + !fs::is_directory(data_root / ("c" + std::to_string(shards_in_t)))); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file From 996d22e9e276f0225082a7304c54adf935cc0cf9 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 18 Sep 2024 11:34:33 -0400 Subject: [PATCH 20/39] Document that ZarrStream_append will block for compression and flushing --- include/acquire.zarr.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index e2a33051..eb287663 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -239,6 +239,8 @@ extern "C" /** * @brief Append data to the Zarr stream. + * @details This function will block while chunks are compressed and written + * to the store. It will return when all data has been written. * @param[in, out] stream The Zarr stream struct. * @param[in] data The data to append. * @param[in] bytes_in The number of bytes in @p data. It should be at least From e546f7d23e849d94e45ea03b8895421d89fb5abd Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 18 Sep 2024 15:38:03 -0400 Subject: [PATCH 21/39] Respond to PR comments. --- CMakeLists.txt | 3 + src/CMakeLists.txt | 6 +- src/driver/CMakeLists.txt | 110 +++++++++++++------------- tests/CMakeLists.txt | 4 +- tests/driver/CMakeLists.txt | 154 ++++++++++++++++++------------------ 5 files changed, 138 insertions(+), 139 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2925e3f8..0419a48b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,9 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +option(NOTEST "Disable all tests" OFF) +option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) + add_subdirectory(src) add_subdirectory(tests) add_subdirectory(examples) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9a3b42a..31635d84 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,3 @@ -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - -add_subdirectory(driver) +if (BUILD_ACQUIRE_DRIVER_ZARR) + add_subdirectory(driver) +endif () diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt index e223b5db..5b72798a 100644 --- a/src/driver/CMakeLists.txt +++ b/src/driver/CMakeLists.txt @@ -1,61 +1,57 @@ -option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) +if (NOT TARGET acquire-core-logger) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) +endif () -if (BUILD_ACQUIRE_DRIVER_ZARR) - if (NOT TARGET acquire-core-logger) - add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) - endif () +set(tgt acquire-driver-zarr) +add_library(${tgt} MODULE + common/dimension.hh + common/dimension.cpp + common/thread.pool.hh + common/thread.pool.cpp + common/s3.connection.hh + common/s3.connection.cpp + common/utilities.hh + common/utilities.cpp + writers/sink.hh + writers/sink.creator.hh + writers/sink.creator.cpp + writers/file.sink.hh + writers/file.sink.cpp + writers/s3.sink.hh + writers/s3.sink.cpp + writers/array.writer.hh + writers/array.writer.cpp + writers/zarrv2.array.writer.hh + writers/zarrv2.array.writer.cpp + writers/zarrv3.array.writer.hh + writers/zarrv3.array.writer.cpp + writers/blosc.compressor.hh + writers/blosc.compressor.cpp + zarr.hh + zarr.cpp + zarr.v2.hh + zarr.v2.cpp + zarr.v3.hh + zarr.v3.cpp + zarr.driver.c +) - set(tgt acquire-driver-zarr) - add_library(${tgt} MODULE - common/dimension.hh - common/dimension.cpp - common/thread.pool.hh - common/thread.pool.cpp - common/s3.connection.hh - common/s3.connection.cpp - common/utilities.hh - common/utilities.cpp - writers/sink.hh - writers/sink.creator.hh - writers/sink.creator.cpp - writers/file.sink.hh - writers/file.sink.cpp - writers/s3.sink.hh - writers/s3.sink.cpp - writers/array.writer.hh - writers/array.writer.cpp - writers/zarrv2.array.writer.hh - writers/zarrv2.array.writer.cpp - writers/zarrv3.array.writer.hh - writers/zarrv3.array.writer.cpp - writers/blosc.compressor.hh - writers/blosc.compressor.cpp - zarr.hh - zarr.cpp - zarr.v2.hh - zarr.v2.cpp - zarr.v3.hh - zarr.v3.cpp - zarr.driver.c - ) +target_include_directories(${tgt} PRIVATE + $ +) - target_include_directories(${tgt} PRIVATE - $ - ) +target_enable_simd(${tgt}) +target_link_libraries(${tgt} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp +) +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) - target_enable_simd(${tgt}) - target_link_libraries(${tgt} PRIVATE - acquire-core-logger - acquire-core-platform - acquire-device-kit - acquire-device-properties - blosc_static - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - - install(TARGETS ${tgt} LIBRARY DESTINATION lib) -endif () \ No newline at end of file +install(TARGETS ${tgt} LIBRARY DESTINATION lib) \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 87878b37..d10a9f44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,7 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () - add_subdirectory(driver) + if (BUILD_ACQUIRE_DRIVER_ZARR) + add_subdirectory(driver) + endif () endif () diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt index 1fd661a2..b2ea8266 100644 --- a/tests/driver/CMakeLists.txt +++ b/tests/driver/CMakeLists.txt @@ -1,87 +1,85 @@ -if (BUILD_ACQUIRE_DRIVER_ZARR) - set(NOTEST "TRUE") - add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) - set(NOTEST "FALSE") +set(NOTEST "TRUE") +add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) +set(NOTEST "FALSE") - # - # PARAMETERS - # - set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project +# +# PARAMETERS +# +set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project - # - # Tests - # - set(tests - list-devices - unit-tests - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 - ) +# +# Tests +# +set(tests + list-devices + unit-tests + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 +) - foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) +foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") - endforeach () + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") +endforeach () - # - # Copy driver to tests - # - list(POP_FRONT tests onename) +# +# Copy driver to tests +# +list(POP_FRONT tests onename) - foreach (driver - acquire-driver-common - acquire-driver-zarr +foreach (driver + acquire-driver-common + acquire-driver-zarr +) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" ) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" - ) - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) - endforeach () + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) endforeach () -endif () \ No newline at end of file +endforeach () \ No newline at end of file From 42a494055bf9a835d12dd944b7bfca3135cb1134 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 09:45:17 -0400 Subject: [PATCH 22/39] Respond to PR comments. --- include/acquire.zarr.h | 223 ++++------------------------------------- include/zarr.types.h | 10 +- 2 files changed, 20 insertions(+), 213 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index eb287663..cbbbaf86 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -1,5 +1,4 @@ -#ifndef H_ACQUIRE_ZARR_V0 -#define H_ACQUIRE_ZARR_V0 +#pragma once #include "zarr.types.h" @@ -8,7 +7,19 @@ extern "C" { #endif - typedef struct ZarrStreamSettings_s ZarrStreamSettings; + typedef struct ZarrStreamSettings_s + { + ZarrS3Settings s3_settings; + ZarrCompressionSettings compression_settings; + ZarrDimensionProperties* dimensions; + size_t dimension_count; + char* store_path; + char* custom_metadata; + ZarrDataType data_type; + ZarrVersion version; + bool multiscale; + } ZarrStreamSettings; + typedef struct ZarrStream_s ZarrStream; /** @@ -22,7 +33,7 @@ extern "C" * @param level The log level. * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + ZarrStatusCode Zarr_set_log_level(ZarrLogLevel level); /** * @brief Get the log level for the Zarr API. @@ -35,191 +46,7 @@ extern "C" * @param status The status code. * @return A human-readable status message. */ - const char* Zarr_get_error_message(ZarrStatus status); - - /** - * @brief Create a Zarr stream settings struct. - * @return A pointer to the Zarr stream settings struct, or NULL on failure. - */ - ZarrStreamSettings* ZarrStreamSettings_create(); - - /** - * @brief Destroy a Zarr stream settings struct. - * @details This function frees the memory allocated for the Zarr stream - * settings struct. - * @param[in] settings The Zarr stream settings struct. - */ - void ZarrStreamSettings_destroy(ZarrStreamSettings* settings); - - /** - * @brief Copy a Zarr stream settings struct. - * @param[in] settings The Zarr stream settings struct to copy. - * @return A copy of the Zarr stream settings struct. - */ - ZarrStreamSettings* ZarrStreamSettings_copy( - const ZarrStreamSettings* settings); - - /** - * @brief Set store path and S3 settings for the Zarr stream. - * @param[in, out] settings - * @param[in] store_path The store path for the Zarr stream. Directory path - * when acquiring to the filesystem, key prefix when acquiring to S3. - * @param[in] bytes_of_store_path The length of @p store_path in bytes, - * including the null terminator. - * @param[in] s3_settings Optional S3 settings. If NULL, the store path is - * assumed to be a directory path. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, - const char* store_path, - size_t bytes_of_store_path, - const ZarrS3Settings* s3_settings); - - /** - * @brief Set the data type, compressor, codec, compression_settings level, - * and shuffle for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] compression_settings The compression_settings settings. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_compression( - ZarrStreamSettings* settings, - const ZarrCompressionSettings* compression_settings); - - /** - * @brief Set the data type for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] data_type The data type. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, - ZarrDataType data_type); - - /** - * @brief Reserve space for dimensions in the Zarr stream settings struct. - * @detail *Must* precede calls to ZarrStreamSettings_set_dimension. We - * require at least 3 dimensions to validate settings, but you may set up to - * 32 dimensions. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] count The number of dimensions to reserve space for. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_reserve_dimensions( - ZarrStreamSettings* settings, - size_t count); - - /** - * @brief Set properties for an acquisition dimension. - * @detail The order of the dimensions in the Zarr stream is the order in - * which they are set. The first dimension set is the slowest varying - * dimension, and the last dimension set is the fastest varying dimension. - * For example, if the dimensions are set in the order z, y, x, the fastest - * varying dimension is x, the next fastest varying dimension is y, and the - * slowest varying dimension is z. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] index The index of the dimension to set. Must be less than the - * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. - * @param[in] dimension The dimension's settings. - */ - ZarrStatus ZarrStreamSettings_set_dimension( - ZarrStreamSettings* settings, - size_t index, - const ZarrDimensionProperties* dimension); - - /** - * @brief Set the multiscale flag for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] multiscale A flag indicating whether to stream to multiple - * levels of detail. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, - uint8_t multiscale); - - /** - * @brief Set JSON-formatted custom metadata for the Zarr stream. - * @details This metadata will be written to acquire-zarr.json in the - * metadata directory of the Zarr store. This parameter is optional. - * @param settings[in, out] settings The Zarr stream settings struct. - * @param external_metadata JSON-formatted external metadata. - * @param bytes_of_external_metadata The length of @p custom_metadata in - * bytes, including the null terminator. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_custom_metadata( - ZarrStreamSettings* settings, - const char* external_metadata, - size_t bytes_of_external_metadata); - - /** - * @brief Get the store path for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The store path for the Zarr stream, or NULL on failure. - */ - const char* ZarrStreamSettings_get_store_path( - const ZarrStreamSettings* settings); - - /** - * @brief Get the S3 settings for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The S3 settings for the Zarr stream, or an uninitialized struct on - * failure. - */ - ZarrS3Settings ZarrStreamSettings_get_s3_settings( - const ZarrStreamSettings* settings); - - /** - * @brief Get the compression settings for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The compression settings for the Zarr stream, or an uninitialized - * struct on failure. - */ - ZarrCompressionSettings ZarrStreamSettings_get_compression( - const ZarrStreamSettings* settings); - - /** - * @brief Get the data type for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The data type for the Zarr stream, or ZarrDataType_uint8 on failure. - */ - ZarrDataType ZarrStreamSettings_get_data_type( - const ZarrStreamSettings* settings); - - /** - * @brief Get the number of dimensions in the Zarr stream settings struct. - * @param settings The Zarr stream settings struct. - * @return The number of dimensions in the Zarr stream settings struct, or 0 on - * failure. - */ - size_t ZarrStreamSettings_get_dimension_count( - const ZarrStreamSettings* settings); - - /** - * @brief Get the properties for an acquisition dimension. - * @param settings The Zarr stream settings struct. - * @param index The index of the dimension to get. - * @return The properties for the @p index th dimension, or an uninitialized struct on - * failure. - */ - ZarrDimensionProperties ZarrStreamSettings_get_dimension( - const ZarrStreamSettings* settings, - size_t index); - - /** - * @brief Get the multiscale flag for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The multiscale flag for the Zarr stream, or false on failure. - */ - bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); - - /** - * @brief Get the JSON-formatted custom metadata for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The JSON-formatted custom metadata for the Zarr stream, or NULL on - * failure. - */ - const char* ZarrStreamSettings_get_custom_metadata( - const ZarrStreamSettings* settings); + const char* Zarr_get_status_message(ZarrStatusCode status); /** * @brief Create a Zarr stream. @@ -248,27 +75,11 @@ extern "C" * @param[out] bytes_out The number of bytes written to the stream. * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrStatus ZarrStream_append(ZarrStream* stream, + ZarrStatusCode ZarrStream_append(ZarrStream* stream, const void* data, size_t bytes_in, size_t* bytes_out); - /** - * @brief Get the version (i.e., 2 or 3) of the Zarr stream. - * @param stream The Zarr stream struct. - * @return The version of the Zarr stream. - */ - ZarrVersion ZarrStream_get_version(const ZarrStream* stream); - - /** - * @brief Get a copy of the settings for the Zarr stream. - * @param stream The Zarr stream struct. - * @return A copy of the settings for the Zarr stream. - */ - ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); - #ifdef __cplusplus } #endif - -#endif // H_ACQUIRE_ZARR_V0 diff --git a/include/zarr.types.h b/include/zarr.types.h index 56a54a56..2dfd0dd1 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -23,7 +23,7 @@ extern "C" ZarrStatus_CompressionError, ZarrStatus_InvalidSettings, ZarrStatusCount, - } ZarrStatus; + } ZarrStatusCode; typedef enum { @@ -34,7 +34,7 @@ extern "C" typedef enum { - ZarrLogLevel_Debug, + ZarrLogLevel_Debug = 0, ZarrLogLevel_Info, ZarrLogLevel_Warning, ZarrLogLevel_Error, @@ -44,7 +44,7 @@ extern "C" typedef enum { - ZarrDataType_uint8, + ZarrDataType_uint8 = 0, ZarrDataType_uint16, ZarrDataType_uint32, ZarrDataType_uint64, @@ -87,13 +87,9 @@ extern "C" typedef struct { const char* endpoint; - size_t bytes_of_endpoint; const char* bucket_name; - size_t bytes_of_bucket_name; const char* access_key_id; - size_t bytes_of_access_key_id; const char* secret_access_key; - size_t bytes_of_secret_access_key; } ZarrS3Settings; /** From 2853ad1d3dd159240f9c1a2f286305ee155d1b01 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 10:52:28 -0400 Subject: [PATCH 23/39] wip --- include/acquire.zarr.h | 8 +- src/streaming/CMakeLists.txt | 4 +- src/streaming/acquire.zarr.cpp | 98 ++++++++ src/streaming/zarr.api.cpp | 56 ----- src/streaming/zarr.stream.cpp | 86 +------ src/streaming/zarr.stream.hh | 25 +- tests/unit-tests/CMakeLists.txt | 3 - tests/unit-tests/get-stream-parameters.cpp | 235 ------------------ tests/unit-tests/set-stream-parameters.cpp | 273 --------------------- tests/unit-tests/stream-get-settings.cpp | 124 ---------- 10 files changed, 135 insertions(+), 777 deletions(-) create mode 100644 src/streaming/acquire.zarr.cpp delete mode 100644 src/streaming/zarr.api.cpp delete mode 100644 tests/unit-tests/get-stream-parameters.cpp delete mode 100644 tests/unit-tests/set-stream-parameters.cpp delete mode 100644 tests/unit-tests/stream-get-settings.cpp diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index cbbbaf86..e7fa57c9 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -11,13 +11,13 @@ extern "C" { ZarrS3Settings s3_settings; ZarrCompressionSettings compression_settings; - ZarrDimensionProperties* dimensions; - size_t dimension_count; - char* store_path; + ZarrDimensionProperties* dimensions; /**< The properties of each dimension in the dataset. */ + size_t dimension_count; /**< The number of dimensions in the dataset. */ + char* store_path; /**< Path to the store. Filesystem path or S3 key prefix. */ char* custom_metadata; + bool multiscale; ZarrDataType data_type; ZarrVersion version; - bool multiscale; } ZarrStreamSettings; typedef struct ZarrStream_s ZarrStream; diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 8feacc77..4085b738 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -2,9 +2,7 @@ set(tgt acquire-zarr) add_library(${tgt} macros.hh - zarr.api.cpp - stream.settings.hh - stream.settings.cpp + acquire.zarr.cpp zarr.stream.hh zarr.stream.cpp ) diff --git a/src/streaming/acquire.zarr.cpp b/src/streaming/acquire.zarr.cpp new file mode 100644 index 00000000..9b6cff9d --- /dev/null +++ b/src/streaming/acquire.zarr.cpp @@ -0,0 +1,98 @@ +#include "zarr.types.h" +#include "zarr.stream.hh" +#include "macros.hh" + +#include // uint32_t + +#define ACQUIRE_ZARR_API_VERSION 0 + +extern "C" +{ + uint32_t Zarr_get_api_version() + { + return ACQUIRE_ZARR_API_VERSION; + } + + ZarrStatusCode Zarr_set_log_level(ZarrLogLevel level) + { + EXPECT_VALID_ARGUMENT( + level < ZarrLogLevelCount, "Invalid log level: %d", level); + + Logger::set_log_level(level); + return ZarrStatus_Success; + } + + ZarrLogLevel Zarr_get_log_level() + { + return Logger::get_log_level(); + } + + const char* Zarr_get_status_message(ZarrStatusCode error) + { + switch (error) { + case ZarrStatus_Success: + return "Success"; + case ZarrStatus_InvalidArgument: + return "Invalid argument"; + case ZarrStatus_Overflow: + return "Buffer overflow"; + case ZarrStatus_InvalidIndex: + return "Invalid index"; + case ZarrStatus_NotYetImplemented: + return "Not yet implemented"; + case ZarrStatus_InternalError: + return "Internal error"; + case ZarrStatus_OutOfMemory: + return "Out of memory"; + case ZarrStatus_IOError: + return "I/O error"; + case ZarrStatus_CompressionError: + return "Compression error"; + case ZarrStatus_InvalidSettings: + return "Invalid settings"; + default: + return "Unknown error"; + } + } + + ZarrStream_s* ZarrStream_create(struct ZarrStreamSettings_s* settings, + ZarrVersion version) + { + + ZarrStream_s* stream = nullptr; + + try { + stream = new ZarrStream_s(settings, version); + } catch (const std::bad_alloc&) { + LOG_ERROR("Failed to allocate memory for Zarr stream"); + } catch (const std::exception& e) { + LOG_ERROR("Error creating Zarr stream: %s", e.what()); + } + + return stream; + } + + void ZarrStream_destroy(struct ZarrStream_s* stream) + { + delete stream; + } + + ZarrStatusCode ZarrStream_append(struct ZarrStream_s* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out) + { + EXPECT_VALID_ARGUMENT(stream, "Null pointer: stream"); + EXPECT_VALID_ARGUMENT(data, "Null pointer: data"); + EXPECT_VALID_ARGUMENT(bytes_out, "Null pointer: bytes_out"); + + try { + *bytes_out = stream->append(data, bytes_in); + } catch (const std::exception& e) { + LOG_ERROR("Error appending data: %s", e.what()); + return ZarrStatus_InternalError; + } + + return ZarrStatus_Success; + } +} \ No newline at end of file diff --git a/src/streaming/zarr.api.cpp b/src/streaming/zarr.api.cpp deleted file mode 100644 index e399d9a9..00000000 --- a/src/streaming/zarr.api.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "zarr.types.h" -#include "macros.hh" - -#include // uint32_t - -#define ACQUIRE_ZARR_API_VERSION 0 - -extern "C" -{ - uint32_t Zarr_get_api_version() - { - return ACQUIRE_ZARR_API_VERSION; - } - - const char* Zarr_get_error_message(ZarrStatus error) - { - switch (error) { - case ZarrStatus_Success: - return "Success"; - case ZarrStatus_InvalidArgument: - return "Invalid argument"; - case ZarrStatus_Overflow: - return "Buffer overflow"; - case ZarrStatus_InvalidIndex: - return "Invalid index"; - case ZarrStatus_NotYetImplemented: - return "Not yet implemented"; - case ZarrStatus_InternalError: - return "Internal error"; - case ZarrStatus_OutOfMemory: - return "Out of memory"; - case ZarrStatus_IOError: - return "I/O error"; - case ZarrStatus_CompressionError: - return "Compression error"; - case ZarrStatus_InvalidSettings: - return "Invalid settings"; - default: - return "Unknown error"; - } - } - - ZarrStatus Zarr_set_log_level(ZarrLogLevel level) - { - EXPECT_VALID_ARGUMENT( - level < ZarrLogLevelCount, "Invalid log level: %d", level); - - Logger::set_log_level(level); - return ZarrStatus_Success; - } - - ZarrLogLevel Zarr_get_log_level() - { - return Logger::get_log_level(); - } -} \ No newline at end of file diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp index 15f4128b..ba02ed2f 100644 --- a/src/streaming/zarr.stream.cpp +++ b/src/streaming/zarr.stream.cpp @@ -231,10 +231,13 @@ validate_settings(const struct ZarrStreamSettings_s* settings, ZarrStream::ZarrStream_s(struct ZarrStreamSettings_s* settings, ZarrVersion version) - : settings_(*settings) - , version_(version) + : version_(version) , error_() { + if (!validate_settings(settings, version)) { + throw std::runtime_error("Invalid Zarr stream settings"); + } + // create the data store EXPECT(create_store_(), "%s", error_.c_str()); @@ -286,14 +289,14 @@ ZarrStream_s::create_store_() if (is_s3_acquisition(&settings_)) { // TODO (aliddell): implement this } else { - if (fs::exists(settings_.store_path)) { + if (fs::exists(settings_.store_path_)) { // remove everything inside the store path std::error_code ec; - fs::remove_all(settings_.store_path, ec); + fs::remove_all(settings_.store_path_, ec); if (ec) { set_error_("Failed to remove existing store path '" + - settings_.store_path + "': " + ec.message()); + settings_.store_path_ + "': " + ec.message()); return false; } } @@ -301,9 +304,9 @@ ZarrStream_s::create_store_() // create the store path { std::error_code ec; - if (!fs::create_directories(settings_.store_path, ec)) { + if (!fs::create_directories(settings_.store_path_, ec)) { set_error_("Failed to create store path '" + - settings_.store_path + "': " + ec.message()); + settings_.store_path_ + "': " + ec.message()); return false; } } @@ -372,72 +375,3 @@ ZarrStream_s::write_multiscale_frames_(const uint8_t* data, // TODO (aliddell): implement this } - -// C API -extern "C" -{ - ZarrStream* ZarrStream_create(struct ZarrStreamSettings_s* settings, - ZarrVersion version) - { - if (!validate_settings(settings, version)) { - return nullptr; - } - - // initialize the stream - ZarrStream_s* stream = nullptr; - - try { - stream = new ZarrStream(settings, version); - } catch (const std::bad_alloc&) { - LOG_ERROR("Failed to allocate memory for Zarr stream"); - } catch (const std::exception& e) { - LOG_ERROR("Error creating Zarr stream: %s", e.what()); - } - - return stream; - } - - void ZarrStream_destroy(ZarrStream* stream) - { - delete stream; - } - - ZarrStatus ZarrStream_append(ZarrStream* stream, - const void* data, - size_t bytes_in, - size_t* bytes_out) - { - EXPECT_VALID_ARGUMENT(stream, "Null pointer: stream"); - EXPECT_VALID_ARGUMENT(data, "Null pointer: data"); - EXPECT_VALID_ARGUMENT(bytes_out, "Null pointer: bytes_out"); - - try { - *bytes_out = stream->append(data, bytes_in); - } catch (const std::exception& e) { - LOG_ERROR("Error appending data: %s", e.what()); - return ZarrStatus_InternalError; - } - - return ZarrStatus_Success; - } - - ZarrVersion ZarrStream_get_version(const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING("Null pointer: stream. Returning ZarrVersion_2"); - return ZarrVersion_2; - } - - return stream->version(); - } - - ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING("Null pointer: stream. Returning nullptr"); - return nullptr; - } - - return ZarrStreamSettings_copy(&stream->settings()); - } -} \ No newline at end of file diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index 0300bb90..e2812703 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -22,13 +22,32 @@ struct ZarrStream_s size_t append(const void* data, size_t nbytes); ZarrVersion version() const { return version_; } - const ZarrStreamSettings_s& settings() const { return settings_; } private: - struct ZarrStreamSettings_s settings_; - ZarrVersion version_; // Zarr version. 2 or 3. std::string error_; // error message. If nonempty, an error occurred. + ZarrVersion version_; + + std::string store_path_; + + std::string s3_endpoint; + std::string s3_bucket_name; + std::string s3_access_key_id; + std::string s3_secret_access_key; + + std::string custom_metadata; + + ZarrDataType dtype; + + ZarrCompressor compressor; + ZarrCompressionCodec compression_codec; + uint8_t compression_level; + uint8_t compression_shuffle; + + std::vector dimensions; + + bool multiscale; + /** * @brief Set an error message. * @param msg The error message to set. diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index 9e8090de..40f6a8d3 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -1,10 +1,7 @@ set(project acquire-zarr) set(tests - set-stream-parameters - get-stream-parameters create-stream - stream-get-settings ) foreach (name ${tests}) diff --git a/tests/unit-tests/get-stream-parameters.cpp b/tests/unit-tests/get-stream-parameters.cpp deleted file mode 100644 index 5dea2cf6..00000000 --- a/tests/unit-tests/get-stream-parameters.cpp +++ /dev/null @@ -1,235 +0,0 @@ -#include "acquire.zarr.h" -#include "stream.settings.hh" -#include "unit.test.macros.hh" - -void -check_preliminaries(ZarrStreamSettings* settings) -{ - CHECK(settings); - - CHECK(settings->store_path.empty()); - - CHECK(settings->s3_endpoint.empty()); - CHECK(settings->s3_bucket_name.empty()); - CHECK(settings->s3_access_key_id.empty()); - CHECK(settings->s3_secret_access_key.empty()); - - EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); - - EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint8); - - EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_None); - EXPECT_EQ( - int, "%d", settings->compression_codec, ZarrCompressionCodec_None); - EXPECT_EQ(int, "%d", settings->compression_level, 0); - EXPECT_EQ(int, "%d", settings->compression_shuffle, 0); - - CHECK(settings->dimensions.empty()); - - CHECK(!settings->multiscale); -} - -void -get_store_path(ZarrStreamSettings* settings) -{ - EXPECT_STR_EQ(ZarrStreamSettings_get_store_path(settings), ""); - - settings->store_path = TEST ".zarr"; - EXPECT_STR_EQ(ZarrStreamSettings_get_store_path(settings), - settings->store_path.c_str()); -} - -void -get_s3_settings(ZarrStreamSettings* settings) -{ - auto s3_settings = ZarrStreamSettings_get_s3_settings(settings); - EXPECT_STR_EQ(s3_settings.endpoint, ""); - EXPECT_EQ(int, "%d", s3_settings.bytes_of_endpoint, 1); - - EXPECT_STR_EQ(s3_settings.bucket_name, ""); - EXPECT_EQ(int, "%d", s3_settings.bytes_of_bucket_name, 1); - - EXPECT_STR_EQ(s3_settings.access_key_id, ""); - EXPECT_EQ(int, "%d", s3_settings.bytes_of_access_key_id, 1); - - EXPECT_STR_EQ(s3_settings.secret_access_key, ""); - EXPECT_EQ(int, "%d", s3_settings.bytes_of_secret_access_key, 1); - - settings->s3_endpoint = "https://s3.amazonaws.com"; - settings->s3_bucket_name = "bucket"; - settings->s3_access_key_id = "access_key"; - settings->s3_secret_access_key = "secret_access_key"; - - s3_settings = ZarrStreamSettings_get_s3_settings(settings); - EXPECT_STR_EQ(s3_settings.endpoint, settings->s3_endpoint.c_str()); - EXPECT_EQ(int, - "%d", - s3_settings.bytes_of_endpoint, - settings->s3_endpoint.size() + 1); - - EXPECT_STR_EQ(s3_settings.bucket_name, settings->s3_bucket_name.c_str()); - EXPECT_EQ(int, - "%d", - s3_settings.bytes_of_bucket_name, - settings->s3_bucket_name.size() + 1); - - EXPECT_STR_EQ(s3_settings.access_key_id, - settings->s3_access_key_id.c_str()); - EXPECT_EQ(int, - "%d", - s3_settings.bytes_of_access_key_id, - settings->s3_access_key_id.size() + 1); - - EXPECT_STR_EQ(s3_settings.secret_access_key, - settings->s3_secret_access_key.c_str()); - EXPECT_EQ(int, - "%d", - s3_settings.bytes_of_secret_access_key, - settings->s3_secret_access_key.size() + 1); -} - -void -get_compression(ZarrStreamSettings* settings) -{ - auto compression_settings = ZarrStreamSettings_get_compression(settings); - - EXPECT_EQ(int, "%d", compression_settings.compressor, ZarrCompressor_None); - EXPECT_EQ(int, "%d", compression_settings.codec, ZarrCompressionCodec_None); - EXPECT_EQ(int, "%d", compression_settings.level, 0); - EXPECT_EQ(int, "%d", compression_settings.shuffle, 0); - - settings->compressor = ZarrCompressor_Blosc1; - settings->compression_codec = ZarrCompressionCodec_BloscZstd; - settings->compression_level = 8; - settings->compression_shuffle = 2; - - compression_settings = ZarrStreamSettings_get_compression(settings); - EXPECT_EQ( - int, "%d", compression_settings.compressor, ZarrCompressor_Blosc1); - EXPECT_EQ( - int, "%d", compression_settings.codec, ZarrCompressionCodec_BloscZstd); - EXPECT_EQ(int, "%d", compression_settings.level, 8); - EXPECT_EQ(int, "%d", compression_settings.shuffle, 2); -} - -void -get_data_type(ZarrStreamSettings* settings) -{ - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_get_data_type(settings), - ZarrDataType_uint8); - - settings->dtype = ZarrDataType_float32; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_get_data_type(settings), - ZarrDataType_float32); -} - -void -get_dimensions(ZarrStreamSettings* settings) -{ - EXPECT_EQ(int, "%d", ZarrStreamSettings_get_dimension_count(settings), 0); - - settings->dimensions.resize(3); - EXPECT_EQ(int, "%d", ZarrStreamSettings_get_dimension_count(settings), 3); - - { - auto& dim = settings->dimensions[0]; - dim.name = "time"; - dim.type = ZarrDimensionType_Time; - dim.array_size_px = 100; - dim.chunk_size_px = 13; - dim.shard_size_chunks = 7; - } - { - auto& dim = settings->dimensions[1]; - dim.name = "height"; - dim.array_size_px = 300; - dim.chunk_size_px = 19; - dim.shard_size_chunks = 13; - } - { - auto& dim = settings->dimensions[2]; - dim.name = "width"; - dim.array_size_px = 200; - dim.chunk_size_px = 17; - dim.shard_size_chunks = 11; - } - - // can't get beyond the last dimension - auto dim = ZarrStreamSettings_get_dimension(settings, 3); - EXPECT_STR_EQ(dim.name, ""); - EXPECT_EQ(int, "%d", dim.bytes_of_name, 0); - EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", dim.array_size_px, 0); - EXPECT_EQ(int, "%d", dim.chunk_size_px, 0); - EXPECT_EQ(int, "%d", dim.shard_size_chunks, 0); - - dim = ZarrStreamSettings_get_dimension(settings, 0); - EXPECT_STR_EQ(dim.name, "time"); - EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("time")); - EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Time); - EXPECT_EQ(int, "%d", dim.array_size_px, 100); - EXPECT_EQ(int, "%d", dim.chunk_size_px, 13); - EXPECT_EQ(int, "%d", dim.shard_size_chunks, 7); - - dim = ZarrStreamSettings_get_dimension(settings, 1); - EXPECT_STR_EQ(dim.name, "height"); - EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("height")); - EXPECT_EQ(int, "%d", dim.array_size_px, 300); - EXPECT_EQ(int, "%d", dim.chunk_size_px, 19); - EXPECT_EQ(int, "%d", dim.shard_size_chunks, 13); - - dim = ZarrStreamSettings_get_dimension(settings, 2); - EXPECT_STR_EQ(dim.name, "width"); - EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("width")); - EXPECT_EQ(int, "%d", dim.array_size_px, 200); - EXPECT_EQ(int, "%d", dim.chunk_size_px, 17); - EXPECT_EQ(int, "%d", dim.shard_size_chunks, 11); -} - -void -get_multiscale(ZarrStreamSettings* settings) -{ - CHECK(!ZarrStreamSettings_get_multiscale(settings)); - - settings->multiscale = true; - CHECK(ZarrStreamSettings_get_multiscale(settings)); -} - -void -get_custom_metadata(ZarrStreamSettings* settings) -{ - EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); - - settings->custom_metadata = "this ain't even json"; // oops - EXPECT_STR_EQ(settings->custom_metadata.c_str(), "this ain't even json"); -} - -int -main() -{ - int retval = 1; - - ZarrStreamSettings* settings = ZarrStreamSettings_create(); - try { - check_preliminaries(settings); - get_store_path(settings); - get_s3_settings(settings); - get_compression(settings); - get_data_type(settings); - get_dimensions(settings); - get_multiscale(settings); - get_custom_metadata(settings); - - retval = 0; - } catch (const std::exception& exception) { - LOG_ERROR("%s", exception.what()); - } - ZarrStreamSettings_destroy(settings); - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/set-stream-parameters.cpp b/tests/unit-tests/set-stream-parameters.cpp deleted file mode 100644 index f8dedcda..00000000 --- a/tests/unit-tests/set-stream-parameters.cpp +++ /dev/null @@ -1,273 +0,0 @@ -#include "acquire.zarr.h" -#include "stream.settings.hh" -#include "unit.test.macros.hh" - -void -check_preliminaries(ZarrStreamSettings* settings) -{ - CHECK(settings); - - CHECK(settings->store_path.empty()); - - CHECK(settings->s3_endpoint.empty()); - CHECK(settings->s3_bucket_name.empty()); - CHECK(settings->s3_access_key_id.empty()); - CHECK(settings->s3_secret_access_key.empty()); - - EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); - - EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint8); - - EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_None); - EXPECT_EQ( - int, "%d", settings->compression_codec, ZarrCompressionCodec_None); - EXPECT_EQ(int, "%d", settings->compression_level, 0); - EXPECT_EQ(int, "%d", settings->compression_shuffle, 0); - - CHECK(settings->dimensions.empty()); - - CHECK(!settings->multiscale); -} - -void -set_store(ZarrStreamSettings* settings) -{ - std::string store_path = TEST ".zarr"; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_store( - settings, store_path.c_str(), store_path.size() + 1, nullptr), - ZarrStatus_Success); - - EXPECT_STR_EQ(settings->store_path.c_str(), store_path.c_str()); - settings->store_path = ""; // reset - - ZarrS3Settings s3_settings{ - .endpoint = "https://s3.amazonaws.com", - .bytes_of_endpoint = sizeof("https://s3.amazonaws.com"), - .bucket_name = "bucket", - .bytes_of_bucket_name = sizeof("bucket"), - .access_key_id = "access_key", - .bytes_of_access_key_id = sizeof("access_key"), - .secret_access_key = "secret_access_key", - .bytes_of_secret_access_key = sizeof("secret_access_key"), - }; - - EXPECT_EQ( - int, - "%d", - ZarrStreamSettings_set_store( - settings, store_path.c_str(), store_path.size() + 1, &s3_settings), - ZarrStatus_Success); - - EXPECT_STR_EQ(settings->store_path.c_str(), store_path.c_str()); - EXPECT_STR_EQ(settings->s3_endpoint.c_str(), s3_settings.endpoint); - EXPECT_STR_EQ(settings->s3_bucket_name.c_str(), s3_settings.bucket_name); - EXPECT_STR_EQ(settings->s3_access_key_id.c_str(), - s3_settings.access_key_id); - EXPECT_STR_EQ(settings->s3_secret_access_key.c_str(), - s3_settings.secret_access_key); -} - -void -set_compression(ZarrStreamSettings* settings) -{ - ZarrCompressionSettings compression_settings{ - .compressor = ZarrCompressor_Blosc1, - .codec = ZarrCompressionCodec_BloscLZ4, - .level = 5, - .shuffle = 1, - }; - - EXPECT_EQ( - int, - "%d", - ZarrStreamSettings_set_compression(settings, &compression_settings), - ZarrStatus_Success); - - EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_Blosc1); - EXPECT_EQ( - int, "%d", settings->compression_codec, ZarrCompressionCodec_BloscLZ4); - EXPECT_EQ(int, "%d", settings->compression_level, 5); - EXPECT_EQ(int, "%d", settings->compression_shuffle, 1); -} - -void -set_data_type(ZarrStreamSettings* settings) -{ - ZarrDataType dtype = ZarrDataType_uint16; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_data_type(settings, dtype), - ZarrStatus_Success); - EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint16); -} - -void -set_dimensions(ZarrStreamSettings* settings) -{ - ZarrStreamSettings_reserve_dimensions(settings, 3); - EXPECT_EQ(int, "%d", settings->dimensions.size(), 3); - - ZarrDimensionProperties dim{ - .name = " time ", - .bytes_of_name = sizeof(" time "), - .type = ZarrDimensionType_Time, - .array_size_px = 100, - .chunk_size_px = 13, - .shard_size_chunks = 7, - }; - - // can't set a dimension that is out of bounds - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 3, &dim), - ZarrStatus_InvalidIndex); - - // set the first dimension - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 0, &dim), - ZarrStatus_Success); - - EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); - EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); - EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); - EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); - EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); - - // other dimensions should still be unset - EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), ""); - EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 0); - EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 0); - EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 0); - - EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), ""); - EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 0); - EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 0); - EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 0); - - // set the 3rd dimension before the 2nd - dim.name = "width "; - dim.bytes_of_name = sizeof("width "); - dim.type = ZarrDimensionType_Space; - dim.array_size_px = 200; - dim.chunk_size_px = 17; - dim.shard_size_chunks = 11; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 2, &dim), - ZarrStatus_Success); - - EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); - EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); - EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); - EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); - EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); - - // 2nd dimension should still be unset - EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), ""); - EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 0); - EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 0); - EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 0); - - EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), "width"); - EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 200); - EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 17); - EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 11); - - // set the 2nd dimension - dim.name = "height"; - dim.bytes_of_name = sizeof("height"); - dim.type = ZarrDimensionType_Space; - dim.array_size_px = 300; - dim.chunk_size_px = 19; - dim.shard_size_chunks = 13; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 1, &dim), - ZarrStatus_Success); - - EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); - EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); - EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); - EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); - EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); - - EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), "height"); - EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 300); - EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 19); - EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 13); - - EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), "width"); - EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); - EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 200); - EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 17); - EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 11); -} - -void -set_multiscale(ZarrStreamSettings* settings) -{ - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_multiscale(settings, true), - ZarrStatus_Success); - CHECK(settings->multiscale); - - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_multiscale(settings, false), - ZarrStatus_Success); - CHECK(!settings->multiscale); -} - -void -set_custom_metadata(ZarrStreamSettings* settings) -{ - // fails when not JSON formatted - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_custom_metadata( - settings, "this is not json", sizeof("this is not json")), - ZarrStatus_InvalidArgument); - EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); - - // succeeds when JSON formatted - EXPECT_EQ( - int, - "%d", - ZarrStreamSettings_set_custom_metadata( - settings, "{\"key\": \"value\"}", sizeof("{\"key\": \"value\"}")), - ZarrStatus_Success); - // whitespace is removed - EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{\"key\":\"value\"}"); -} - -int -main() -{ - int retval = 1; - - ZarrStreamSettings* settings = ZarrStreamSettings_create(); - try { - check_preliminaries(settings); - set_store(settings); - set_compression(settings); - set_data_type(settings); - set_dimensions(settings); - set_multiscale(settings); - set_custom_metadata(settings); - - retval = 0; - } catch (const std::exception& exception) { - LOG_ERROR("%s", exception.what()); - } - ZarrStreamSettings_destroy(settings); - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/stream-get-settings.cpp b/tests/unit-tests/stream-get-settings.cpp deleted file mode 100644 index 22f3d662..00000000 --- a/tests/unit-tests/stream-get-settings.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include "acquire.zarr.h" -#include "zarr.stream.hh" -#include "unit.test.macros.hh" - -#include - -namespace fs = std::filesystem; - -void -configure_stream_dimensions(ZarrStreamSettings* settings) -{ - ZarrStreamSettings_reserve_dimensions(settings, 3); - - ZarrDimensionProperties dim = { - .name = "t", - .bytes_of_name = sizeof("t"), - .type = ZarrDimensionType_Time, - .array_size_px = 100, - .chunk_size_px = 10, - }; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 0, &dim), - ZarrStatus_Success); - - dim.name = "y"; - dim.type = ZarrDimensionType_Space; - dim.array_size_px = 200; - dim.chunk_size_px = 20; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 1, &dim), - ZarrStatus_Success); - - dim.name = "x"; - dim.array_size_px = 300; - dim.chunk_size_px = 30; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 2, &dim), - ZarrStatus_Success); -} - -void -compare_settings(const ZarrStreamSettings* settings, - const ZarrStreamSettings* settings_copy) -{ - EXPECT_STR_EQ(settings->store_path.c_str(), - settings_copy->store_path.c_str()); - - EXPECT_STR_EQ(settings->s3_endpoint.c_str(), - settings_copy->s3_endpoint.c_str()); - EXPECT_STR_EQ(settings->s3_bucket_name.c_str(), - settings_copy->s3_bucket_name.c_str()); - EXPECT_STR_EQ(settings->s3_access_key_id.c_str(), - settings_copy->s3_access_key_id.c_str()); - EXPECT_STR_EQ(settings->s3_secret_access_key.c_str(), - settings_copy->s3_secret_access_key.c_str()); - - EXPECT_STR_EQ(settings->custom_metadata.c_str(), - settings_copy->custom_metadata.c_str()); - - EXPECT_EQ(int, "%d", settings->dtype, settings_copy->dtype); - - EXPECT_EQ(int, "%d", settings->compressor, settings_copy->compressor); - EXPECT_EQ(int, "%d", settings->compression_codec, settings_copy->compression_codec); - EXPECT_EQ(int, "%d", settings->compression_level, settings_copy->compression_level); - EXPECT_EQ(int, "%d", settings->compression_shuffle, settings_copy->compression_shuffle); - - EXPECT_EQ(int, "%d", settings->dimensions.size(), 3); - for (auto i = 0; i < 3; ++i) { - const auto& dim = &settings->dimensions[i]; - const auto& dim_copy = &settings_copy->dimensions[i]; - EXPECT_STR_EQ(dim->name.c_str(), dim_copy->name.c_str()); - EXPECT_EQ(int, "%d", dim->type, dim_copy->type); - EXPECT_EQ(int, "%d", dim->array_size_px, dim_copy->array_size_px); - EXPECT_EQ(int, "%d", dim->chunk_size_px, dim_copy->chunk_size_px); - EXPECT_EQ(int, "%d", dim->shard_size_chunks, dim_copy->shard_size_chunks); - } - - EXPECT_EQ(bool, "%d", settings->multiscale, settings_copy->multiscale); -} - -int -main() -{ - int retval = 1; - - ZarrStream* stream; - ZarrStreamSettings *settings = ZarrStreamSettings_create(), *settings_copy; - if (!settings) { - LOG_ERROR("Failed to create ZarrStreamSettings"); - return retval; - } - settings->store_path = TEST ".zarr"; - - try { - configure_stream_dimensions(settings); - stream = ZarrStream_create(settings, ZarrVersion_2); - CHECK(nullptr != stream); - CHECK(fs::is_directory(settings->store_path)); - - EXPECT_EQ(int, "%d", ZarrStream_get_version(stream), ZarrVersion_2); - - settings_copy = ZarrStream_get_settings(stream); - CHECK(nullptr != settings_copy); - CHECK(settings != settings_copy); - compare_settings(settings, settings_copy); - - retval = 0; - } catch (const std::exception& exception) { - LOG_ERROR("%s", exception.what()); - } - - // cleanup - if (fs::is_directory(settings->store_path)) { - fs::remove_all(settings->store_path); - } - ZarrStreamSettings_destroy(settings); - ZarrStreamSettings_destroy(settings_copy); - ZarrStream_destroy(stream); - - return retval; -} \ No newline at end of file From 25bfb8b372e9b153b9da1318c1b844e4577e257a Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 11:00:17 -0400 Subject: [PATCH 24/39] Reorder and document settings fields --- include/acquire.zarr.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index cbbbaf86..9bc1ab2f 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -9,15 +9,15 @@ extern "C" typedef struct ZarrStreamSettings_s { - ZarrS3Settings s3_settings; - ZarrCompressionSettings compression_settings; - ZarrDimensionProperties* dimensions; - size_t dimension_count; - char* store_path; - char* custom_metadata; - ZarrDataType data_type; - ZarrVersion version; - bool multiscale; + char* store_path; /**< Path to the store. Filesystem path or S3 key prefix. */ + char* custom_metadata; /**< JSON-formatted custom metadata to be stored with the dataset. */ + ZarrS3Settings* s3_settings; /**< Optional S3 settings for the store. */ + ZarrCompressionSettings* compression_settings; /**< Optional chunk compression settings for the store. */ + ZarrDimensionProperties* dimensions; /**< The properties of each dimension in the dataset. */ + size_t dimension_count; /**< The number of dimensions in the dataset. */ + bool multiscale; /**< Whether to stream to multiple levels of detail. */ + ZarrDataType data_type; /**< The pixel data type of the dataset. */ + ZarrVersion version; /**< The version of the Zarr format to use. 2 or 3. */ } ZarrStreamSettings; typedef struct ZarrStream_s ZarrStream; From b7aeda7eb76687dba801f61d739d395efa87ebd1 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 11:39:14 -0400 Subject: [PATCH 25/39] wip --- src/streaming/stream.settings.hh | 11 ----------- src/streaming/zarr.stream.hh | 15 +++++++++++---- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/streaming/stream.settings.hh b/src/streaming/stream.settings.hh index a23e48cf..da25a9b0 100644 --- a/src/streaming/stream.settings.hh +++ b/src/streaming/stream.settings.hh @@ -7,17 +7,6 @@ #include #include -struct ZarrDimension_s -{ - std::string name; /* Name of the dimension */ - ZarrDimensionType type; /* Type of dimension */ - - uint32_t array_size_px; /* Size of the array along this dimension */ - uint32_t chunk_size_px; /* Size of a chunk along this dimension */ - uint32_t shard_size_chunks; /* Number of chunks in a shard along this - dimension */ -}; - struct ZarrStreamSettings_s { public: diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index e2812703..6e134d75 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -1,12 +1,21 @@ #pragma once -#include "stream.settings.hh" - #include #include // size_t #include // unique_ptr +struct ZarrDimension_s +{ + std::string name; /* Name of the dimension */ + ZarrDimensionType type; /* Type of dimension */ + + uint32_t array_size_px; /* Size of the array along this dimension */ + uint32_t chunk_size_px; /* Size of a chunk along this dimension */ + uint32_t shard_size_chunks; /* Number of chunks in a shard along this + dimension */ +}; + struct ZarrStream_s { public: @@ -21,8 +30,6 @@ struct ZarrStream_s */ size_t append(const void* data, size_t nbytes); - ZarrVersion version() const { return version_; } - private: std::string error_; // error message. If nonempty, an error occurred. From e3e6e3f1e407a765ef77b6e30fbe8036440e6519 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 11:39:51 -0400 Subject: [PATCH 26/39] Remove version specifier from ZarrStream_create. --- include/acquire.zarr.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index 9bc1ab2f..1ba44980 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -51,11 +51,9 @@ extern "C" /** * @brief Create a Zarr stream. * @param[in, out] settings The settings for the Zarr stream. - * @param[in] version The version of the Zarr stream. 2 or 3. * @return A pointer to the Zarr stream struct, or NULL on failure. */ - ZarrStream* ZarrStream_create(ZarrStreamSettings* settings, - ZarrVersion version); + ZarrStream* ZarrStream_create(ZarrStreamSettings* settings); /** * @brief Destroy a Zarr stream. From 401231b3db5107c4ca82aa663f1ff51d54fa511d Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 12:06:29 -0400 Subject: [PATCH 27/39] wip --- src/streaming/zarr.stream.cpp | 13 +++++-------- src/streaming/zarr.stream.hh | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp index ba02ed2f..c9060931 100644 --- a/src/streaming/zarr.stream.cpp +++ b/src/streaming/zarr.stream.cpp @@ -12,21 +12,18 @@ namespace { bool is_s3_acquisition(const struct ZarrStreamSettings_s* settings) { - return !settings->s3_endpoint.empty() && - !settings->s3_bucket_name.empty() && - !settings->s3_access_key_id.empty() && - !settings->s3_secret_access_key.empty(); + return nullptr != settings->s3_settings; } bool is_compressed_acquisition(const struct ZarrStreamSettings_s* settings) { - return settings->compressor != ZarrCompressor_None; + return nullptr != settings->compression_settings; } [[nodiscard]] bool -validate_s3_settings(const struct ZarrStreamSettings_s* settings) +validate_s3_settings(const ZarrS3Settings* settings) { if (settings->s3_endpoint.empty()) { LOG_ERROR("S3 endpoint is empty"); @@ -186,8 +183,8 @@ validate_settings(const struct ZarrStreamSettings_s* settings, return false; } - if (settings->dtype >= ZarrDataTypeCount) { - LOG_ERROR("Invalid data type: %d", settings->dtype); + if (settings->data_type >= ZarrDataTypeCount) { + LOG_ERROR("Invalid data type: %d", settings->data_type); return false; } diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index 6e134d75..555033bb 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -13,7 +13,7 @@ struct ZarrDimension_s uint32_t array_size_px; /* Size of the array along this dimension */ uint32_t chunk_size_px; /* Size of a chunk along this dimension */ uint32_t shard_size_chunks; /* Number of chunks in a shard along this - dimension */ + * dimension */ }; struct ZarrStream_s From d15f431e637fc439537841c7b2f929734d84b2cd Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 12:16:11 -0400 Subject: [PATCH 28/39] Document the settings struct a bit. --- include/acquire.zarr.h | 33 +++++++++++++++++++++++++++++++-- include/zarr.types.h | 22 +++++++++++----------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index 1ba44980..b4f352a9 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -7,6 +7,21 @@ extern "C" { #endif + /** + * @brief The settings for a Zarr stream. + * @details This struct contains the settings for a Zarr stream, including + * the store path, custom metadata, S3 settings, chunk compression settings, + * dimension properties, whether to stream to multiple levels of detail, the + * pixel data type, and the Zarr format version. + * @note The store path can be a filesystem path or an S3 key prefix. For example, + * supplying an endpoint "s3://my-endpoint.com" and a bucket "my-bucket" with a + * store_path of "my-dataset.zarr" will result in the store being written to + * "s3://my-endpoint.com/my-bucket/my-dataset.zarr". + * @note The dimensions array may be allocated with ZarrStreamSettings_create_dimension_array + * and freed with ZarrStreamSettings_destroy_dimension_array. The order in which you + * set the dimension properties in the array should match the order of the dimensions + * from slowest to fastest changing, for example, [Z, Y, X] for a 3D dataset. + */ typedef struct ZarrStreamSettings_s { char* store_path; /**< Path to the store. Filesystem path or S3 key prefix. */ @@ -31,7 +46,7 @@ extern "C" /** * @brief Set the log level for the Zarr API. * @param level The log level. - * @return ZarrStatus_Success on success, or an error code on failure. + * @return ZarrStatusCode_Success on success, or an error code on failure. */ ZarrStatusCode Zarr_set_log_level(ZarrLogLevel level); @@ -48,6 +63,20 @@ extern "C" */ const char* Zarr_get_status_message(ZarrStatusCode status); + /** + * @brief Allocate memory for the dimension array in the Zarr stream settings struct. + * @param[in, out] settings The Zarr stream settings struct. + * @param dimension_count The number of dimensions in the dataset to allocate memory for. + * @return ZarrStatusCode_Success on success, or an error code on failure. + */ + ZarrStatusCode ZarrStreamSettings_create_dimension_array(ZarrStreamSettings* settings, size_t dimension_count); + + /** + * @brief Free memory for the dimension array in the Zarr stream settings struct. + * @param[in, out] settings The Zarr stream settings struct containing the dimension array to free. + */ + void ZarrStreamSettings_destroy_dimension_array(ZarrStreamSettings* settings); + /** * @brief Create a Zarr stream. * @param[in, out] settings The settings for the Zarr stream. @@ -71,7 +100,7 @@ extern "C" * @param[in] bytes_in The number of bytes in @p data. It should be at least * the size of a single frame. * @param[out] bytes_out The number of bytes written to the stream. - * @return ZarrStatus_Success on success, or an error code on failure. + * @return ZarrStatusCode_Success on success, or an error code on failure. */ ZarrStatusCode ZarrStream_append(ZarrStream* stream, const void* data, diff --git a/include/zarr.types.h b/include/zarr.types.h index 2dfd0dd1..13ee89be 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -12,17 +12,17 @@ extern "C" typedef enum { - ZarrStatus_Success = 0, - ZarrStatus_InvalidArgument, - ZarrStatus_Overflow, - ZarrStatus_InvalidIndex, - ZarrStatus_NotYetImplemented, - ZarrStatus_InternalError, - ZarrStatus_OutOfMemory, - ZarrStatus_IOError, - ZarrStatus_CompressionError, - ZarrStatus_InvalidSettings, - ZarrStatusCount, + ZarrStatusCode_Success = 0, + ZarrStatusCode_InvalidArgument, + ZarrStatusCode_Overflow, + ZarrStatusCode_InvalidIndex, + ZarrStatusCode_NotYetImplemented, + ZarrStatusCode_InternalError, + ZarrStatusCode_OutOfMemory, + ZarrStatusCode_IOError, + ZarrStatusCode_CompressionError, + ZarrStatusCode_InvalidSettings, + ZarrStatusCodeCount, } ZarrStatusCode; typedef enum From e6ad8273db5f6bfcc49175e969654e62581f3a47 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 15:09:03 -0400 Subject: [PATCH 29/39] wip --- src/streaming/acquire.zarr.cpp | 31 +++-- src/streaming/macros.hh | 4 +- src/streaming/zarr.stream.cpp | 204 ++++++++++++++++++++++------- src/streaming/zarr.stream.hh | 48 +++++-- tests/unit-tests/create-stream.cpp | 63 ++++----- 5 files changed, 231 insertions(+), 119 deletions(-) diff --git a/src/streaming/acquire.zarr.cpp b/src/streaming/acquire.zarr.cpp index 9b6cff9d..62acd007 100644 --- a/src/streaming/acquire.zarr.cpp +++ b/src/streaming/acquire.zarr.cpp @@ -19,7 +19,7 @@ extern "C" level < ZarrLogLevelCount, "Invalid log level: %d", level); Logger::set_log_level(level); - return ZarrStatus_Success; + return ZarrStatusCode_Success; } ZarrLogLevel Zarr_get_log_level() @@ -30,39 +30,38 @@ extern "C" const char* Zarr_get_status_message(ZarrStatusCode error) { switch (error) { - case ZarrStatus_Success: + case ZarrStatusCode_Success: return "Success"; - case ZarrStatus_InvalidArgument: + case ZarrStatusCode_InvalidArgument: return "Invalid argument"; - case ZarrStatus_Overflow: + case ZarrStatusCode_Overflow: return "Buffer overflow"; - case ZarrStatus_InvalidIndex: + case ZarrStatusCode_InvalidIndex: return "Invalid index"; - case ZarrStatus_NotYetImplemented: + case ZarrStatusCode_NotYetImplemented: return "Not yet implemented"; - case ZarrStatus_InternalError: + case ZarrStatusCode_InternalError: return "Internal error"; - case ZarrStatus_OutOfMemory: + case ZarrStatusCode_OutOfMemory: return "Out of memory"; - case ZarrStatus_IOError: + case ZarrStatusCode_IOError: return "I/O error"; - case ZarrStatus_CompressionError: + case ZarrStatusCode_CompressionError: return "Compression error"; - case ZarrStatus_InvalidSettings: + case ZarrStatusCode_InvalidSettings: return "Invalid settings"; default: return "Unknown error"; } } - ZarrStream_s* ZarrStream_create(struct ZarrStreamSettings_s* settings, - ZarrVersion version) + ZarrStream_s* ZarrStream_create(struct ZarrStreamSettings_s* settings) { ZarrStream_s* stream = nullptr; try { - stream = new ZarrStream_s(settings, version); + stream = new ZarrStream_s(settings); } catch (const std::bad_alloc&) { LOG_ERROR("Failed to allocate memory for Zarr stream"); } catch (const std::exception& e) { @@ -90,9 +89,9 @@ extern "C" *bytes_out = stream->append(data, bytes_in); } catch (const std::exception& e) { LOG_ERROR("Error appending data: %s", e.what()); - return ZarrStatus_InternalError; + return ZarrStatusCode_InternalError; } - return ZarrStatus_Success; + return ZarrStatusCode_Success; } } \ No newline at end of file diff --git a/src/streaming/macros.hh b/src/streaming/macros.hh index 05b094b3..2cc75bc1 100644 --- a/src/streaming/macros.hh +++ b/src/streaming/macros.hh @@ -15,7 +15,7 @@ do { \ if (!(e)) { \ LOG_ERROR(__VA_ARGS__); \ - return ZarrStatus_InvalidArgument; \ + return ZarrStatusCode_InvalidArgument; \ } \ } while (0) @@ -23,6 +23,6 @@ do { \ if (!(e)) { \ LOG_ERROR(__VA_ARGS__); \ - return ZarrStatus_InvalidIndex; \ + return ZarrStatusCode_InvalidIndex; \ } \ } while (0) \ No newline at end of file diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp index c9060931..ab94dc51 100644 --- a/src/streaming/zarr.stream.cpp +++ b/src/streaming/zarr.stream.cpp @@ -21,26 +21,59 @@ is_compressed_acquisition(const struct ZarrStreamSettings_s* settings) return nullptr != settings->compression_settings; } +[[nodiscard]] +std::string +trim(const char* s) +{ + if (!s || !*s) { + return {}; + } + + const size_t length = strlen(s); + + // trim left + std::string trimmed(s, length); + trimmed.erase(trimmed.begin(), + std::find_if(trimmed.begin(), trimmed.end(), [](char c) { + return !std::isspace(c); + })); + + // trim right + trimmed.erase(std::find_if(trimmed.rbegin(), + trimmed.rend(), + [](char c) { return !std::isspace(c); }) + .base(), + trimmed.end()); + + return trimmed; +} + [[nodiscard]] bool validate_s3_settings(const ZarrS3Settings* settings) { - if (settings->s3_endpoint.empty()) { + std::string trimmed = trim(settings->endpoint); + if (trimmed.empty()) { LOG_ERROR("S3 endpoint is empty"); return false; } - if (settings->s3_bucket_name.length() < 3 || - settings->s3_bucket_name.length() > 63) { + + trimmed = trim(settings->bucket_name); + if (trimmed.length() < 3 || trimmed.length() > 63) { LOG_ERROR("Invalid length for S3 bucket name: %zu. Must be between 3 " "and 63 characters", - settings->s3_bucket_name.length()); + trimmed.length()); return false; } - if (settings->s3_access_key_id.empty()) { + + trimmed = trim(settings->access_key_id); + if (trimmed.empty()) { LOG_ERROR("S3 access key ID is empty"); return false; } - if (settings->s3_secret_access_key.empty()) { + + trimmed = trim(settings->secret_access_key); + if (trimmed.empty()) { LOG_ERROR("S3 secret access key is empty"); return false; } @@ -81,36 +114,37 @@ validate_filesystem_store_path(std::string_view data_root) [[nodiscard]] bool -validate_compression_settings(const ZarrStreamSettings_s* settings) +validate_compression_settings(const ZarrCompressionSettings* settings) { if (settings->compressor >= ZarrCompressorCount) { LOG_ERROR("Invalid compressor: %d", settings->compressor); return false; } - if (settings->compression_codec >= ZarrCompressionCodecCount) { - LOG_ERROR("Invalid compression codec: %d", settings->compression_codec); + if (settings->codec >= ZarrCompressionCodecCount) { + LOG_ERROR("Invalid compression codec: %d", settings->codec); return false; } - // we know the compressor is not None, so the codec must be set - if (settings->compression_codec == ZarrCompressionCodec_None) { + // if compressing, we require a compression codec + if (settings->compressor != ZarrCompressor_None && + settings->codec == ZarrCompressionCodec_None) { LOG_ERROR("Compression codec must be set when using a compressor"); return false; } - if (settings->compression_level == 0 || settings->compression_level > 9) { - LOG_ERROR("Invalid compression level: %d. Must be between 1 and 9", - settings->compression_level); + if (settings->level > 9) { + LOG_ERROR("Invalid compression level: %d. Must be between 0 and 9", + settings->level); return false; } - if (settings->compression_shuffle != BLOSC_NOSHUFFLE && - settings->compression_shuffle != BLOSC_SHUFFLE && - settings->compression_shuffle != BLOSC_BITSHUFFLE) { + if (settings->shuffle != BLOSC_NOSHUFFLE && + settings->shuffle != BLOSC_SHUFFLE && + settings->shuffle != BLOSC_BITSHUFFLE) { LOG_ERROR("Invalid shuffle: %d. Must be %d (no shuffle), %d (byte " "shuffle), or %d (bit shuffle)", - settings->compression_shuffle, + settings->shuffle, BLOSC_NOSHUFFLE, BLOSC_SHUFFLE, BLOSC_BITSHUFFLE); @@ -122,31 +156,55 @@ validate_compression_settings(const ZarrStreamSettings_s* settings) [[nodiscard]] bool -validate_dimension(const struct ZarrDimension_s& dimension, +validate_custom_metadata(std::string_view metadata) +{ + if (metadata.empty()) { + return true; // custom metadata is optional + } + + // parse the JSON + auto val = nlohmann::json::parse(metadata, + nullptr, // callback + false, // allow exceptions + true // ignore comments + ); + + if (val.is_discarded()) { + LOG_ERROR("Invalid JSON: %s", metadata.data()); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_dimension(const ZarrDimensionProperties* dimension, ZarrVersion version, bool is_append) { - if (dimension.name.empty()) { + std::string trimmed = trim(dimension->name); + if (trimmed.empty()) { LOG_ERROR("Invalid name. Must not be empty"); return false; } - if (dimension.type >= ZarrDimensionTypeCount) { - LOG_ERROR("Invalid dimension type: %d", dimension.type); + if (dimension->type >= ZarrDimensionTypeCount) { + LOG_ERROR("Invalid dimension type: %d", dimension->type); return false; } - if (!is_append && dimension.array_size_px == 0) { + if (!is_append && dimension->array_size_px == 0) { LOG_ERROR("Array size must be nonzero"); return false; } - if (dimension.chunk_size_px == 0) { - LOG_ERROR("Chunk size must be nonzero"); + if (dimension->chunk_size_px == 0) { + LOG_ERROR("Invalid chunk size: %zu", dimension->chunk_size_px); return false; } - if (version == ZarrVersion_3 && dimension.shard_size_chunks == 0) { + if (version == ZarrVersion_3 && dimension->shard_size_chunks == 0) { LOG_ERROR("Shard size must be nonzero"); return false; } @@ -156,13 +214,14 @@ validate_dimension(const struct ZarrDimension_s& dimension, [[nodiscard]] bool -validate_settings(const struct ZarrStreamSettings_s* settings, - ZarrVersion version) +validate_settings(const struct ZarrStreamSettings_s* settings) { if (!settings) { LOG_ERROR("Null pointer: settings"); return false; } + + auto version = settings->version; if (version < ZarrVersion_2 || version >= ZarrVersionCount) { LOG_ERROR("Invalid Zarr version: %d", version); return false; @@ -176,7 +235,8 @@ validate_settings(const struct ZarrStreamSettings_s* settings, return false; } - if (is_s3_acquisition(settings) && !validate_s3_settings(settings)) { + if (is_s3_acquisition(settings) && + !validate_s3_settings(settings->s3_settings)) { return false; } else if (!is_s3_acquisition(settings) && !validate_filesystem_store_path(store_path)) { @@ -189,33 +249,42 @@ validate_settings(const struct ZarrStreamSettings_s* settings, } if (is_compressed_acquisition(settings) && - !validate_compression_settings(settings)) { + !validate_compression_settings(settings->compression_settings)) { + return false; + } + + if (!validate_custom_metadata(settings->custom_metadata)) { + return false; + } + + if (nullptr == settings->dimensions) { + LOG_ERROR("Null pointer: dimensions"); return false; } // we must have at least 3 dimensions - if (settings->dimensions.size() < 3) { + const size_t ndims = settings->dimension_count; + if (ndims < 3) { LOG_ERROR("Invalid number of dimensions: %zu. Must be at least 3", - settings->dimensions.size()); + ndims); return false; } // check the final dimension (width), must be space - if (settings->dimensions.back().type != ZarrDimensionType_Space) { + if (settings->dimensions[ndims - 1].type != ZarrDimensionType_Space) { LOG_ERROR("Last dimension must be of type Space"); return false; } // check the penultimate dimension (height), must be space - if (settings->dimensions[settings->dimensions.size() - 2].type != - ZarrDimensionType_Space) { + if (settings->dimensions[ndims - 2].type != ZarrDimensionType_Space) { LOG_ERROR("Second to last dimension must be of type Space"); return false; } // validate the dimensions individually - for (size_t i = 0; i < settings->dimensions.size(); ++i) { - if (!validate_dimension(settings->dimensions[i], version, i == 0)) { + for (size_t i = 0; i < ndims; ++i) { + if (!validate_dimension(settings->dimensions + i, version, i == 0)) { return false; } } @@ -226,15 +295,15 @@ validate_settings(const struct ZarrStreamSettings_s* settings, /* ZarrStream_s implementation */ -ZarrStream::ZarrStream_s(struct ZarrStreamSettings_s* settings, - ZarrVersion version) - : version_(version) - , error_() +ZarrStream::ZarrStream_s(struct ZarrStreamSettings_s* settings) + : error_() { - if (!validate_settings(settings, version)) { + if (!validate_settings(settings)) { throw std::runtime_error("Invalid Zarr stream settings"); } + commit_settings_(settings); + // create the data store EXPECT(create_store_(), "%s", error_.c_str()); @@ -274,6 +343,41 @@ ZarrStream::append(const void* data, size_t nbytes) return 0; } +void +ZarrStream_s::commit_settings_(const struct ZarrStreamSettings_s* settings) +{ + version_ = settings->version; + store_path_ = trim(settings->store_path); + custom_metadata_ = trim(settings->custom_metadata); + + if (is_s3_acquisition(settings)) { + s3_endpoint_ = trim(settings->s3_settings->endpoint); + s3_bucket_name_ = trim(settings->s3_settings->bucket_name); + s3_access_key_id_ = trim(settings->s3_settings->access_key_id); + s3_secret_access_key_ = trim(settings->s3_settings->secret_access_key); + is_s3_acquisition_ = true; + } + + if (is_compressed_acquisition(settings)) { + compressor_ = settings->compression_settings->compressor; + compression_codec_ = settings->compression_settings->codec; + compression_level_ = settings->compression_settings->level; + compression_shuffle_ = settings->compression_settings->shuffle; + is_compressed_acquisition_ = true; + } + + dtype_ = settings->data_type; + + for (auto i = 0; i < settings->dimension_count; ++i) { + const auto& dim = settings->dimensions[i]; + dimensions_.emplace_back(dim.name, + dim.type, + dim.array_size_px, + dim.chunk_size_px, + dim.shard_size_chunks); + } +} + void ZarrStream_s::set_error_(const std::string& msg) { @@ -283,17 +387,17 @@ ZarrStream_s::set_error_(const std::string& msg) bool ZarrStream_s::create_store_() { - if (is_s3_acquisition(&settings_)) { + if (is_s3_acquisition_) { // TODO (aliddell): implement this } else { - if (fs::exists(settings_.store_path_)) { + if (fs::exists(store_path_)) { // remove everything inside the store path std::error_code ec; - fs::remove_all(settings_.store_path_, ec); + fs::remove_all(store_path_, ec); if (ec) { set_error_("Failed to remove existing store path '" + - settings_.store_path_ + "': " + ec.message()); + store_path_ + "': " + ec.message()); return false; } } @@ -301,9 +405,9 @@ ZarrStream_s::create_store_() // create the store path { std::error_code ec; - if (!fs::create_directories(settings_.store_path_, ec)) { + if (!fs::create_directories(store_path_, ec)) { set_error_("Failed to create store path '" + - settings_.store_path_ + "': " + ec.message()); + store_path_ + "': " + ec.message()); return false; } } @@ -322,7 +426,7 @@ ZarrStream_s::create_writers_() void ZarrStream_s::create_scaled_frames_() { - if (settings_.multiscale) { + if (multiscale_) { // TODO (aliddell): implement this } } @@ -366,7 +470,7 @@ void ZarrStream_s::write_multiscale_frames_(const uint8_t* data, size_t bytes_of_data) { - if (!settings_.multiscale) { + if (multiscale_) { return; } diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index 555033bb..14646d59 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -7,6 +7,20 @@ struct ZarrDimension_s { + public: + ZarrDimension_s(const char* name, + ZarrDimensionType type, + uint32_t array_size_px, + uint32_t chunk_size_px, + uint32_t shard_size_chunks) + : name(name) + , type(type) + , array_size_px(array_size_px) + , chunk_size_px(chunk_size_px) + , shard_size_chunks(shard_size_chunks) + { + } + std::string name; /* Name of the dimension */ ZarrDimensionType type; /* Type of dimension */ @@ -19,7 +33,7 @@ struct ZarrDimension_s struct ZarrStream_s { public: - ZarrStream_s(struct ZarrStreamSettings_s* settings, ZarrVersion version); + ZarrStream_s(struct ZarrStreamSettings_s* settings); ~ZarrStream_s(); /** @@ -37,23 +51,31 @@ struct ZarrStream_s std::string store_path_; - std::string s3_endpoint; - std::string s3_bucket_name; - std::string s3_access_key_id; - std::string s3_secret_access_key; + bool is_s3_acquisition_; + std::string s3_endpoint_; + std::string s3_bucket_name_; + std::string s3_access_key_id_; + std::string s3_secret_access_key_; + + std::string custom_metadata_; - std::string custom_metadata; + ZarrDataType dtype_; - ZarrDataType dtype; + bool is_compressed_acquisition_; + ZarrCompressor compressor_; + ZarrCompressionCodec compression_codec_; + uint8_t compression_level_; + uint8_t compression_shuffle_; - ZarrCompressor compressor; - ZarrCompressionCodec compression_codec; - uint8_t compression_level; - uint8_t compression_shuffle; + std::vector dimensions_; - std::vector dimensions; + bool multiscale_; - bool multiscale; + /** + * @brief Copy settings to the stream. + * @param settings Struct containing settings to copy. + */ + void commit_settings_(const struct ZarrStreamSettings_s* settings); /** * @brief Set an error message. diff --git a/tests/unit-tests/create-stream.cpp b/tests/unit-tests/create-stream.cpp index 1decf818..63f2839f 100644 --- a/tests/unit-tests/create-stream.cpp +++ b/tests/unit-tests/create-stream.cpp @@ -9,36 +9,27 @@ namespace fs = std::filesystem; void configure_stream_dimensions(ZarrStreamSettings* settings) { - ZarrStreamSettings_reserve_dimensions(settings, 3); + CHECK(ZarrStatusCode_Success == ZarrStreamSettings_create_dimension_array(settings, 3)); + ZarrDimensionProperties *dim = settings->dimensions; - ZarrDimensionProperties dim = { + *dim = ZarrDimensionProperties{ .name = "t", .bytes_of_name = sizeof("t"), .type = ZarrDimensionType_Time, .array_size_px = 100, .chunk_size_px = 10, }; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 0, &dim), - ZarrStatus_Success); - dim.name = "y"; - dim.type = ZarrDimensionType_Space; - dim.array_size_px = 200; - dim.chunk_size_px = 20; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 1, &dim), - ZarrStatus_Success); + dim = settings->dimensions + 1; + dim->name = "y"; + dim->type = ZarrDimensionType_Space; + dim->array_size_px = 200; + dim->chunk_size_px = 20; - dim.name = "x"; - dim.array_size_px = 300; - dim.chunk_size_px = 30; - EXPECT_EQ(int, - "%d", - ZarrStreamSettings_set_dimension(settings, 2, &dim), - ZarrStatus_Success); + dim = settings->dimensions + 2; + dim->name = "x"; + dim->array_size_px = 300; + dim->chunk_size_px = 30; } int @@ -47,29 +38,25 @@ main() int retval = 1; ZarrStream* stream; - ZarrStreamSettings* settings = ZarrStreamSettings_create(); - if (!settings) { - LOG_ERROR("Failed to create ZarrStreamSettings"); - return retval; - } + ZarrStreamSettings settings; + settings.version = ZarrVersion_2; try { // try to create a stream with no store path - stream = ZarrStream_create(settings, ZarrVersion_2); + stream = ZarrStream_create(&settings); CHECK(nullptr == stream); // try to create a stream with no dimensions - settings->store_path = TEST ".zarr"; - stream = ZarrStream_create(settings, ZarrVersion_2); + settings.store_path = static_cast(TEST ".zarr"); + stream = ZarrStream_create(&settings); CHECK(nullptr == stream); - CHECK(!fs::exists(settings->store_path)); + CHECK(!fs::exists(settings.store_path)); - configure_stream_dimensions(settings); - stream = ZarrStream_create(settings, ZarrVersion_2); + // allocate dimensions + configure_stream_dimensions(&settings); + stream = ZarrStream_create(&settings); CHECK(nullptr != stream); - CHECK(fs::is_directory(settings->store_path)); - - EXPECT_EQ(int, "%d", ZarrStream_get_version(stream), ZarrVersion_2); + CHECK(fs::is_directory(settings.store_path)); retval = 0; } catch (const std::exception& exception) { @@ -77,10 +64,10 @@ main() } // cleanup - if (fs::is_directory(settings->store_path)) { - fs::remove_all(settings->store_path); + if (fs::is_directory(settings.store_path)) { + fs::remove_all(settings.store_path); } - ZarrStreamSettings_destroy(settings); + ZarrStreamSettings_destroy_dimension_array(&settings); ZarrStream_destroy(stream); return retval; } \ No newline at end of file From 59bea6ce223f3d17dd70570c9146b6b7de0511ad Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 15:13:27 -0400 Subject: [PATCH 30/39] Fix up some parameters. --- include/acquire.zarr.h | 4 ++-- include/zarr.types.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index b4f352a9..54a7288a 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -24,8 +24,8 @@ extern "C" */ typedef struct ZarrStreamSettings_s { - char* store_path; /**< Path to the store. Filesystem path or S3 key prefix. */ - char* custom_metadata; /**< JSON-formatted custom metadata to be stored with the dataset. */ + const char* store_path; /**< Path to the store. Filesystem path or S3 key prefix. */ + const char* custom_metadata; /**< JSON-formatted custom metadata to be stored with the dataset. */ ZarrS3Settings* s3_settings; /**< Optional S3 settings for the store. */ ZarrCompressionSettings* compression_settings; /**< Optional chunk compression settings for the store. */ ZarrDimensionProperties* dimensions; /**< The properties of each dimension in the dataset. */ diff --git a/include/zarr.types.h b/include/zarr.types.h index 13ee89be..6e04c03d 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -112,7 +112,6 @@ extern "C" typedef struct { const char* name; /**< Name of the dimension */ - size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ ZarrDimensionType type; /**< Type of the dimension */ uint32_t array_size_px; /**< Size of the array along this dimension in pixels */ From 3a0b63aa80f4b64dde488b374d5275a76768fa67 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 15:56:06 -0400 Subject: [PATCH 31/39] Update ZarrStream implementation to use settings struct. --- src/streaming/acquire.zarr.cpp | 36 +++++++++++++++++++++++++++++- src/streaming/zarr.stream.cpp | 28 ++++++++++++++++++----- src/streaming/zarr.stream.hh | 22 +++++++++--------- tests/unit-tests/create-stream.cpp | 21 ++++++++++------- 4 files changed, 82 insertions(+), 25 deletions(-) diff --git a/src/streaming/acquire.zarr.cpp b/src/streaming/acquire.zarr.cpp index 62acd007..061ae71e 100644 --- a/src/streaming/acquire.zarr.cpp +++ b/src/streaming/acquire.zarr.cpp @@ -1,4 +1,4 @@ -#include "zarr.types.h" +#include "acquire.zarr.h" #include "zarr.stream.hh" #include "macros.hh" @@ -55,6 +55,40 @@ extern "C" } } + ZarrStatusCode ZarrStreamSettings_create_dimension_array( + struct ZarrStreamSettings_s* settings, + size_t dimension_count) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(dimension_count >= 3, + "Invalid dimension count: %zu", + dimension_count); + + ZarrDimensionProperties* dimensions = nullptr; + + try { + dimensions = new ZarrDimensionProperties[dimension_count]; + } catch (const std::bad_alloc&) { + LOG_ERROR("Failed to allocate memory for dimensions"); + return ZarrStatusCode_OutOfMemory; + } + + ZarrStreamSettings_destroy_dimension_array(settings); + settings->dimensions = dimensions; + settings->dimension_count = dimension_count; + + return ZarrStatusCode_Success; + } + + void ZarrStreamSettings_destroy_dimension_array( + struct ZarrStreamSettings_s* settings) + { + if (nullptr != settings && nullptr != settings->dimensions) { + delete[] settings->dimensions; + settings->dimensions = nullptr; + } + } + ZarrStream_s* ZarrStream_create(struct ZarrStreamSettings_s* settings) { diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp index ab94dc51..e1eb3b74 100644 --- a/src/streaming/zarr.stream.cpp +++ b/src/streaming/zarr.stream.cpp @@ -156,9 +156,9 @@ validate_compression_settings(const ZarrCompressionSettings* settings) [[nodiscard]] bool -validate_custom_metadata(std::string_view metadata) +validate_custom_metadata(const char* metadata) { - if (metadata.empty()) { + if (nullptr == metadata || !*metadata) { return true; // custom metadata is optional } @@ -170,7 +170,7 @@ validate_custom_metadata(std::string_view metadata) ); if (val.is_discarded()) { - LOG_ERROR("Invalid JSON: %s", metadata.data()); + LOG_ERROR("Invalid JSON: %s", metadata); return false; } @@ -227,6 +227,10 @@ validate_settings(const struct ZarrStreamSettings_s* settings) return false; } + if (nullptr == settings->store_path) { + LOG_ERROR("Null pointer: store_path"); + return false; + } std::string_view store_path(settings->store_path); // we require the store path (root of the dataset) to be nonempty @@ -343,6 +347,20 @@ ZarrStream::append(const void* data, size_t nbytes) return 0; } +bool +ZarrStream_s::is_s3_acquisition_() const +{ + return s3_endpoint_.has_value() && s3_bucket_name_.has_value() && + s3_access_key_id_.has_value() && s3_secret_access_key_.has_value(); +} + +bool +ZarrStream_s::is_compressed_acquisition_() const +{ + return compressor_.has_value() && compression_codec_.has_value() && + compression_level_.has_value() && compression_shuffle_.has_value(); +} + void ZarrStream_s::commit_settings_(const struct ZarrStreamSettings_s* settings) { @@ -355,7 +373,6 @@ ZarrStream_s::commit_settings_(const struct ZarrStreamSettings_s* settings) s3_bucket_name_ = trim(settings->s3_settings->bucket_name); s3_access_key_id_ = trim(settings->s3_settings->access_key_id); s3_secret_access_key_ = trim(settings->s3_settings->secret_access_key); - is_s3_acquisition_ = true; } if (is_compressed_acquisition(settings)) { @@ -363,7 +380,6 @@ ZarrStream_s::commit_settings_(const struct ZarrStreamSettings_s* settings) compression_codec_ = settings->compression_settings->codec; compression_level_ = settings->compression_settings->level; compression_shuffle_ = settings->compression_settings->shuffle; - is_compressed_acquisition_ = true; } dtype_ = settings->data_type; @@ -387,7 +403,7 @@ ZarrStream_s::set_error_(const std::string& msg) bool ZarrStream_s::create_store_() { - if (is_s3_acquisition_) { + if (is_s3_acquisition_()) { // TODO (aliddell): implement this } else { if (fs::exists(store_path_)) { diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index 14646d59..15947c1e 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -4,6 +4,7 @@ #include // size_t #include // unique_ptr +#include struct ZarrDimension_s { @@ -51,26 +52,27 @@ struct ZarrStream_s std::string store_path_; - bool is_s3_acquisition_; - std::string s3_endpoint_; - std::string s3_bucket_name_; - std::string s3_access_key_id_; - std::string s3_secret_access_key_; + std::optional s3_endpoint_; + std::optional s3_bucket_name_; + std::optional s3_access_key_id_; + std::optional s3_secret_access_key_; std::string custom_metadata_; ZarrDataType dtype_; - bool is_compressed_acquisition_; - ZarrCompressor compressor_; - ZarrCompressionCodec compression_codec_; - uint8_t compression_level_; - uint8_t compression_shuffle_; + std::optional compressor_; + std::optional compression_codec_; + std::optional compression_level_; + std::optional compression_shuffle_; std::vector dimensions_; bool multiscale_; + [[nodiscard]] bool is_s3_acquisition_() const; + [[nodiscard]] bool is_compressed_acquisition_() const; + /** * @brief Copy settings to the stream. * @param settings Struct containing settings to copy. diff --git a/tests/unit-tests/create-stream.cpp b/tests/unit-tests/create-stream.cpp index 63f2839f..c66bfacf 100644 --- a/tests/unit-tests/create-stream.cpp +++ b/tests/unit-tests/create-stream.cpp @@ -14,22 +14,26 @@ configure_stream_dimensions(ZarrStreamSettings* settings) *dim = ZarrDimensionProperties{ .name = "t", - .bytes_of_name = sizeof("t"), .type = ZarrDimensionType_Time, .array_size_px = 100, .chunk_size_px = 10, }; dim = settings->dimensions + 1; - dim->name = "y"; - dim->type = ZarrDimensionType_Space; - dim->array_size_px = 200; - dim->chunk_size_px = 20; + *dim = ZarrDimensionProperties{ + .name = "y", + .type = ZarrDimensionType_Space, + .array_size_px = 200, + .chunk_size_px = 20, + }; dim = settings->dimensions + 2; - dim->name = "x"; - dim->array_size_px = 300; - dim->chunk_size_px = 30; + *dim = ZarrDimensionProperties{ + .name = "x", + .type = ZarrDimensionType_Space, + .array_size_px = 300, + .chunk_size_px = 30, + }; } int @@ -39,6 +43,7 @@ main() ZarrStream* stream; ZarrStreamSettings settings; + memset(&settings, 0, sizeof(settings)); settings.version = ZarrVersion_2; try { From 68dcb03e5be409d010ac654a079f41862e076d5a Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 15:59:30 -0400 Subject: [PATCH 32/39] Remove some redundant code --- src/streaming/stream.settings.cpp | 509 ------------------------------ src/streaming/stream.settings.hh | 35 -- 2 files changed, 544 deletions(-) delete mode 100644 src/streaming/stream.settings.cpp delete mode 100644 src/streaming/stream.settings.hh diff --git a/src/streaming/stream.settings.cpp b/src/streaming/stream.settings.cpp deleted file mode 100644 index ad36f7ba..00000000 --- a/src/streaming/stream.settings.cpp +++ /dev/null @@ -1,509 +0,0 @@ -#include "macros.hh" -#include "stream.settings.hh" -#include "acquire.zarr.h" - -#include -#include - -#include // memcpy, strnlen -#include - -#define SETTINGS_GET_STRING(settings, member) \ - do { \ - if (!settings) { \ - LOG_ERROR("Null pointer: %s", #settings); \ - return nullptr; \ - } \ - return settings->member.c_str(); \ - } while (0) - -namespace fs = std::filesystem; - -namespace { -const size_t zarr_dimension_min = 3; -const size_t zarr_dimension_max = 32; - -[[nodiscard]] -std::string -trim(const char* s, size_t bytes_of_s) -{ - // trim left - std::string trimmed(s, bytes_of_s); - trimmed.erase(trimmed.begin(), - std::find_if(trimmed.begin(), trimmed.end(), [](char c) { - return !std::isspace(c); - })); - - // trim right - trimmed.erase(std::find_if(trimmed.rbegin(), - trimmed.rend(), - [](char c) { return !std::isspace(c); }) - .base(), - trimmed.end()); - - return trimmed; -} - -[[nodiscard]] -bool -validate_s3_settings(const ZarrS3Settings* settings) -{ - size_t len; - - if (len = strnlen(settings->endpoint, settings->bytes_of_endpoint); - len == 0) { - LOG_ERROR("S3 endpoint is empty"); - return false; - } - - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html - if (len = strnlen(settings->bucket_name, settings->bytes_of_bucket_name); - len < 4 || len > 64) { - LOG_ERROR("Invalid length for S3 bucket name: %zu. Must be between 3 " - "and 63 characters", - len); - return false; - } - - if (len = - strnlen(settings->access_key_id, settings->bytes_of_access_key_id); - len == 0) { - LOG_ERROR("S3 access key ID is empty"); - return false; - } - - if (len = strnlen(settings->secret_access_key, - settings->bytes_of_secret_access_key); - len == 0) { - LOG_ERROR("S3 secret access key is empty"); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_filesystem_store_path(std::string_view data_root) -{ - fs::path path(data_root); - fs::path parent_path = path.parent_path(); - if (parent_path.empty()) { - parent_path = "."; - } - - // parent path must exist and be a directory - if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { - LOG_ERROR("Parent path '%s' does not exist or is not a directory", - parent_path.c_str()); - return false; - } - - // parent path must be writable - const auto perms = fs::status(parent_path).permissions(); - const bool is_writable = - (perms & (fs::perms::owner_write | fs::perms::group_write | - fs::perms::others_write)) != fs::perms::none; - - if (!is_writable) { - LOG_ERROR("Parent path '%s' is not writable", parent_path.c_str()); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_compression_settings(const ZarrCompressionSettings* settings) -{ - if (settings->compressor >= ZarrCompressorCount) { - LOG_ERROR("Invalid compressor: %d", settings->compressor); - return false; - } - - if (settings->codec >= ZarrCompressionCodecCount) { - LOG_ERROR("Invalid compression codec: %d", settings->codec); - return false; - } - - // if compressing, we require a compression codec - if (settings->compressor != ZarrCompressor_None && - settings->codec == ZarrCompressionCodec_None) { - LOG_ERROR("Compression codec must be set when using a compressor"); - return false; - } - - if (settings->level > 9) { - LOG_ERROR("Invalid compression level: %d. Must be between 0 and 9", - settings->level); - return false; - } - - if (settings->shuffle != BLOSC_NOSHUFFLE && - settings->shuffle != BLOSC_SHUFFLE && - settings->shuffle != BLOSC_BITSHUFFLE) { - LOG_ERROR("Invalid shuffle: %d. Must be %d (no shuffle), %d (byte " - "shuffle), or %d (bit shuffle)", - settings->shuffle, - BLOSC_NOSHUFFLE, - BLOSC_SHUFFLE, - BLOSC_BITSHUFFLE); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_dimension(const ZarrDimensionProperties* dimension) -{ - std::string trimmed = trim(dimension->name, dimension->bytes_of_name - 1); - if (trimmed.empty()) { - LOG_ERROR("Invalid name. Must not be empty"); - return false; - } - - if (dimension->type >= ZarrDimensionTypeCount) { - LOG_ERROR("Invalid dimension type: %d", dimension->type); - return false; - } - - if (dimension->chunk_size_px == 0) { - LOG_ERROR("Invalid chunk size: %zu", dimension->chunk_size_px); - return false; - } - - return true; -} -} // namespace - -ZarrStreamSettings_s::ZarrStreamSettings_s() - : store_path() - , s3_endpoint() - , s3_bucket_name() - , s3_access_key_id() - , s3_secret_access_key() - , custom_metadata("{}") - , dtype(ZarrDataType_uint8) - , compressor(ZarrCompressor_None) - , compression_codec(ZarrCompressionCodec_None) - , compression_level(0) - , compression_shuffle(BLOSC_NOSHUFFLE) - , dimensions() - , multiscale(false) -{ -} - -extern "C" -{ - ZarrStreamSettings* ZarrStreamSettings_create() - { - try { - return new ZarrStreamSettings(); - } catch (const std::bad_alloc&) { - return nullptr; - } - } - - void ZarrStreamSettings_destroy(ZarrStreamSettings* settings) - { - delete settings; - } - - ZarrStreamSettings* ZarrStreamSettings_copy( - const ZarrStreamSettings* settings) - { - if (!settings) { - LOG_ERROR("Null pointer: settings"); - return nullptr; - } - - ZarrStreamSettings* copy = ZarrStreamSettings_create(); - if (!copy) { - LOG_ERROR("Failed to allocate memory for copy"); - return nullptr; - } - - *copy = *settings; - - return copy; - } - - /* Setters */ - ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, - const char* store_path, - size_t bytes_of_store_path, - const ZarrS3Settings* s3_settings) - { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); - - bytes_of_store_path = strnlen(store_path, bytes_of_store_path); - EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, - "Invalid store path. Must not be empty"); - - std::string_view store_path_sv(store_path, bytes_of_store_path); - if (store_path_sv.empty()) { - LOG_ERROR("Invalid store path. Must not be empty"); - return ZarrStatus_InvalidArgument; - } - - if (nullptr != s3_settings) { - if (!validate_s3_settings(s3_settings)) { - return ZarrStatus_InvalidArgument; - } - } else if (!validate_filesystem_store_path(store_path)) { - return ZarrStatus_InvalidArgument; - } - - if (nullptr != s3_settings) { - settings->s3_endpoint = s3_settings->endpoint; - settings->s3_bucket_name = s3_settings->bucket_name; - settings->s3_access_key_id = s3_settings->access_key_id; - settings->s3_secret_access_key = s3_settings->secret_access_key; - } - - settings->store_path = store_path; - - return ZarrStatus_Success; - } - - ZarrStatus ZarrStreamSettings_set_compression( - ZarrStreamSettings* settings, - const ZarrCompressionSettings* compression_settings) - { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(compression_settings, - "Null pointer: compression_settings"); - - if (!validate_compression_settings(compression_settings)) { - return ZarrStatus_InvalidArgument; - } - - settings->compressor = compression_settings->compressor; - settings->compression_codec = compression_settings->codec; - settings->compression_level = compression_settings->level; - settings->compression_shuffle = compression_settings->shuffle; - - return ZarrStatus_Success; - } - - ZarrStatus ZarrStreamSettings_set_custom_metadata( - ZarrStreamSettings* settings, - const char* external_metadata, - size_t bytes_of_external_metadata) - { - if (!settings) { - LOG_ERROR("Null pointer: settings"); - return ZarrStatus_InvalidArgument; - } - - if (!external_metadata) { - LOG_ERROR("Null pointer: custom_metadata"); - return ZarrStatus_InvalidArgument; - } - - if (bytes_of_external_metadata == 0) { - LOG_ERROR("Invalid length: %zu. Must be greater than 0", - bytes_of_external_metadata); - return ZarrStatus_InvalidArgument; - } - - size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); - if (nbytes < 2) { - settings->custom_metadata = "{}"; - return ZarrStatus_Success; - } - - auto val = nlohmann::json::parse(external_metadata, - external_metadata + nbytes, - nullptr, // callback - false, // allow exceptions - true // ignore comments - ); - - if (val.is_discarded()) { - LOG_ERROR("Invalid JSON: %s", external_metadata); - return ZarrStatus_InvalidArgument; - } - settings->custom_metadata = val.dump(); - - return ZarrStatus_Success; - } - - ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, - ZarrDataType data_type) - { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT( - data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); - - settings->dtype = data_type; - return ZarrStatus_Success; - } - - ZarrStatus ZarrStreamSettings_reserve_dimensions( - ZarrStreamSettings* settings, - size_t count) - { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT( - count >= zarr_dimension_min && count <= zarr_dimension_max, - "Invalid count: %zu. Count must be between %d and %d", - count, - zarr_dimension_min, - zarr_dimension_max); - - settings->dimensions.resize(count); - return ZarrStatus_Success; - } - - ZarrStatus ZarrStreamSettings_set_dimension( - ZarrStreamSettings* settings, - size_t index, - const ZarrDimensionProperties* dimension) - { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); - EXPECT_VALID_INDEX(index < settings->dimensions.size(), - "Invalid index: %zu. Must be less than %zu", - index, - settings->dimensions.size()); - - if (!validate_dimension(dimension)) { - return ZarrStatus_InvalidArgument; - } - - struct ZarrDimension_s& dim = settings->dimensions[index]; - - dim.name = trim(dimension->name, dimension->bytes_of_name - 1); - dim.type = dimension->type; - dim.array_size_px = dimension->array_size_px; - dim.chunk_size_px = dimension->chunk_size_px; - dim.shard_size_chunks = dimension->shard_size_chunks; - - return ZarrStatus_Success; - } - - ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, - uint8_t multiscale) - { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - - settings->multiscale = multiscale > 0; - return ZarrStatus_Success; - } - - /* Getters */ - const char* ZarrStreamSettings_get_store_path( - const ZarrStreamSettings* settings) - { - SETTINGS_GET_STRING(settings, store_path); - } - - ZarrS3Settings ZarrStreamSettings_get_s3_settings( - const ZarrStreamSettings* settings) - { - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); - return {}; - } - - ZarrS3Settings s3_settings = { - settings->s3_endpoint.c_str(), - settings->s3_endpoint.length() + 1, - settings->s3_bucket_name.c_str(), - settings->s3_bucket_name.length() + 1, - settings->s3_access_key_id.c_str(), - settings->s3_access_key_id.length() + 1, - settings->s3_secret_access_key.c_str(), - settings->s3_secret_access_key.length() + 1, - }; - return s3_settings; - } - - const char* ZarrStreamSettings_get_custom_metadata( - const ZarrStreamSettings* settings) - { - SETTINGS_GET_STRING(settings, custom_metadata); - } - - ZarrDataType ZarrStreamSettings_get_data_type( - const ZarrStreamSettings* settings) - { - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); - return ZarrDataType_uint8; - } - return static_cast(settings->dtype); - } - - ZarrCompressionSettings ZarrStreamSettings_get_compression( - const ZarrStreamSettings* settings) - { - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty compression."); - return {}; - } - - ZarrCompressionSettings compression = { - .compressor = settings->compressor, - .codec = - static_cast(settings->compression_codec), - .level = settings->compression_level, - .shuffle = settings->compression_shuffle, - }; - return compression; - } - - size_t ZarrStreamSettings_get_dimension_count( - const ZarrStreamSettings* settings) - { - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning 0."); - return 0; - } - return settings->dimensions.size(); - } - - ZarrDimensionProperties ZarrStreamSettings_get_dimension( - const ZarrStreamSettings* settings, - size_t index) - { - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty dimension."); - return {}; - } - - if (index >= settings->dimensions.size()) { - LOG_ERROR("Invalid index: %zu. Must be less than %zu", - index, - settings->dimensions.size()); - return {}; - } - - const auto& dim = settings->dimensions[index]; - - ZarrDimensionProperties dimension = { - .name = dim.name.c_str(), - .bytes_of_name = dim.name.size() + 1, - .type = dim.type, - .array_size_px = dim.array_size_px, - .chunk_size_px = dim.chunk_size_px, - .shard_size_chunks = dim.shard_size_chunks, - }; - - return dimension; - } - - bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) - { - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning false."); - return false; - } - return settings->multiscale; - } -} \ No newline at end of file diff --git a/src/streaming/stream.settings.hh b/src/streaming/stream.settings.hh deleted file mode 100644 index da25a9b0..00000000 --- a/src/streaming/stream.settings.hh +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "zarr.types.h" - -#include // size_t -#include // uint8_t -#include -#include - -struct ZarrStreamSettings_s -{ - public: - ZarrStreamSettings_s(); - std::string store_path; /* Path to the Zarr store on the local filesystem */ - - std::string s3_endpoint; /* Endpoint for the S3 service */ - std::string s3_bucket_name; /* Name of the S3 bucket */ - std::string s3_access_key_id; /* Access key ID for the S3 service */ - std::string s3_secret_access_key; /* Secret access key for the S3 service */ - - std::string custom_metadata; /* JSON formatted external metadata for the - base array */ - - ZarrDataType dtype; /* Data type of the base array */ - - ZarrCompressor compressor; /* Compression library to use */ - ZarrCompressionCodec compression_codec; /* Compression codec to use */ - uint8_t compression_level; /* Compression level to use */ - uint8_t compression_shuffle; /* Whether and how to shuffle the data before - compressing */ - - std::vector dimensions; /* Dimensions of the base array */ - - bool multiscale; /* Whether to stream to multiple resolutions */ -}; From d934260de11ac2bcd4deaa89f14a728ca5db30df Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Mon, 23 Sep 2024 10:32:44 -0400 Subject: [PATCH 33/39] Use string_view --- src/streaming/array.writer.hh | 2 +- src/streaming/zarr.stream.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/streaming/array.writer.hh b/src/streaming/array.writer.hh index e04eac61..6be9d599 100644 --- a/src/streaming/array.writer.hh +++ b/src/streaming/array.writer.hh @@ -1,6 +1,6 @@ #pragma once -#include "stream.settings.hh" +#include "zarr.stream.hh" #include "thread.pool.hh" #include "s3.connection.hh" #include "blosc.compression.params.hh" diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index fa4d5f74..0f6bdbb1 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -11,7 +11,7 @@ struct ZarrDimension_s { public: - ZarrDimension_s(const char* name, + ZarrDimension_s(std::string_view name, ZarrDimensionType type, uint32_t array_size_px, uint32_t chunk_size_px, From 7a61ded43ddaa5bb8cfbba8f2eef330aaa2d9d4c Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Thu, 3 Oct 2024 11:30:00 -0400 Subject: [PATCH 34/39] Respond to PR comments --- src/streaming/array.writer.cpp | 15 ++++ src/streaming/array.writer.hh | 4 + src/streaming/zarrv2.array.writer.cpp | 75 ++++++++++--------- src/streaming/zarrv2.array.writer.hh | 12 +-- tests/unit-tests/zarrv2-writer-write-even.cpp | 10 ++- .../zarrv2-writer-write-ragged-append-dim.cpp | 10 ++- ...arrv2-writer-write-ragged-internal-dim.cpp | 10 ++- 7 files changed, 87 insertions(+), 49 deletions(-) diff --git a/src/streaming/array.writer.cpp b/src/streaming/array.writer.cpp index 1c0275c4..b51c9746 100644 --- a/src/streaming/array.writer.cpp +++ b/src/streaming/array.writer.cpp @@ -436,3 +436,18 @@ zarr::ArrayWriter::rollover_() close_sinks_(); ++append_chunk_index_; } + +bool +zarr::finalize_array(std::unique_ptr&& writer) +{ + writer->is_finalizing_ = true; + try { + writer->flush_(); + } catch (const std::exception& exc) { + LOG_ERROR("Failed to finalize array writer: ", exc.what()); + return false; + } + + writer.reset(); + return true; +} diff --git a/src/streaming/array.writer.hh b/src/streaming/array.writer.hh index 0eb19681..8945dabb 100644 --- a/src/streaming/array.writer.hh +++ b/src/streaming/array.writer.hh @@ -99,5 +99,9 @@ class ArrayWriter [[nodiscard]] virtual bool write_array_metadata_() = 0; void close_sinks_(); + + friend bool finalize_array(std::unique_ptr&& writer); }; + +bool finalize_array(std::unique_ptr&& writer); } // namespace zarr diff --git a/src/streaming/zarrv2.array.writer.cpp b/src/streaming/zarrv2.array.writer.cpp index 1c1785ae..cbf06618 100644 --- a/src/streaming/zarrv2.array.writer.cpp +++ b/src/streaming/zarrv2.array.writer.cpp @@ -9,8 +9,9 @@ #include namespace { -std::string -sample_type_to_dtype(ZarrDataType t) +[[nodiscard]] +bool +sample_type_to_dtype(ZarrDataType t, std::string& t_str) { const std::string dtype_prefix = @@ -18,56 +19,57 @@ sample_type_to_dtype(ZarrDataType t) switch (t) { case ZarrDataType_uint8: - return dtype_prefix + "u1"; + t_str = dtype_prefix + "u1"; + break; case ZarrDataType_uint16: - return dtype_prefix + "u2"; + t_str = dtype_prefix + "u2"; + break; case ZarrDataType_uint32: - return dtype_prefix + "u4"; + t_str = dtype_prefix + "u4"; + break; case ZarrDataType_uint64: - return dtype_prefix + "u8"; + t_str = dtype_prefix + "u8"; + break; case ZarrDataType_int8: - return dtype_prefix + "i1"; + t_str = dtype_prefix + "i1"; + break; case ZarrDataType_int16: - return dtype_prefix + "i2"; + t_str = dtype_prefix + "i2"; + break; case ZarrDataType_int32: - return dtype_prefix + "i4"; + t_str = dtype_prefix + "i4"; + break; case ZarrDataType_int64: - return dtype_prefix + "i8"; + t_str = dtype_prefix + "i8"; + break; case ZarrDataType_float32: - return dtype_prefix + "f4"; + t_str = dtype_prefix + "f4"; + break; case ZarrDataType_float64: - return dtype_prefix + "f8"; + t_str = dtype_prefix + "f8"; + break; default: - throw std::runtime_error("Invalid data type: " + - std::to_string(static_cast(t))); + LOG_ERROR("Unsupported sample type: ", t); + return false; } + + return true; } } // namespace zarr::ZarrV2ArrayWriter::ZarrV2ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool) - : ArrayWriter(config, thread_pool, s3_connection_pool) + ArrayWriterConfig&& config, + std::shared_ptr thread_pool) + : ArrayWriter(std::move(config), thread_pool) { } -zarr::ZarrV2ArrayWriter::~ZarrV2ArrayWriter() -{ - is_finalizing_ = true; - try { - flush_(); - } catch (const std::exception& exc) { - LOG_ERROR("Failed to finalize array writer: ", exc.what()); - } catch (...) { - LOG_ERROR("Failed to finalize array writer: (unknown)"); - } -} - -ZarrVersion -zarr::ZarrV2ArrayWriter::version_() const +zarr::ZarrV2ArrayWriter::ZarrV2ArrayWriter( + ArrayWriterConfig&& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool) + : ArrayWriter(std::move(config), thread_pool, s3_connection_pool) { - return ZarrVersion_2; } bool @@ -127,6 +129,11 @@ zarr::ZarrV2ArrayWriter::write_array_metadata_() using json = nlohmann::json; + std::string dtype; + if (!sample_type_to_dtype(config_.dtype, dtype)) { + return false; + } + std::vector array_shape, chunk_shape; size_t append_size = frames_written_; @@ -149,7 +156,7 @@ zarr::ZarrV2ArrayWriter::write_array_metadata_() metadata["zarr_format"] = 2; metadata["shape"] = array_shape; metadata["chunks"] = chunk_shape; - metadata["dtype"] = sample_type_to_dtype(config_.dtype); + metadata["dtype"] = dtype; metadata["fill_value"] = 0; metadata["order"] = "C"; metadata["filters"] = nullptr; diff --git a/src/streaming/zarrv2.array.writer.hh b/src/streaming/zarrv2.array.writer.hh index c756ad75..b96b1152 100644 --- a/src/streaming/zarrv2.array.writer.hh +++ b/src/streaming/zarrv2.array.writer.hh @@ -6,15 +6,15 @@ namespace zarr { class ZarrV2ArrayWriter final : public ArrayWriter { public: - ZarrV2ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool); + ZarrV2ArrayWriter(ArrayWriterConfig&& config, + std::shared_ptr thread_pool); - ~ZarrV2ArrayWriter() override; + ZarrV2ArrayWriter(ArrayWriterConfig&& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool); private: - ZarrVersion version_() const override; + ZarrVersion version_() const override { return ZarrVersion_2; }; bool flush_impl_() override; bool write_array_metadata_() override; bool should_rollover_() const override; diff --git a/tests/unit-tests/zarrv2-writer-write-even.cpp b/tests/unit-tests/zarrv2-writer-write-even.cpp index 2ca77c29..d5b2b87d 100644 --- a/tests/unit-tests/zarrv2-writer-write-even.cpp +++ b/tests/unit-tests/zarrv2-writer-write-even.cpp @@ -107,14 +107,18 @@ main() }; { - zarr::ZarrV2ArrayWriter writer(config, thread_pool, nullptr); + auto writer = std::make_unique( + std::move(config), thread_pool); const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); + std::vector data_(frame_size, std::byte(0)); + std::span data(data_); for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer.write_frame(data.data(), frame_size)); + CHECK(writer->write_frame(data)); } + + CHECK(finalize_array(std::move(writer))); } check_json(); diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp index fbadb5e8..340bc93c 100644 --- a/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp +++ b/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp @@ -91,14 +91,18 @@ main() }; { - zarr::ZarrV2ArrayWriter writer(config, thread_pool, nullptr); + auto writer = std::make_unique( + std::move(config), thread_pool); const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); + std::vector data_(frame_size, std::byte(0)); + std::span data(data_); for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer.write_frame(data.data(), frame_size)); + CHECK(writer->write_frame(data)); } + + CHECK(finalize_array(std::move(writer))); } check_json(); diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp index ef22b368..1299d898 100644 --- a/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp +++ b/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp @@ -98,14 +98,18 @@ main() }; { - zarr::ZarrV2ArrayWriter writer(config, thread_pool, nullptr); + auto writer = std::make_unique( + std::move(config), thread_pool); const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); + std::vector data_(frame_size, std::byte(0)); + std::span data(data_); for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer.write_frame(data.data(), frame_size)); + CHECK(writer->write_frame(data)); } + + CHECK(finalize_array(std::move(writer))); } check_json(); From c455ad799444fb358b4b70da0dfc1512b74b4dfb Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Thu, 3 Oct 2024 11:46:06 -0400 Subject: [PATCH 35/39] Put #include back --- src/streaming/zarr.dimension.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/streaming/zarr.dimension.hh b/src/streaming/zarr.dimension.hh index 615c8db0..9c0174c0 100644 --- a/src/streaming/zarr.dimension.hh +++ b/src/streaming/zarr.dimension.hh @@ -2,6 +2,7 @@ #include "zarr.types.h" +#include #include #include From e3ea0d3ee4ac157df89b6d27fdf4cd465f9dfef2 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Thu, 3 Oct 2024 14:05:29 -0400 Subject: [PATCH 36/39] Respond to PR comments. --- src/streaming/array.writer.cpp | 2 - src/streaming/sink.creator.cpp | 6 - src/streaming/zarrv2.array.writer.cpp | 2 - src/streaming/zarrv3.array.writer.cpp | 101 ++++----- src/streaming/zarrv3.array.writer.hh | 4 +- .../common-chunk-internal-offset.cpp | 184 ----------------- .../unit-tests/common-chunk-lattice-index.cpp | 83 -------- .../common-shard-index-for-chunk.cpp | 195 ------------------ .../common-shard-internal-index.cpp | 75 ------- tests/unit-tests/common-tile-group-offset.cpp | 108 ---------- tests/unit-tests/zarrv3-writer-write-even.cpp | 67 +++--- .../zarrv3-writer-write-ragged-append-dim.cpp | 52 ++--- ...arrv3-writer-write-ragged-internal-dim.cpp | 60 +++--- 13 files changed, 149 insertions(+), 790 deletions(-) delete mode 100644 tests/unit-tests/common-chunk-internal-offset.cpp delete mode 100644 tests/unit-tests/common-chunk-lattice-index.cpp delete mode 100644 tests/unit-tests/common-shard-index-for-chunk.cpp delete mode 100644 tests/unit-tests/common-shard-internal-index.cpp delete mode 100644 tests/unit-tests/common-tile-group-offset.cpp diff --git a/src/streaming/array.writer.cpp b/src/streaming/array.writer.cpp index b51c9746..496625df 100644 --- a/src/streaming/array.writer.cpp +++ b/src/streaming/array.writer.cpp @@ -381,8 +381,6 @@ zarr::ArrayWriter::compress_buffers_() } catch (const std::exception& exc) { err = "Failed to compress chunk: " + std::string(exc.what()); - } catch (...) { - err = "Failed to compress chunk (unknown)"; } latch.count_down(); diff --git a/src/streaming/sink.creator.cpp b/src/streaming/sink.creator.cpp index 7d2b4e6c..d1343c2c 100644 --- a/src/streaming/sink.creator.cpp +++ b/src/streaming/sink.creator.cpp @@ -342,9 +342,6 @@ zarr::SinkCreator::make_files_(std::queue& file_paths, } catch (const std::exception& exc) { err = "Failed to create file '" + filename + "': " + exc.what(); - } catch (...) { - err = "Failed to create file '" + filename + - "': (unknown)."; } latch.count_down(); @@ -396,9 +393,6 @@ zarr::SinkCreator::make_files_( } catch (const std::exception& exc) { err = "Failed to create file '" + filename + "': " + exc.what(); - } catch (...) { - err = "Failed to create file '" + filename + - "': (unknown)."; } latch.count_down(); diff --git a/src/streaming/zarrv2.array.writer.cpp b/src/streaming/zarrv2.array.writer.cpp index cbf06618..b93b4aaa 100644 --- a/src/streaming/zarrv2.array.writer.cpp +++ b/src/streaming/zarrv2.array.writer.cpp @@ -103,8 +103,6 @@ zarr::ZarrV2ArrayWriter::flush_impl_() } catch (const std::exception& exc) { err = "Failed to write chunk: " + std::string(exc.what()); - } catch (...) { - err = "Failed to write chunk: (unknown)"; } latch.count_down(); diff --git a/src/streaming/zarrv3.array.writer.cpp b/src/streaming/zarrv3.array.writer.cpp index 84f3314d..7c5119b0 100644 --- a/src/streaming/zarrv3.array.writer.cpp +++ b/src/streaming/zarrv3.array.writer.cpp @@ -47,17 +47,26 @@ sample_type_to_dtype(ZarrDataType t) } // namespace zarr::ZarrV3ArrayWriter::ZarrV3ArrayWriter( - const ArrayWriterConfig& array_spec, + ArrayWriterConfig&& config, + std::shared_ptr thread_pool) + : ZarrV3ArrayWriter(std::move(config), thread_pool, nullptr) +{ +} + +zarr::ZarrV3ArrayWriter::ZarrV3ArrayWriter( + ArrayWriterConfig&& config, std::shared_ptr thread_pool, std::shared_ptr s3_connection_pool) - : ArrayWriter(array_spec, thread_pool, s3_connection_pool) - , shard_file_offsets_(number_of_shards(array_spec.dimensions), 0) - , shard_tables_{ number_of_shards(array_spec.dimensions) } + : ArrayWriter(std::move(config), thread_pool, s3_connection_pool) { - const auto cps = chunks_per_shard(array_spec.dimensions); + const auto number_of_shards = config_.dimensions->number_of_shards(); + const auto chunks_per_shard = config_.dimensions->chunks_per_shard(); + + shard_file_offsets_.resize(number_of_shards, 0); + shard_tables_.resize(number_of_shards); for (auto& table : shard_tables_) { - table.resize(2 * cps); + table.resize(2 * chunks_per_shard); std::fill_n( table.begin(), table.size(), std::numeric_limits::max()); } @@ -70,8 +79,6 @@ zarr::ZarrV3ArrayWriter::~ZarrV3ArrayWriter() flush_(); } catch (const std::exception& exc) { LOG_ERROR("Failed to finalize array writer: %s", exc.what()); - } catch (...) { - LOG_ERROR("Failed to finalize array writer: (unknown)"); } } @@ -89,13 +96,13 @@ zarr::ZarrV3ArrayWriter::flush_impl_() return false; } - const auto n_shards = number_of_shards(config_.dimensions); + const auto n_shards = config_.dimensions->number_of_shards(); CHECK(data_sinks_.size() == n_shards); // get shard indices for each chunk std::vector> chunk_in_shards(n_shards); for (auto i = 0; i < chunk_buffers_.size(); ++i) { - const auto index = shard_index_for_chunk(i, config_.dimensions); + const auto index = config_.dimensions->shard_index_for_chunk(i); chunk_in_shards.at(index).push_back(i); } @@ -107,27 +114,27 @@ zarr::ZarrV3ArrayWriter::flush_impl_() auto& chunk_table = shard_tables_.at(i); size_t* file_offset = &shard_file_offsets_.at(i); - EXPECT(thread_pool_->push_to_job_queue([&sink = data_sinks_.at(i), - &chunks, - &chunk_table, - file_offset, - write_table, - &latch, - this]( - std::string& err) mutable { + EXPECT(thread_pool_->push_job([&sink = data_sinks_.at(i), + &chunks, + &chunk_table, + file_offset, + write_table, + &latch, + this](std::string& err) mutable { bool success = false; try { for (const auto& chunk_idx : chunks) { auto& chunk = chunk_buffers_.at(chunk_idx); - success = - sink->write(*file_offset, chunk.data(), chunk.size()); + std::span data{ reinterpret_cast(chunk.data()), + chunk.size() }; + success = sink->write(*file_offset, data); if (!success) { break; } const auto internal_idx = - shard_internal_index(chunk_idx, config_.dimensions); + config_.dimensions->shard_internal_index(chunk_idx); chunk_table.at(2 * internal_idx) = *file_offset; chunk_table.at(2 * internal_idx + 1) = chunk.size(); @@ -135,17 +142,14 @@ zarr::ZarrV3ArrayWriter::flush_impl_() } if (success && write_table) { - const auto* table = - reinterpret_cast(chunk_table.data()); - success = - sink->write(*file_offset, - table, - chunk_table.size() * sizeof(uint64_t)); + auto* table = + reinterpret_cast(chunk_table.data()); + std::span data{ table, + chunk_table.size() * sizeof(uint64_t) }; + success = sink->write(*file_offset, data); } } catch (const std::exception& exc) { err = "Failed to write chunk: " + std::string(exc.what()); - } catch (...) { - err = "Failed to write chunk: (unknown)"; } latch.count_down(); @@ -183,22 +187,22 @@ zarr::ZarrV3ArrayWriter::write_array_metadata_() std::vector array_shape, chunk_shape, shard_shape; size_t append_size = frames_written_; - for (auto dim = config_.dimensions.rbegin() + 2; - dim < config_.dimensions.rend() - 1; - ++dim) { - CHECK(dim->array_size_px); - append_size = (append_size + dim->array_size_px - 1) / dim->array_size_px; + for (auto i = config_.dimensions->ndims() - 3; i > 0; --i) { + const auto& dim = config_.dimensions->at(i); + const auto& array_size_px = dim.array_size_px; + CHECK(array_size_px); + append_size = (append_size + array_size_px - 1) / array_size_px; } array_shape.push_back(append_size); - chunk_shape.push_back(config_.dimensions.front().chunk_size_px); - shard_shape.push_back(config_.dimensions.front().shard_size_chunks); - for (auto dim = config_.dimensions.begin() + 1; - dim != config_.dimensions.end(); - ++dim) { - array_shape.push_back(dim->array_size_px); - chunk_shape.push_back(dim->chunk_size_px); - shard_shape.push_back(dim->shard_size_chunks); + const auto& final_dim = config_.dimensions->final_dim(); + chunk_shape.push_back(final_dim.chunk_size_px); + shard_shape.push_back(final_dim.shard_size_chunks); + for (auto i = 1; i < config_.dimensions->ndims(); ++i) { + const auto& dim = config_.dimensions->at(i); + array_shape.push_back(dim.array_size_px); + chunk_shape.push_back(dim.chunk_size_px); + shard_shape.push_back(dim.shard_size_chunks); } json metadata; @@ -245,21 +249,22 @@ zarr::ZarrV3ArrayWriter::write_array_metadata_() }) }, }); - const std::string metadata_str = metadata.dump(4); - const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str(); + std::string metadata_str = metadata.dump(4); + std::span data = { reinterpret_cast(metadata_str.data()), + metadata_str.size() }; - return metadata_sink_->write(0, metadata_bytes, metadata_str.size()); + return metadata_sink_->write(0, data); } bool zarr::ZarrV3ArrayWriter::should_rollover_() const { const auto& dims = config_.dimensions; - const auto& append_dim = dims.front(); + const auto& append_dim = dims->final_dim(); size_t frames_before_flush = append_dim.chunk_size_px * append_dim.shard_size_chunks; - for (auto i = 1; i < dims.size() - 2; ++i) { - frames_before_flush *= dims[i].array_size_px; + for (auto i = 1; i < dims->ndims() - 2; ++i) { + frames_before_flush *= dims->at(i).array_size_px; } CHECK(frames_before_flush > 0); diff --git a/src/streaming/zarrv3.array.writer.hh b/src/streaming/zarrv3.array.writer.hh index 9a123050..1015ca13 100644 --- a/src/streaming/zarrv3.array.writer.hh +++ b/src/streaming/zarrv3.array.writer.hh @@ -6,8 +6,10 @@ namespace zarr { struct ZarrV3ArrayWriter final : public ArrayWriter { public: + ZarrV3ArrayWriter(ArrayWriterConfig&& config, + std::shared_ptr thread_pool); ZarrV3ArrayWriter( - const ArrayWriterConfig& array_spec, + ArrayWriterConfig&& config, std::shared_ptr thread_pool, std::shared_ptr s3_connection_pool); diff --git a/tests/unit-tests/common-chunk-internal-offset.cpp b/tests/unit-tests/common-chunk-internal-offset.cpp deleted file mode 100644 index ff45eb01..00000000 --- a/tests/unit-tests/common-chunk-internal-offset.cpp +++ /dev/null @@ -1,184 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - - try { - EXPECT_INT_EQ(zarr::chunk_internal_offset(0, dims, ZarrDataType_uint16), - 0); - EXPECT_INT_EQ(zarr::chunk_internal_offset(1, dims, ZarrDataType_uint16), - 512); - EXPECT_INT_EQ(zarr::chunk_internal_offset(2, dims, ZarrDataType_uint16), - 0); - EXPECT_INT_EQ(zarr::chunk_internal_offset(3, dims, ZarrDataType_uint16), - 512); - EXPECT_INT_EQ(zarr::chunk_internal_offset(4, dims, ZarrDataType_uint16), - 0); - EXPECT_INT_EQ(zarr::chunk_internal_offset(5, dims, ZarrDataType_uint16), - 1024); - EXPECT_INT_EQ(zarr::chunk_internal_offset(6, dims, ZarrDataType_uint16), - 1536); - EXPECT_INT_EQ(zarr::chunk_internal_offset(7, dims, ZarrDataType_uint16), - 1024); - EXPECT_INT_EQ(zarr::chunk_internal_offset(8, dims, ZarrDataType_uint16), - 1536); - EXPECT_INT_EQ(zarr::chunk_internal_offset(9, dims, ZarrDataType_uint16), - 1024); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(10, dims, ZarrDataType_uint16), 0); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(11, dims, ZarrDataType_uint16), 512); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(12, dims, ZarrDataType_uint16), 0); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(13, dims, ZarrDataType_uint16), 512); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(14, dims, ZarrDataType_uint16), 0); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(15, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(16, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(17, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(18, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(19, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(20, dims, ZarrDataType_uint16), 3072); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(21, dims, ZarrDataType_uint16), 3584); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(22, dims, ZarrDataType_uint16), 3072); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(23, dims, ZarrDataType_uint16), 3584); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(24, dims, ZarrDataType_uint16), 3072); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(25, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(26, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(27, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(28, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(29, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(30, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(31, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(32, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(33, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(34, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(35, dims, ZarrDataType_uint16), 5120); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(36, dims, ZarrDataType_uint16), 5632); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(37, dims, ZarrDataType_uint16), 5120); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(38, dims, ZarrDataType_uint16), 5632); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(39, dims, ZarrDataType_uint16), 5120); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(40, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(41, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(42, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(43, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(44, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(45, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(46, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(47, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(48, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(49, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(50, dims, ZarrDataType_uint16), 7168); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(51, dims, ZarrDataType_uint16), 7680); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(52, dims, ZarrDataType_uint16), 7168); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(53, dims, ZarrDataType_uint16), 7680); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(54, dims, ZarrDataType_uint16), 7168); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(55, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(56, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(57, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(58, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(59, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(60, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(61, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(62, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(63, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(64, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(65, dims, ZarrDataType_uint16), 9216); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(66, dims, ZarrDataType_uint16), 9728); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(67, dims, ZarrDataType_uint16), 9216); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(68, dims, ZarrDataType_uint16), 9728); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(69, dims, ZarrDataType_uint16), 9216); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(70, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(71, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(72, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(73, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(74, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(75, dims, ZarrDataType_uint16), 0); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-chunk-lattice-index.cpp b/tests/unit-tests/common-chunk-lattice-index.cpp deleted file mode 100644 index 3dc94d44..00000000 --- a/tests/unit-tests/common-chunk-lattice-index.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - try { - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - - EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 0, dims), 1); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-index-for-chunk.cpp b/tests/unit-tests/common-shard-index-for-chunk.cpp deleted file mode 100644 index 6c012b43..00000000 --- a/tests/unit-tests/common-shard-index-for-chunk.cpp +++ /dev/null @@ -1,195 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - try { - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - 0, - 5, // 5 timepoints / chunk - 2); // 2 chunks / shard - dims.emplace_back("c", - ZarrDimensionType_Channel, - 8, - 4, // 8 / 4 = 2 chunks - 2); // 4 / 2 = 2 shards - dims.emplace_back("z", - ZarrDimensionType_Space, - 6, - 2, // 6 / 2 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("y", - ZarrDimensionType_Space, - 48, - 16, // 48 / 16 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("x", - ZarrDimensionType_Space, - 64, - 16, // 64 / 16 = 4 chunks - 2); // 4 / 2 = 2 shards - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(12, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(13, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(14, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(15, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(16, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(17, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(18, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(19, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(20, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(21, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(22, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(23, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(24, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(25, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(26, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(27, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(28, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(29, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(30, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(31, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(32, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(33, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(34, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(35, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(36, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(37, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(38, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(39, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(40, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(41, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(42, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(43, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(44, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(45, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(46, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(47, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(48, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(49, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(50, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(51, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(52, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(53, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(54, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(55, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(56, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(57, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(58, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(59, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(60, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(61, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(62, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(63, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(64, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(65, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(66, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(67, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(68, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(69, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(70, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(71, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(72, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(73, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(74, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(75, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(76, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(77, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(78, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(79, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(80, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(81, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(82, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(83, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(84, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(85, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(86, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(87, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(88, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(89, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(90, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(91, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(92, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(93, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(94, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(95, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(96, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(97, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(98, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(99, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(100, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(101, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(102, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(103, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(104, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(105, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(106, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(107, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(108, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(109, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(110, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(111, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(112, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(113, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(114, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(115, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(116, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(117, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(118, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(119, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(120, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(121, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(122, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(123, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(124, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(125, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(126, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(127, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(128, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(129, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(130, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(131, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(132, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(133, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(134, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(135, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(136, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(137, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(138, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(139, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(140, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(141, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(142, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(143, dims), 17); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-internal-index.cpp b/tests/unit-tests/common-shard-internal-index.cpp deleted file mode 100644 index 346c13bf..00000000 --- a/tests/unit-tests/common-shard-internal-index.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - 0, - 32, // 32 timepoints / chunk - 1); // 1 shard - dims.emplace_back("y", - ZarrDimensionType_Space, - 960, - 320, // 3 chunks - 2); // 2 ragged shards - dims.emplace_back("x", - ZarrDimensionType_Space, - 1080, - 270, // 4 chunks - 3); // 2 ragged shards - - try { - EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(0, dims), 0); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(1, dims), 1); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(2, dims), 2); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); - EXPECT_INT_EQ(zarr::shard_internal_index(3, dims), 0); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(4, dims), 3); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(5, dims), 4); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(6, dims), 5); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 1); - EXPECT_INT_EQ(zarr::shard_internal_index(7, dims), 3); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 2); - EXPECT_INT_EQ(zarr::shard_internal_index(8, dims), 0); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 2); - EXPECT_INT_EQ(zarr::shard_internal_index(9, dims), 1); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 2); - EXPECT_INT_EQ(zarr::shard_internal_index(10, dims), 2); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 3); - EXPECT_INT_EQ(zarr::shard_internal_index(11, dims), 0); - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-tile-group-offset.cpp b/tests/unit-tests/common-tile-group-offset.cpp deleted file mode 100644 index be9efecc..00000000 --- a/tests/unit-tests/common-tile-group-offset.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - - try { - EXPECT_INT_EQ(zarr::tile_group_offset(0, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(1, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(2, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(3, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(4, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(5, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(6, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(7, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(8, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(9, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(10, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(11, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(12, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(13, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(14, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(15, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(16, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(17, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(18, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(19, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(20, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(21, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(22, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(23, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(24, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(25, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(26, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(27, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(28, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(29, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(30, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(31, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(32, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(33, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(34, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(35, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(36, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(37, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(38, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(39, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(40, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(41, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(42, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(43, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(44, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(45, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(46, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(47, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(48, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(49, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(50, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(51, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(52, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(53, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(54, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(55, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(56, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(57, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(58, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(59, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(60, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(61, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(62, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(63, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(64, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(65, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(66, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(67, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(68, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(69, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(70, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(71, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(72, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(73, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(74, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(75, dims), 0); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/zarrv3-writer-write-even.cpp b/tests/unit-tests/zarrv3-writer-write-even.cpp index d3414780..08b9449f 100644 --- a/tests/unit-tests/zarrv3-writer-write-even.cpp +++ b/tests/unit-tests/zarrv3-writer-write-even.cpp @@ -1,14 +1,11 @@ #include "zarrv3.array.writer.hh" -#include "macros.hh" +#include "unit.test.macros.hh" #include "zarr.common.hh" #include #include -#define EXPECT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - namespace fs = std::filesystem; namespace { @@ -70,32 +67,32 @@ check_json() const auto& shard_shape = meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - EXPECT_EQ(array_shape.size(), 5); - EXPECT_EQ(array_shape[0].get(), array_timepoints); - EXPECT_EQ(array_shape[1].get(), array_channels); - EXPECT_EQ(array_shape[2].get(), array_planes); - EXPECT_EQ(array_shape[3].get(), array_height); - EXPECT_EQ(array_shape[4].get(), array_width); - - EXPECT_EQ(chunk_shape.size(), 5); - EXPECT_EQ(chunk_shape[0].get(), chunk_timepoints); - EXPECT_EQ(chunk_shape[1].get(), chunk_channels); - EXPECT_EQ(chunk_shape[2].get(), chunk_planes); - EXPECT_EQ(chunk_shape[3].get(), chunk_height); - EXPECT_EQ(chunk_shape[4].get(), chunk_width); - - EXPECT_EQ(shard_shape.size(), 5); - EXPECT_EQ(shard_shape[0].get(), shard_timepoints); - EXPECT_EQ(shard_shape[1].get(), shard_channels); - EXPECT_EQ(shard_shape[2].get(), shard_planes); - EXPECT_EQ(shard_shape[3].get(), shard_height); - EXPECT_EQ(shard_shape[4].get(), shard_width); + EXPECT_EQ(int, array_shape.size(), 5); + EXPECT_EQ(int, array_shape[0].get(), array_timepoints); + EXPECT_EQ(int, array_shape[1].get(), array_channels); + EXPECT_EQ(int, array_shape[2].get(), array_planes); + EXPECT_EQ(int, array_shape[3].get(), array_height); + EXPECT_EQ(int, array_shape[4].get(), array_width); + + EXPECT_EQ(int, chunk_shape.size(), 5); + EXPECT_EQ(int, chunk_shape[0].get(), chunk_timepoints); + EXPECT_EQ(int, chunk_shape[1].get(), chunk_channels); + EXPECT_EQ(int, chunk_shape[2].get(), chunk_planes); + EXPECT_EQ(int, chunk_shape[3].get(), chunk_height); + EXPECT_EQ(int, chunk_shape[4].get(), chunk_width); + + EXPECT_EQ(int, shard_shape.size(), 5); + EXPECT_EQ(int, shard_shape[0].get(), shard_timepoints); + EXPECT_EQ(int, shard_shape[1].get(), shard_channels); + EXPECT_EQ(int, shard_shape[2].get(), shard_planes); + EXPECT_EQ(int, shard_shape[3].get(), shard_height); + EXPECT_EQ(int, shard_shape[4].get(), shard_width); } int main() { - Logger::set_log_level(ZarrLogLevel_Debug); + Logger::set_log_level(LogLevel_Debug); int retval = 1; @@ -107,7 +104,7 @@ main() std::thread::hardware_concurrency(), [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); - std::vector dims; + std::vector dims; dims.emplace_back("t", ZarrDimensionType_Time, array_timepoints, @@ -130,9 +127,11 @@ main() shard_height); dims.emplace_back( "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); + auto dimensions = + std::make_unique(std::move(dims), dtype); zarr::ArrayWriterConfig config = { - .dimensions = dims, + .dimensions = std::move(dimensions), .dtype = dtype, .level_of_detail = level_of_detail, .bucket_name = std::nullopt, @@ -141,14 +140,18 @@ main() }; { - zarr::ZarrV3ArrayWriter writer(config, thread_pool, nullptr); + auto writer = std::make_unique( + std::move(config), thread_pool); const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); + std::vector data_(frame_size, std::byte(0)); + std::span data(data_); - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write_frame(data.data(), frame_size)); + for (auto i = 0; i < n_frames; ++i) { // 2 time points + CHECK(writer->write_frame(data)); } + + CHECK(finalize_array(std::move(writer))); } check_json(); @@ -186,7 +189,7 @@ main() const auto x_file = y_dir / std::to_string(x); CHECK(fs::is_regular_file(x_file)); const auto file_size = fs::file_size(x_file); - EXPECT_EQ(file_size, expected_file_size); + EXPECT_EQ(int, file_size, expected_file_size); } CHECK(!fs::is_regular_file( diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp index 45959819..a3b893c7 100644 --- a/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp +++ b/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp @@ -1,14 +1,11 @@ #include "zarrv3.array.writer.hh" -#include "macros.hh" +#include "unit.test.macros.hh" #include "zarr.common.hh" #include #include -#define EXPECT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - namespace fs = std::filesystem; namespace { @@ -58,26 +55,26 @@ check_json() const auto& shard_shape = meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - EXPECT_EQ(array_shape.size(), 3); - EXPECT_EQ(array_shape[0].get(), array_planes); - EXPECT_EQ(array_shape[1].get(), array_height); - EXPECT_EQ(array_shape[2].get(), array_width); + EXPECT_EQ(int, array_shape.size(), 3); + EXPECT_EQ(int, array_shape[0].get(), array_planes); + EXPECT_EQ(int, array_shape[1].get(), array_height); + EXPECT_EQ(int, array_shape[2].get(), array_width); - EXPECT_EQ(chunk_shape.size(), 3); - EXPECT_EQ(chunk_shape[0].get(), chunk_planes); - EXPECT_EQ(chunk_shape[1].get(), chunk_height); - EXPECT_EQ(chunk_shape[2].get(), chunk_width); + EXPECT_EQ(int, chunk_shape.size(), 3); + EXPECT_EQ(int, chunk_shape[0].get(), chunk_planes); + EXPECT_EQ(int, chunk_shape[1].get(), chunk_height); + EXPECT_EQ(int, chunk_shape[2].get(), chunk_width); - EXPECT_EQ(shard_shape.size(), 3); - EXPECT_EQ(shard_shape[0].get(), shard_planes); - EXPECT_EQ(shard_shape[1].get(), shard_height); - EXPECT_EQ(shard_shape[2].get(), shard_width); + EXPECT_EQ(int, shard_shape.size(), 3); + EXPECT_EQ(int, shard_shape[0].get(), shard_planes); + EXPECT_EQ(int, shard_shape[1].get(), shard_height); + EXPECT_EQ(int, shard_shape[2].get(), shard_width); } int main() { - Logger::set_log_level(ZarrLogLevel_Debug); + Logger::set_log_level(LogLevel_Debug); int retval = 1; @@ -89,7 +86,7 @@ main() std::thread::hardware_concurrency(), [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); - std::vector dims; + std::vector dims; dims.emplace_back("z", ZarrDimensionType_Space, array_planes, @@ -102,9 +99,11 @@ main() shard_height); dims.emplace_back( "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); + auto dimensions = + std::make_unique(std::move(dims), dtype); zarr::ArrayWriterConfig config = { - .dimensions = dims, + .dimensions = std::move(dimensions), .dtype = dtype, .level_of_detail = 4, .bucket_name = std::nullopt, @@ -113,15 +112,18 @@ main() }; { - zarr::ZarrV3ArrayWriter writer(config, thread_pool, nullptr); + auto writer = std::make_unique( + std::move(config), thread_pool); const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); + std::vector data_(frame_size, std::byte(0)); + std::span data(data_); - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write_frame(data.data(), frame_size) == - frame_size); + for (auto i = 0; i < n_frames; ++i) { // 2 time points + CHECK(writer->write_frame(data)); } + + CHECK(finalize_array(std::move(writer))); } check_json(); @@ -149,7 +151,7 @@ main() const auto x_file = y_dir / std::to_string(x); CHECK(fs::is_regular_file(x_file)); const auto file_size = fs::file_size(x_file); - EXPECT_EQ(file_size, expected_file_size); + EXPECT_EQ(int, file_size, expected_file_size); } CHECK( diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp index 2083775f..d05b9504 100644 --- a/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp +++ b/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp @@ -1,14 +1,11 @@ #include "zarrv3.array.writer.hh" -#include "macros.hh" +#include "unit.test.macros.hh" #include "zarr.common.hh" #include #include -#define EXPECT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - namespace fs = std::filesystem; namespace { @@ -66,29 +63,29 @@ check_json() const auto& shard_shape = meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - EXPECT_EQ(array_shape.size(), 4); - EXPECT_EQ(array_shape[0].get(), array_timepoints); - EXPECT_EQ(array_shape[1].get(), array_planes); - EXPECT_EQ(array_shape[2].get(), array_height); - EXPECT_EQ(array_shape[3].get(), array_width); - - EXPECT_EQ(chunk_shape.size(), 4); - EXPECT_EQ(chunk_shape[0].get(), chunk_timepoints); - EXPECT_EQ(chunk_shape[1].get(), chunk_planes); - EXPECT_EQ(chunk_shape[2].get(), chunk_height); - EXPECT_EQ(chunk_shape[3].get(), chunk_width); - - EXPECT_EQ(shard_shape.size(), 4); - EXPECT_EQ(shard_shape[0].get(), shard_timepoints); - EXPECT_EQ(shard_shape[1].get(), shard_planes); - EXPECT_EQ(shard_shape[2].get(), shard_height); - EXPECT_EQ(shard_shape[3].get(), shard_width); + EXPECT_EQ(int, array_shape.size(), 4); + EXPECT_EQ(int, array_shape[0].get(), array_timepoints); + EXPECT_EQ(int, array_shape[1].get(), array_planes); + EXPECT_EQ(int, array_shape[2].get(), array_height); + EXPECT_EQ(int, array_shape[3].get(), array_width); + + EXPECT_EQ(int, chunk_shape.size(), 4); + EXPECT_EQ(int, chunk_shape[0].get(), chunk_timepoints); + EXPECT_EQ(int, chunk_shape[1].get(), chunk_planes); + EXPECT_EQ(int, chunk_shape[2].get(), chunk_height); + EXPECT_EQ(int, chunk_shape[3].get(), chunk_width); + + EXPECT_EQ(int, shard_shape.size(), 4); + EXPECT_EQ(int, shard_shape[0].get(), shard_timepoints); + EXPECT_EQ(int, shard_shape[1].get(), shard_planes); + EXPECT_EQ(int, shard_shape[2].get(), shard_height); + EXPECT_EQ(int, shard_shape[3].get(), shard_width); } int main() { - Logger::set_log_level(ZarrLogLevel_Debug); + Logger::set_log_level(LogLevel_Debug); int retval = 1; @@ -100,7 +97,7 @@ main() std::thread::hardware_concurrency(), [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); - std::vector dims; + std::vector dims; dims.emplace_back("t", ZarrDimensionType_Time, array_timepoints, @@ -118,9 +115,11 @@ main() shard_height); dims.emplace_back( "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); + auto dimensions = + std::make_unique(std::move(dims), dtype); zarr::ArrayWriterConfig config = { - .dimensions = dims, + .dimensions = std::move(dimensions), .dtype = dtype, .level_of_detail = 5, .bucket_name = std::nullopt, @@ -129,15 +128,18 @@ main() }; { - zarr::ZarrV3ArrayWriter writer(config, thread_pool, nullptr); + auto writer = std::make_unique( + std::move(config), thread_pool); const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, 0); + std::vector data_(frame_size, std::byte(0)); + std::span data(data_); - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write_frame(data.data(), frame_size) == - frame_size); + for (auto i = 0; i < n_frames; ++i) { // 2 time points + CHECK(writer->write_frame(data)); } + + CHECK(finalize_array(std::move(writer))); } const auto chunk_size = chunk_width * chunk_height * chunk_planes * From 0e1482a0fd0d42c747b6cf6c2b301b4cf3966011 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Thu, 3 Oct 2024 14:06:19 -0400 Subject: [PATCH 37/39] Remove dead files --- .../common-chunk-internal-offset.cpp | 184 ----------------- .../unit-tests/common-chunk-lattice-index.cpp | 83 -------- .../common-shard-index-for-chunk.cpp | 195 ------------------ .../common-shard-internal-index.cpp | 75 ------- tests/unit-tests/common-tile-group-offset.cpp | 108 ---------- 5 files changed, 645 deletions(-) delete mode 100644 tests/unit-tests/common-chunk-internal-offset.cpp delete mode 100644 tests/unit-tests/common-chunk-lattice-index.cpp delete mode 100644 tests/unit-tests/common-shard-index-for-chunk.cpp delete mode 100644 tests/unit-tests/common-shard-internal-index.cpp delete mode 100644 tests/unit-tests/common-tile-group-offset.cpp diff --git a/tests/unit-tests/common-chunk-internal-offset.cpp b/tests/unit-tests/common-chunk-internal-offset.cpp deleted file mode 100644 index ff45eb01..00000000 --- a/tests/unit-tests/common-chunk-internal-offset.cpp +++ /dev/null @@ -1,184 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - - try { - EXPECT_INT_EQ(zarr::chunk_internal_offset(0, dims, ZarrDataType_uint16), - 0); - EXPECT_INT_EQ(zarr::chunk_internal_offset(1, dims, ZarrDataType_uint16), - 512); - EXPECT_INT_EQ(zarr::chunk_internal_offset(2, dims, ZarrDataType_uint16), - 0); - EXPECT_INT_EQ(zarr::chunk_internal_offset(3, dims, ZarrDataType_uint16), - 512); - EXPECT_INT_EQ(zarr::chunk_internal_offset(4, dims, ZarrDataType_uint16), - 0); - EXPECT_INT_EQ(zarr::chunk_internal_offset(5, dims, ZarrDataType_uint16), - 1024); - EXPECT_INT_EQ(zarr::chunk_internal_offset(6, dims, ZarrDataType_uint16), - 1536); - EXPECT_INT_EQ(zarr::chunk_internal_offset(7, dims, ZarrDataType_uint16), - 1024); - EXPECT_INT_EQ(zarr::chunk_internal_offset(8, dims, ZarrDataType_uint16), - 1536); - EXPECT_INT_EQ(zarr::chunk_internal_offset(9, dims, ZarrDataType_uint16), - 1024); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(10, dims, ZarrDataType_uint16), 0); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(11, dims, ZarrDataType_uint16), 512); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(12, dims, ZarrDataType_uint16), 0); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(13, dims, ZarrDataType_uint16), 512); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(14, dims, ZarrDataType_uint16), 0); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(15, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(16, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(17, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(18, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(19, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(20, dims, ZarrDataType_uint16), 3072); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(21, dims, ZarrDataType_uint16), 3584); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(22, dims, ZarrDataType_uint16), 3072); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(23, dims, ZarrDataType_uint16), 3584); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(24, dims, ZarrDataType_uint16), 3072); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(25, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(26, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(27, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(28, dims, ZarrDataType_uint16), 2560); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(29, dims, ZarrDataType_uint16), 2048); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(30, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(31, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(32, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(33, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(34, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(35, dims, ZarrDataType_uint16), 5120); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(36, dims, ZarrDataType_uint16), 5632); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(37, dims, ZarrDataType_uint16), 5120); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(38, dims, ZarrDataType_uint16), 5632); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(39, dims, ZarrDataType_uint16), 5120); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(40, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(41, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(42, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(43, dims, ZarrDataType_uint16), 4608); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(44, dims, ZarrDataType_uint16), 4096); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(45, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(46, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(47, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(48, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(49, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(50, dims, ZarrDataType_uint16), 7168); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(51, dims, ZarrDataType_uint16), 7680); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(52, dims, ZarrDataType_uint16), 7168); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(53, dims, ZarrDataType_uint16), 7680); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(54, dims, ZarrDataType_uint16), 7168); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(55, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(56, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(57, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(58, dims, ZarrDataType_uint16), 6656); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(59, dims, ZarrDataType_uint16), 6144); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(60, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(61, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(62, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(63, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(64, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(65, dims, ZarrDataType_uint16), 9216); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(66, dims, ZarrDataType_uint16), 9728); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(67, dims, ZarrDataType_uint16), 9216); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(68, dims, ZarrDataType_uint16), 9728); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(69, dims, ZarrDataType_uint16), 9216); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(70, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(71, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(72, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(73, dims, ZarrDataType_uint16), 8704); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(74, dims, ZarrDataType_uint16), 8192); - EXPECT_INT_EQ( - zarr::chunk_internal_offset(75, dims, ZarrDataType_uint16), 0); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-chunk-lattice-index.cpp b/tests/unit-tests/common-chunk-lattice-index.cpp deleted file mode 100644 index 3dc94d44..00000000 --- a/tests/unit-tests/common-chunk-lattice-index.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - try { - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - - EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 2, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 2, dims), 2); - EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 1, dims), 1); - EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 0, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 2, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 1, dims), 0); - EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 0, dims), 1); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-index-for-chunk.cpp b/tests/unit-tests/common-shard-index-for-chunk.cpp deleted file mode 100644 index 6c012b43..00000000 --- a/tests/unit-tests/common-shard-index-for-chunk.cpp +++ /dev/null @@ -1,195 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - try { - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - 0, - 5, // 5 timepoints / chunk - 2); // 2 chunks / shard - dims.emplace_back("c", - ZarrDimensionType_Channel, - 8, - 4, // 8 / 4 = 2 chunks - 2); // 4 / 2 = 2 shards - dims.emplace_back("z", - ZarrDimensionType_Space, - 6, - 2, // 6 / 2 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("y", - ZarrDimensionType_Space, - 48, - 16, // 48 / 16 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("x", - ZarrDimensionType_Space, - 64, - 16, // 64 / 16 = 4 chunks - 2); // 4 / 2 = 2 shards - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(12, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(13, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(14, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(15, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(16, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(17, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(18, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(19, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(20, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(21, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(22, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(23, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(24, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(25, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(26, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(27, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(28, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(29, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(30, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(31, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(32, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(33, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(34, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(35, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(36, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(37, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(38, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(39, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(40, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(41, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(42, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(43, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(44, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(45, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(46, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(47, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(48, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(49, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(50, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(51, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(52, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(53, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(54, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(55, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(56, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(57, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(58, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(59, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(60, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(61, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(62, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(63, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(64, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(65, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(66, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(67, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(68, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(69, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(70, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(71, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(72, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(73, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(74, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(75, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(76, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(77, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(78, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(79, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(80, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(81, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(82, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(83, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(84, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(85, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(86, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(87, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(88, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(89, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(90, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(91, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(92, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(93, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(94, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(95, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(96, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(97, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(98, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(99, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(100, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(101, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(102, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(103, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(104, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(105, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(106, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(107, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(108, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(109, dims), 0); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(110, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(111, dims), 1); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(112, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(113, dims), 2); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(114, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(115, dims), 3); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(116, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(117, dims), 4); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(118, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(119, dims), 5); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(120, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(121, dims), 6); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(122, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(123, dims), 7); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(124, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(125, dims), 8); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(126, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(127, dims), 9); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(128, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(129, dims), 10); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(130, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(131, dims), 11); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(132, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(133, dims), 12); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(134, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(135, dims), 13); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(136, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(137, dims), 14); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(138, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(139, dims), 15); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(140, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(141, dims), 16); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(142, dims), 17); - EXPECT_INT_EQ(zarr::shard_index_for_chunk(143, dims), 17); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-internal-index.cpp b/tests/unit-tests/common-shard-internal-index.cpp deleted file mode 100644 index 346c13bf..00000000 --- a/tests/unit-tests/common-shard-internal-index.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - 0, - 32, // 32 timepoints / chunk - 1); // 1 shard - dims.emplace_back("y", - ZarrDimensionType_Space, - 960, - 320, // 3 chunks - 2); // 2 ragged shards - dims.emplace_back("x", - ZarrDimensionType_Space, - 1080, - 270, // 4 chunks - 3); // 2 ragged shards - - try { - EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(0, dims), 0); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(1, dims), 1); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(2, dims), 2); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); - EXPECT_INT_EQ(zarr::shard_internal_index(3, dims), 0); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(4, dims), 3); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(5, dims), 4); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 0); - EXPECT_INT_EQ(zarr::shard_internal_index(6, dims), 5); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 1); - EXPECT_INT_EQ(zarr::shard_internal_index(7, dims), 3); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 2); - EXPECT_INT_EQ(zarr::shard_internal_index(8, dims), 0); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 2); - EXPECT_INT_EQ(zarr::shard_internal_index(9, dims), 1); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 2); - EXPECT_INT_EQ(zarr::shard_internal_index(10, dims), 2); - - EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 3); - EXPECT_INT_EQ(zarr::shard_internal_index(11, dims), 0); - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/common-tile-group-offset.cpp b/tests/unit-tests/common-tile-group-offset.cpp deleted file mode 100644 index be9efecc..00000000 --- a/tests/unit-tests/common-tile-group-offset.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -#define EXPECT_INT_EQ(a, b) \ - EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - - try { - EXPECT_INT_EQ(zarr::tile_group_offset(0, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(1, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(2, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(3, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(4, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(5, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(6, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(7, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(8, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(9, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(10, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(11, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(12, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(13, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(14, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(15, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(16, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(17, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(18, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(19, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(20, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(21, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(22, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(23, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(24, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(25, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(26, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(27, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(28, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(29, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(30, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(31, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(32, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(33, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(34, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(35, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(36, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(37, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(38, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(39, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(40, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(41, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(42, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(43, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(44, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(45, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(46, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(47, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(48, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(49, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(50, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(51, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(52, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(53, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(54, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(55, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(56, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(57, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(58, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(59, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(60, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(61, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(62, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(63, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(64, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(65, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(66, dims), 0); - EXPECT_INT_EQ(zarr::tile_group_offset(67, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(68, dims), 12); - EXPECT_INT_EQ(zarr::tile_group_offset(69, dims), 24); - EXPECT_INT_EQ(zarr::tile_group_offset(70, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(71, dims), 36); - EXPECT_INT_EQ(zarr::tile_group_offset(72, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(73, dims), 48); - EXPECT_INT_EQ(zarr::tile_group_offset(74, dims), 60); - EXPECT_INT_EQ(zarr::tile_group_offset(75, dims), 0); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); - } - - return retval; -} \ No newline at end of file From a61be1cb53bf5081ff07ba3446610760fa1f7a51 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Thu, 3 Oct 2024 14:59:16 -0400 Subject: [PATCH 38/39] Some cleanup. --- acquire-common | 2 +- src/streaming/zarr.stream.cpp | 8 +------- .../unit-tests/array-dimensions-chunk-internal-offset.cpp | 2 -- tests/unit-tests/array-dimensions-chunk-lattice-index.cpp | 2 -- .../unit-tests/array-dimensions-shard-index-for-chunk.cpp | 2 -- .../unit-tests/array-dimensions-shard-internal-index.cpp | 2 -- tests/unit-tests/array-dimensions-tile-group-offset.cpp | 2 -- .../unit-tests/array-writer-downsample-writer-config.cpp | 2 -- tests/unit-tests/array-writer-write-frame-to-chunks.cpp | 2 -- tests/unit-tests/zarrv2-writer-write-even.cpp | 2 -- .../unit-tests/zarrv2-writer-write-ragged-append-dim.cpp | 2 -- .../zarrv2-writer-write-ragged-internal-dim.cpp | 2 -- tests/unit-tests/zarrv3-writer-write-even.cpp | 2 -- .../unit-tests/zarrv3-writer-write-ragged-append-dim.cpp | 2 -- .../zarrv3-writer-write-ragged-internal-dim.cpp | 2 -- 15 files changed, 2 insertions(+), 34 deletions(-) diff --git a/acquire-common b/acquire-common index 8e3e9957..bbd03f38 160000 --- a/acquire-common +++ b/acquire-common @@ -1 +1 @@ -Subproject commit 8e3e9957c6f5d66753dc0a312c3a729e98f8e737 +Subproject commit bbd03f38d2ce48d25767defd3da49e4297d4be1c diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp index b9e43e78..1ff7d270 100644 --- a/src/streaming/zarr.stream.cpp +++ b/src/streaming/zarr.stream.cpp @@ -421,9 +421,7 @@ ZarrStream_s::create_writers_() void ZarrStream_s::create_scaled_frames_() { - if (multiscale_) { - // TODO (aliddell): implement this - } + // TODO (aliddell): implement this } bool @@ -465,9 +463,5 @@ void ZarrStream_s::write_multiscale_frames_(const uint8_t* data, size_t bytes_of_data) { - if (multiscale_) { - return; - } - // TODO (aliddell): implement this } diff --git a/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp b/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp index 53702fd4..02b58cd8 100644 --- a/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp +++ b/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp @@ -98,8 +98,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } return retval; diff --git a/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp b/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp index 7dd48576..2e9f06ce 100644 --- a/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp +++ b/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp @@ -73,8 +73,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } return retval; diff --git a/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp b/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp index 71519136..307e9dda 100644 --- a/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp +++ b/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp @@ -185,8 +185,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } return retval; diff --git a/tests/unit-tests/array-dimensions-shard-internal-index.cpp b/tests/unit-tests/array-dimensions-shard-internal-index.cpp index 73353be4..ed4591e4 100644 --- a/tests/unit-tests/array-dimensions-shard-internal-index.cpp +++ b/tests/unit-tests/array-dimensions-shard-internal-index.cpp @@ -65,8 +65,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } return retval; diff --git a/tests/unit-tests/array-dimensions-tile-group-offset.cpp b/tests/unit-tests/array-dimensions-tile-group-offset.cpp index f4714af8..c62368db 100644 --- a/tests/unit-tests/array-dimensions-tile-group-offset.cpp +++ b/tests/unit-tests/array-dimensions-tile-group-offset.cpp @@ -98,8 +98,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } return retval; diff --git a/tests/unit-tests/array-writer-downsample-writer-config.cpp b/tests/unit-tests/array-writer-downsample-writer-config.cpp index 32bb8f96..249c2728 100644 --- a/tests/unit-tests/array-writer-downsample-writer-config.cpp +++ b/tests/unit-tests/array-writer-downsample-writer-config.cpp @@ -126,8 +126,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } return retval; diff --git a/tests/unit-tests/array-writer-write-frame-to-chunks.cpp b/tests/unit-tests/array-writer-write-frame-to-chunks.cpp index 0f594bf2..9c426340 100644 --- a/tests/unit-tests/array-writer-write-frame-to-chunks.cpp +++ b/tests/unit-tests/array-writer-write-frame-to-chunks.cpp @@ -76,8 +76,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } // cleanup diff --git a/tests/unit-tests/zarrv2-writer-write-even.cpp b/tests/unit-tests/zarrv2-writer-write-even.cpp index d5b2b87d..0149bd84 100644 --- a/tests/unit-tests/zarrv2-writer-write-even.cpp +++ b/tests/unit-tests/zarrv2-writer-write-even.cpp @@ -173,8 +173,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } // cleanup diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp index 340bc93c..cfae9687 100644 --- a/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp +++ b/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp @@ -142,8 +142,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } // cleanup diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp index 1299d898..cd42b692 100644 --- a/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp +++ b/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp @@ -160,8 +160,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: ", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } // cleanup diff --git a/tests/unit-tests/zarrv3-writer-write-even.cpp b/tests/unit-tests/zarrv3-writer-write-even.cpp index 08b9449f..0b9b3269 100644 --- a/tests/unit-tests/zarrv3-writer-write-even.cpp +++ b/tests/unit-tests/zarrv3-writer-write-even.cpp @@ -212,8 +212,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } // cleanup diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp index a3b893c7..1b4bc606 100644 --- a/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp +++ b/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp @@ -167,8 +167,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } // cleanup diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp index d05b9504..24a0d7b5 100644 --- a/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp +++ b/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp @@ -190,8 +190,6 @@ main() retval = 0; } catch (const std::exception& exc) { LOG_ERROR("Exception: %s\n", exc.what()); - } catch (...) { - LOG_ERROR("Exception: (unknown)"); } // cleanup From 9b53bb83c3266f89c2100391d3fe9bdc955f47cb Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 4 Oct 2024 11:44:16 -0400 Subject: [PATCH 39/39] Respond to PR comments. --- src/streaming/zarrv3.array.writer.cpp | 20 +++++++++----------- src/streaming/zarrv3.array.writer.hh | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/streaming/zarrv3.array.writer.cpp b/src/streaming/zarrv3.array.writer.cpp index e844358f..d58db69e 100644 --- a/src/streaming/zarrv3.array.writer.cpp +++ b/src/streaming/zarrv3.array.writer.cpp @@ -5,7 +5,7 @@ #include -#include // std::fill_n +#include // std::fill #include #include @@ -16,7 +16,6 @@ namespace { std::string sample_type_to_dtype(ZarrDataType t) - { switch (t) { case ZarrDataType_uint8: @@ -67,8 +66,8 @@ zarr::ZarrV3ArrayWriter::ZarrV3ArrayWriter( for (auto& table : shard_tables_) { table.resize(2 * chunks_per_shard); - std::fill_n( - table.begin(), table.size(), std::numeric_limits::max()); + std::fill( + table.begin(), table.end(), std::numeric_limits::max()); } } @@ -91,12 +90,12 @@ zarr::ZarrV3ArrayWriter::flush_impl_() } // write out chunks to shards - bool write_table = is_finalizing_ || should_rollover_(); + auto write_table = is_finalizing_ || should_rollover_(); std::latch latch(n_shards); for (auto i = 0; i < n_shards; ++i) { const auto& chunks = chunk_in_shards.at(i); auto& chunk_table = shard_tables_.at(i); - size_t* file_offset = &shard_file_offsets_.at(i); + auto* file_offset = &shard_file_offsets_.at(i); EXPECT(thread_pool_->push_job([&sink = data_sinks_.at(i), &chunks, @@ -104,7 +103,7 @@ zarr::ZarrV3ArrayWriter::flush_impl_() file_offset, write_table, &latch, - this](std::string& err) mutable { + this](std::string& err) { bool success = false; try { @@ -148,12 +147,11 @@ zarr::ZarrV3ArrayWriter::flush_impl_() // reset shard tables and file offsets if (write_table) { for (auto& table : shard_tables_) { - std::fill_n(table.begin(), - table.size(), - std::numeric_limits::max()); + std::fill( + table.begin(), table.end(), std::numeric_limits::max()); } - std::fill_n(shard_file_offsets_.begin(), shard_file_offsets_.size(), 0); + std::fill(shard_file_offsets_.begin(), shard_file_offsets_.end(), 0); } return true; diff --git a/src/streaming/zarrv3.array.writer.hh b/src/streaming/zarrv3.array.writer.hh index 30cf108a..4681ea3e 100644 --- a/src/streaming/zarrv3.array.writer.hh +++ b/src/streaming/zarrv3.array.writer.hh @@ -3,7 +3,7 @@ #include "array.writer.hh" namespace zarr { -struct ZarrV3ArrayWriter final : public ArrayWriter +struct ZarrV3ArrayWriter : public ArrayWriter { public: ZarrV3ArrayWriter(ArrayWriterConfig&& config,