From b313658336c638f33aaa8b430e856574695d6ead Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 10 Sep 2024 10:42:36 -0700 Subject: [PATCH] Rearrange some files --- {src/include => include}/zarr.h | 0 src/CMakeLists.txt | 98 +---------- src/driver/CMakeLists.txt | 36 +++++ src/streaming/CMakeLists.txt | 58 +++++++ src/{internal => streaming}/array.writer.cpp | 0 src/{internal => streaming}/array.writer.hh | 0 .../blosc.compression.params.cpp | 0 .../blosc.compression.params.hh | 0 src/{internal => streaming}/file.sink.cpp | 0 src/{internal => streaming}/file.sink.hh | 0 src/{internal => streaming}/logger.cpp | 0 src/{internal => streaming}/logger.hh | 0 src/{internal => streaming}/s3.connection.cpp | 0 src/{internal => streaming}/s3.connection.hh | 0 src/{internal => streaming}/s3.sink.cpp | 0 src/{internal => streaming}/s3.sink.hh | 0 src/{internal => streaming}/sink.creator.cpp | 0 src/{internal => streaming}/sink.creator.hh | 0 src/{internal => streaming}/sink.hh | 0 .../stream.settings.cpp | 0 .../stream.settings.hh | 0 src/{internal => streaming}/thread.pool.cpp | 0 src/{internal => streaming}/thread.pool.hh | 0 src/{internal => streaming}/zarr.common.cpp | 0 src/{internal => streaming}/zarr.common.hh | 2 +- src/{internal => streaming}/zarr.stream.cpp | 0 src/{internal => streaming}/zarr.stream.hh | 0 .../zarrv2.array.writer.cpp | 0 .../zarrv2.array.writer.hh | 0 .../zarrv3.array.writer.cpp | 0 .../zarrv3.array.writer.hh | 0 tests/driver/CMakeLists.txt | 152 +++++++++--------- tests/unit-tests/CMakeLists.txt | 4 +- 33 files changed, 176 insertions(+), 174 deletions(-) rename {src/include => include}/zarr.h (100%) create mode 100644 src/driver/CMakeLists.txt create mode 100644 src/streaming/CMakeLists.txt rename src/{internal => streaming}/array.writer.cpp (100%) rename src/{internal => streaming}/array.writer.hh (100%) rename src/{internal => streaming}/blosc.compression.params.cpp (100%) rename src/{internal => streaming}/blosc.compression.params.hh (100%) rename src/{internal => streaming}/file.sink.cpp (100%) rename src/{internal => streaming}/file.sink.hh (100%) rename src/{internal => streaming}/logger.cpp (100%) rename src/{internal => streaming}/logger.hh (100%) rename src/{internal => streaming}/s3.connection.cpp (100%) rename src/{internal => streaming}/s3.connection.hh (100%) rename src/{internal => streaming}/s3.sink.cpp (100%) rename src/{internal => streaming}/s3.sink.hh (100%) rename src/{internal => streaming}/sink.creator.cpp (100%) rename src/{internal => streaming}/sink.creator.hh (100%) rename src/{internal => streaming}/sink.hh (100%) rename src/{internal => streaming}/stream.settings.cpp (100%) rename src/{internal => streaming}/stream.settings.hh (100%) rename src/{internal => streaming}/thread.pool.cpp (100%) rename src/{internal => streaming}/thread.pool.hh (100%) rename src/{internal => streaming}/zarr.common.cpp (100%) rename src/{internal => streaming}/zarr.common.hh (98%) rename src/{internal => streaming}/zarr.stream.cpp (100%) rename src/{internal => streaming}/zarr.stream.hh (100%) rename src/{internal => streaming}/zarrv2.array.writer.cpp (100%) rename src/{internal => streaming}/zarrv2.array.writer.hh (100%) rename src/{internal => streaming}/zarrv3.array.writer.cpp (100%) rename src/{internal => streaming}/zarrv3.array.writer.hh (100%) diff --git a/src/include/zarr.h b/include/zarr.h similarity index 100% rename from src/include/zarr.h rename to include/zarr.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1e0f485c..038903b4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,98 +1,4 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) -####### Acquire Zarr Streaming Library ####### - -set(tgt acquire-zarr) - -add_library(${tgt} STATIC - include/zarr.h - internal/logger.hh - internal/logger.cpp - internal/stream.settings.hh - internal/stream.settings.cpp - internal/zarr.stream.hh - internal/zarr.stream.cpp - internal/zarr.common.hh - internal/zarr.common.cpp - internal/thread.pool.hh - internal/thread.pool.cpp - internal/s3.connection.hh - internal/s3.connection.cpp - internal/blosc.compression.params.hh - internal/blosc.compression.params.cpp - internal/sink.hh - internal/file.sink.hh - internal/file.sink.cpp - internal/s3.sink.hh - internal/s3.sink.cpp - internal/sink.creator.hh - internal/sink.creator.cpp - internal/array.writer.hh - internal/array.writer.cpp - internal/zarrv2.array.writer.hh - internal/zarrv2.array.writer.cpp - internal/zarrv3.array.writer.hh - internal/zarrv3.array.writer.cpp -) - -target_include_directories(${tgt} - PUBLIC - $ - PRIVATE - $ -) - -target_link_libraries(${tgt} PRIVATE - blosc_static - miniocpp::miniocpp -) - -set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -) - -install(TARGETS ${tgt} - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib -) - -# Install public header files -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ - DESTINATION include - FILES_MATCHING PATTERN "*.h" -) - -####### Acquire Zarr Driver ####### - -if (NOT TARGET acquire-core-logger) - add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) -endif () - -set(tgt-driver acquire-driver-zarr) -add_library(${tgt-driver} MODULE - driver/zarr.storage.hh - driver/zarr.storage.cpp - driver/zarr.driver.c -) - -target_include_directories(${tgt-driver} PRIVATE - $ -) - -target_enable_simd(${tgt-driver}) -target_link_libraries(${tgt-driver} PRIVATE - acquire-core-logger - acquire-core-platform - acquire-device-kit - acquire-device-properties - ${tgt} - blosc_static - nlohmann_json::nlohmann_json - miniocpp::miniocpp -) - -set_target_properties(${tgt-driver} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -) - -install(TARGETS ${tgt-driver} LIBRARY DESTINATION lib) +add_subdirectory(streaming) +add_subdirectory(driver) diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt new file mode 100644 index 00000000..033f065b --- /dev/null +++ b/src/driver/CMakeLists.txt @@ -0,0 +1,36 @@ +option(BUILD_ACQUIRE_DRIVER_ZARR ON) + +if (BUILD_ACQUIRE_DRIVER_ZARR) + if (NOT TARGET acquire-core-logger) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) + endif () + + set(tgt acquire-driver-zarr) + add_library(${tgt} MODULE + zarr.storage.hh + zarr.storage.cpp + zarr.driver.c + ) + + target_include_directories(${tgt} PRIVATE + $ + ) + + target_enable_simd(${tgt}) + target_link_libraries(${tgt} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + acquire-zarr + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + + install(TARGETS ${tgt} LIBRARY DESTINATION lib) +endif () \ No newline at end of file diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt new file mode 100644 index 00000000..b399bec5 --- /dev/null +++ b/src/streaming/CMakeLists.txt @@ -0,0 +1,58 @@ +set(tgt acquire-zarr) + +add_library(${tgt} + logger.hh + logger.cpp + stream.settings.hh + stream.settings.cpp + zarr.stream.hh + zarr.stream.cpp + zarr.common.hh + zarr.common.cpp + thread.pool.hh + thread.pool.cpp + s3.connection.hh + s3.connection.cpp + blosc.compression.params.hh + blosc.compression.params.cpp + sink.hh + file.sink.hh + file.sink.cpp + s3.sink.hh + s3.sink.cpp + sink.creator.hh + sink.creator.cpp + array.writer.hh + array.writer.cpp + zarrv2.array.writer.hh + zarrv2.array.writer.cpp + zarrv3.array.writer.hh + zarrv3.array.writer.cpp +) + +target_include_directories(${tgt} + PUBLIC + $ + PRIVATE + $ +) + +target_link_libraries(${tgt} PRIVATE + blosc_static + miniocpp::miniocpp +) + +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt} + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) + +# Install public header files +install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/ + DESTINATION include + FILES_MATCHING PATTERN "*.h" +) \ No newline at end of file diff --git a/src/internal/array.writer.cpp b/src/streaming/array.writer.cpp similarity index 100% rename from src/internal/array.writer.cpp rename to src/streaming/array.writer.cpp diff --git a/src/internal/array.writer.hh b/src/streaming/array.writer.hh similarity index 100% rename from src/internal/array.writer.hh rename to src/streaming/array.writer.hh diff --git a/src/internal/blosc.compression.params.cpp b/src/streaming/blosc.compression.params.cpp similarity index 100% rename from src/internal/blosc.compression.params.cpp rename to src/streaming/blosc.compression.params.cpp diff --git a/src/internal/blosc.compression.params.hh b/src/streaming/blosc.compression.params.hh similarity index 100% rename from src/internal/blosc.compression.params.hh rename to src/streaming/blosc.compression.params.hh diff --git a/src/internal/file.sink.cpp b/src/streaming/file.sink.cpp similarity index 100% rename from src/internal/file.sink.cpp rename to src/streaming/file.sink.cpp diff --git a/src/internal/file.sink.hh b/src/streaming/file.sink.hh similarity index 100% rename from src/internal/file.sink.hh rename to src/streaming/file.sink.hh diff --git a/src/internal/logger.cpp b/src/streaming/logger.cpp similarity index 100% rename from src/internal/logger.cpp rename to src/streaming/logger.cpp diff --git a/src/internal/logger.hh b/src/streaming/logger.hh similarity index 100% rename from src/internal/logger.hh rename to src/streaming/logger.hh diff --git a/src/internal/s3.connection.cpp b/src/streaming/s3.connection.cpp similarity index 100% rename from src/internal/s3.connection.cpp rename to src/streaming/s3.connection.cpp diff --git a/src/internal/s3.connection.hh b/src/streaming/s3.connection.hh similarity index 100% rename from src/internal/s3.connection.hh rename to src/streaming/s3.connection.hh diff --git a/src/internal/s3.sink.cpp b/src/streaming/s3.sink.cpp similarity index 100% rename from src/internal/s3.sink.cpp rename to src/streaming/s3.sink.cpp diff --git a/src/internal/s3.sink.hh b/src/streaming/s3.sink.hh similarity index 100% rename from src/internal/s3.sink.hh rename to src/streaming/s3.sink.hh diff --git a/src/internal/sink.creator.cpp b/src/streaming/sink.creator.cpp similarity index 100% rename from src/internal/sink.creator.cpp rename to src/streaming/sink.creator.cpp diff --git a/src/internal/sink.creator.hh b/src/streaming/sink.creator.hh similarity index 100% rename from src/internal/sink.creator.hh rename to src/streaming/sink.creator.hh diff --git a/src/internal/sink.hh b/src/streaming/sink.hh similarity index 100% rename from src/internal/sink.hh rename to src/streaming/sink.hh diff --git a/src/internal/stream.settings.cpp b/src/streaming/stream.settings.cpp similarity index 100% rename from src/internal/stream.settings.cpp rename to src/streaming/stream.settings.cpp diff --git a/src/internal/stream.settings.hh b/src/streaming/stream.settings.hh similarity index 100% rename from src/internal/stream.settings.hh rename to src/streaming/stream.settings.hh diff --git a/src/internal/thread.pool.cpp b/src/streaming/thread.pool.cpp similarity index 100% rename from src/internal/thread.pool.cpp rename to src/streaming/thread.pool.cpp diff --git a/src/internal/thread.pool.hh b/src/streaming/thread.pool.hh similarity index 100% rename from src/internal/thread.pool.hh rename to src/streaming/thread.pool.hh diff --git a/src/internal/zarr.common.cpp b/src/streaming/zarr.common.cpp similarity index 100% rename from src/internal/zarr.common.cpp rename to src/streaming/zarr.common.cpp diff --git a/src/internal/zarr.common.hh b/src/streaming/zarr.common.hh similarity index 98% rename from src/internal/zarr.common.hh rename to src/streaming/zarr.common.hh index fa5e87cc..5c2657b4 100644 --- a/src/internal/zarr.common.hh +++ b/src/streaming/zarr.common.hh @@ -105,7 +105,7 @@ shard_index_for_chunk(size_t chunk_index, const std::vector& dimensions); /** - * @brief Get the internal index of a chunk within a shard. + * @brief Get the streaming index of a chunk within a shard. * @param chunk_index The index of the chunk. * @param dimensions The dimensions of the array. * @return The index of the chunk within the shard. diff --git a/src/internal/zarr.stream.cpp b/src/streaming/zarr.stream.cpp similarity index 100% rename from src/internal/zarr.stream.cpp rename to src/streaming/zarr.stream.cpp diff --git a/src/internal/zarr.stream.hh b/src/streaming/zarr.stream.hh similarity index 100% rename from src/internal/zarr.stream.hh rename to src/streaming/zarr.stream.hh diff --git a/src/internal/zarrv2.array.writer.cpp b/src/streaming/zarrv2.array.writer.cpp similarity index 100% rename from src/internal/zarrv2.array.writer.cpp rename to src/streaming/zarrv2.array.writer.cpp diff --git a/src/internal/zarrv2.array.writer.hh b/src/streaming/zarrv2.array.writer.hh similarity index 100% rename from src/internal/zarrv2.array.writer.hh rename to src/streaming/zarrv2.array.writer.hh diff --git a/src/internal/zarrv3.array.writer.cpp b/src/streaming/zarrv3.array.writer.cpp similarity index 100% rename from src/internal/zarrv3.array.writer.cpp rename to src/streaming/zarrv3.array.writer.cpp diff --git a/src/internal/zarrv3.array.writer.hh b/src/streaming/zarrv3.array.writer.hh similarity index 100% rename from src/internal/zarrv3.array.writer.hh rename to src/streaming/zarrv3.array.writer.hh diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt index ef7ec66e..5ed2234c 100644 --- a/tests/driver/CMakeLists.txt +++ b/tests/driver/CMakeLists.txt @@ -1,84 +1,86 @@ -set(NOTEST "TRUE") -add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) -set(NOTEST "FALSE") +if (BUILD_ACQUIRE_DRIVER_ZARR) + set(NOTEST "TRUE") + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) + set(NOTEST "FALSE") -# -# PARAMETERS -# -set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project + # + # PARAMETERS + # + set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project -# -# Tests -# -set(tests - list-devices - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 -) - -foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp + # + # Tests + # + set(tests + list-devices + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 ) - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") -endforeach () + foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") + endforeach () -# -# Copy driver to tests -# -list(POP_FRONT tests onename) + # + # Copy driver to tests + # + list(POP_FRONT tests onename) -foreach (driver - acquire-driver-common - acquire-driver-zarr -) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" + foreach (driver + acquire-driver-common + acquire-driver-zarr ) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" + ) - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) + endforeach () endforeach () -endforeach () +endif () \ No newline at end of file diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index 5a92e20e..12741982 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -32,8 +32,8 @@ foreach (name ${tests}) MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" ) target_include_directories(${tgt} PRIVATE - ${CMAKE_SOURCE_DIR}/src/include - ${CMAKE_SOURCE_DIR}/src/internal) + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/streaming) target_link_libraries(${tgt} PRIVATE acquire-zarr miniocpp::miniocpp