From 953c8ed72cfdab74db900171ea201051fad6ae1b Mon Sep 17 00:00:00 2001 From: james-rms Date: Thu, 1 Dec 2022 12:01:30 +1100 Subject: [PATCH] [Humble backport] rosbag2_storage_mcap: merge into rosbag2 repo (#1163) (#1189) * [backport] rosbag2_storage_mcap: merge into rosbag2 repo (#1163) * rosbag2_storage_mcap: merge into ros2/rosbag2 Signed-off-by: James Smith mcap_storage: 'none' is a valid storage preset profile (#86) Signed-off-by: James Smith bloom: add changelog changes 0.6.0 * ci: include rosbag2_storage_mcap Signed-off-by: James Smith * package.xml: include ROS Tooling WG maintainers Signed-off-by: James Smith * rosbag2_storage_mcap: update readme after move Signed-off-by: James Smith Signed-off-by: James Smith * zstd_vendor: do not remove zstd_errors.h Signed-off-by: James Smith Signed-off-by: James Smith --- .github/workflows/lint.yml | 21 + .github/workflows/test.yml | 3 + mcap_vendor/CHANGELOG.rst | 64 ++ mcap_vendor/CMakeLists.txt | 82 ++ mcap_vendor/package.xml | 19 + mcap_vendor/src/main.cpp | 16 + rosbag2_storage_mcap/.clang-format | 37 + rosbag2_storage_mcap/.vscode/settings.json | 13 + rosbag2_storage_mcap/CHANGELOG.rst | 127 +++ rosbag2_storage_mcap/CMakeLists.txt | 116 +++ rosbag2_storage_mcap/CPPLINT.cfg | 2 + rosbag2_storage_mcap/LICENSE | 201 +++++ rosbag2_storage_mcap/README.md | 146 ++++ .../message_definition_cache.hpp | 110 +++ .../visibility_control.hpp | 49 ++ rosbag2_storage_mcap/package.xml | 32 + rosbag2_storage_mcap/plugin_description.xml | 9 + rosbag2_storage_mcap/src/mcap_storage.cpp | 730 ++++++++++++++++++ .../src/message_definition_cache.cpp | 197 +++++ .../mcap_writer_options_zstd.yaml | 8 + .../test_mcap_storage.cpp | 172 +++++ .../test_message_definition_cache.cpp | 126 +++ rosbag2_storage_mcap_testdata/CHANGELOG.rst | 14 + rosbag2_storage_mcap_testdata/CMakeLists.txt | 20 + .../msg/BasicIdl.idl | 7 + .../msg/BasicMsg.msg | 1 + .../msg/ComplexIdl.idl | 9 + .../msg/ComplexMsg.msg | 1 + .../msg/ComplexMsgDependsOnIdl.msg | 1 + rosbag2_storage_mcap_testdata/package.xml | 23 + zstd_vendor/no_internal_headers.patch | 21 +- 31 files changed, 2366 insertions(+), 11 deletions(-) create mode 100644 mcap_vendor/CHANGELOG.rst create mode 100644 mcap_vendor/CMakeLists.txt create mode 100644 mcap_vendor/package.xml create mode 100644 mcap_vendor/src/main.cpp create mode 100644 rosbag2_storage_mcap/.clang-format create mode 100644 rosbag2_storage_mcap/.vscode/settings.json create mode 100644 rosbag2_storage_mcap/CHANGELOG.rst create mode 100644 rosbag2_storage_mcap/CMakeLists.txt create mode 100644 rosbag2_storage_mcap/CPPLINT.cfg create mode 100644 rosbag2_storage_mcap/LICENSE create mode 100644 rosbag2_storage_mcap/README.md create mode 100644 rosbag2_storage_mcap/include/rosbag2_storage_mcap/message_definition_cache.hpp create mode 100644 rosbag2_storage_mcap/include/rosbag2_storage_mcap/visibility_control.hpp create mode 100644 rosbag2_storage_mcap/package.xml create mode 100644 rosbag2_storage_mcap/plugin_description.xml create mode 100644 rosbag2_storage_mcap/src/mcap_storage.cpp create mode 100644 rosbag2_storage_mcap/src/message_definition_cache.cpp create mode 100644 rosbag2_storage_mcap/test/rosbag2_storage_mcap/mcap_writer_options_zstd.yaml create mode 100644 rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_mcap_storage.cpp create mode 100644 rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_message_definition_cache.cpp create mode 100644 rosbag2_storage_mcap_testdata/CHANGELOG.rst create mode 100644 rosbag2_storage_mcap_testdata/CMakeLists.txt create mode 100644 rosbag2_storage_mcap_testdata/msg/BasicIdl.idl create mode 100644 rosbag2_storage_mcap_testdata/msg/BasicMsg.msg create mode 100644 rosbag2_storage_mcap_testdata/msg/ComplexIdl.idl create mode 100644 rosbag2_storage_mcap_testdata/msg/ComplexMsg.msg create mode 100644 rosbag2_storage_mcap_testdata/msg/ComplexMsgDependsOnIdl.msg create mode 100644 rosbag2_storage_mcap_testdata/package.xml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fe6cf18a5b..8228bed4c4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -61,6 +61,27 @@ jobs: rosbag2_transport shared_queues_vendor + ament_lint_clang_format: # Linters applicable to C++ packages formatted with clang-format + name: ament_${{ matrix.linter }} + runs-on: ubuntu-latest + container: + image: rostooling/setup-ros-docker:ubuntu-focal-ros-rolling-ros-base-latest + strategy: + fail-fast: false + matrix: + linter: [cppcheck, cpplint, clang_format] + include: + - linter: clang_format + arguments: "--config rosbag2_storage_mcap/.clang-format" + steps: + - uses: actions/checkout@v2 + - uses: ros-tooling/action-ros-lint@v0.1 + with: + linter: ${{ matrix.linter }} + arguments: ${{ matrix.arguments }} + distribution: rolling + package-name: rosbag2_storage_mcap + ament_lint_python: # Linters applicable to Python packages name: ament_${{ matrix.linter }} runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1019ebe888..b025c876e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,7 @@ jobs: uses: ros-tooling/action-ros-ci@v0.2 with: package-name: | + mcap_vendor ros2bag rosbag2 rosbag2_compression @@ -29,6 +30,8 @@ jobs: rosbag2_py rosbag2_storage rosbag2_storage_default_plugins + rosbag2_storage_mcap + rosbag2_storage_mcap_testdata rosbag2_test_common rosbag2_tests rosbag2_transport diff --git a/mcap_vendor/CHANGELOG.rst b/mcap_vendor/CHANGELOG.rst new file mode 100644 index 0000000000..69abb3253b --- /dev/null +++ b/mcap_vendor/CHANGELOG.rst @@ -0,0 +1,64 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package mcap_vendor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.6.0 (2022-11-28) +------------------ +* Fix Windows build (`#73 `_) + Update mcap version to newest windows-compatible release. + Add visibility macros for tests. + Add clang-format preprocessor indentation for visibility_control to be readable. +* Contributors: Emerson Knapp + +0.5.0 (2022-11-02) +------------------ +* mcap_vendor: update to v0.6.0 (`#69 `_) +* Cleanup in `mcap_vendor` package (`#62 `_) +* Switch to using the vendored zstd library. (`#59 `_) +* Contributors: Chris Lalancette, Michael Orlov, James Smith + +0.4.0 (2022-10-06) +------------------ + +0.2.0 (2022-09-08) +------------------ +* Support timestamp-ordered playback (`#50 `_) +* Support regex topic filtering +* Contributors: James Smith + +0.1.7 (2022-08-15) +------------------ +* Add all lz4 sources to fix undefined symbols at runtime (`#46 `_) +* Contributors: Emerson Knapp + +0.1.6 (2022-07-22) +------------------ +* Upgrade mcap to fix LZ4 error and segfault (`#42 `_) + Incorporates fixes from https://github.com/foxglove/mcap/pull/478 and https://github.com/foxglove/mcap/pull/482 +* Add missing buildtool_depend on git (`#37 `_) + This vendor package uses git to fetch sources for other packages. It should declare a dependency on that build tool. + This should address the current cause of RPM build failures for RHEL: https://build.ros2.org/view/Rbin_rhel_el864/job/Rbin_rhel_el864__mcap_vendor__rhel_8_x86_64__binary/ +* Contributors: Jacob Bandes-Storch, Scott K Logan + +0.1.5 (2022-04-25) +------------------ +* Test Foxy & Galactic in CI, fix missing test_depends in mcap_vendor/package.xml (`#33 `_) +* Contributors: Jacob Bandes-Storch + +0.1.4 (2022-04-21) +------------------ +* fix: minor issues (`#31 `_) + * remove unnecessary block + * use target_link_libraries instead of ament_target_dependencies + * remove ros environment + * add prefix to compile definition +* Update email address for Foxglove maintainers (`#32 `_) +* Contributors: Daisuke Nishimatsu, Jacob Bandes-Storch + +0.1.3 (2022-04-20) +------------------ + +0.1.2 (2022-04-20) +------------------ +* Added mcap_vendor package. Updated CMakeLists.txt to fetch dependencies with FetchContent rather than Conan. +* Contributors: Jacob Bandes-Storch diff --git a/mcap_vendor/CMakeLists.txt b/mcap_vendor/CMakeLists.txt new file mode 100644 index 0000000000..25c008daf2 --- /dev/null +++ b/mcap_vendor/CMakeLists.txt @@ -0,0 +1,82 @@ +cmake_minimum_required(VERSION 3.5) +project(mcap_vendor LANGUAGES C CXX ASM) + +## Dependencies +find_package(ament_cmake REQUIRED) +find_package(zstd_vendor REQUIRED) +find_package(zstd REQUIRED) + +## Compile options +if(NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) +endif() +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wall -Wextra -Wpedantic -Werror) +endif() +if (MSVC) + add_compile_options(/W4 /WX + /wd4251 # suppress warning about having template instances (such as std::string) as public class members + ) + # suppress error disallowing `fopen` in favor of `fopen_s` + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) +endif() + +## Define vendor macro +macro(build_mcap_vendor) + include(FetchContent) + fetchcontent_declare(mcap + GIT_REPOSITORY https://github.com/foxglove/mcap.git + GIT_TAG dc6561d9ba867901709e36526dcf7f7359861e9c # releases/cpp/v0.7.0 + ) + fetchcontent_makeavailable(mcap) + + fetchcontent_declare(lz4 + GIT_REPOSITORY https://github.com/lz4/lz4.git + GIT_TAG d44371841a2f1728a3f36839fd4b7e872d0927d3 # v1.9.3 + ) + fetchcontent_makeavailable(lz4) + + file(GLOB _lz4_srcs + ${lz4_SOURCE_DIR}/lib/*.c) + + add_library(mcap SHARED + src/main.cpp + ${_lz4_srcs} + ) + + set(_mcap_include_dir ${mcap_SOURCE_DIR}/cpp/mcap/include) + + target_include_directories(mcap PRIVATE + ${lz4_SOURCE_DIR}/lib + ) + target_include_directories(mcap PUBLIC + "$" + "$" + ) + ament_target_dependencies(mcap zstd) + + install( + DIRECTORY ${_mcap_include_dir}/mcap + DESTINATION include/${PROJECT_NAME} + ) + + install( + TARGETS mcap + EXPORT mcap + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin + ) +endmacro() + +## Call vendor macro +build_mcap_vendor() + +ament_export_include_directories(include/${PROJECT_NAME}) +ament_export_targets(mcap HAS_LIBRARY_TARGET) +ament_export_dependencies(zstd_vendor zstd) + +## Package +ament_package() diff --git a/mcap_vendor/package.xml b/mcap_vendor/package.xml new file mode 100644 index 0000000000..28b3bba1d6 --- /dev/null +++ b/mcap_vendor/package.xml @@ -0,0 +1,19 @@ + + + + mcap_vendor + 0.6.0 + mcap vendor package + Foxglove + ROS Tooling Working Group + Apache License 2.0 + + ament_cmake + git + + zstd_vendor + + + ament_cmake + + diff --git a/mcap_vendor/src/main.cpp b/mcap_vendor/src/main.cpp new file mode 100644 index 0000000000..1e4a31148e --- /dev/null +++ b/mcap_vendor/src/main.cpp @@ -0,0 +1,16 @@ +// Copyright 2022, Foxglove Technologies. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define MCAP_IMPLEMENTATION +#include diff --git a/rosbag2_storage_mcap/.clang-format b/rosbag2_storage_mcap/.clang-format new file mode 100644 index 0000000000..e91709d083 --- /dev/null +++ b/rosbag2_storage_mcap/.clang-format @@ -0,0 +1,37 @@ +--- +Language: Cpp +Standard: c++17 +BasedOnStyle: Google + +AllowShortFunctionsOnASingleLine: Empty +AllowShortLambdasOnASingleLine: Empty +AccessModifierOffset: -2 +TabWidth: 2 +ContinuationIndentWidth: 2 +UseTab: Never +BreakConstructorInitializers: BeforeComma +BraceWrapping: + AfterClass: true + AfterFunction: true + AfterNamespace: true + AfterStruct: true + AfterEnum: true +BreakBeforeBraces: Custom +ColumnLimit: 100 +ConstructorInitializerAllOnOneLineOrOnePerLine: false +DerivePointerAlignment: false +FixNamespaceComments: true +PointerAlignment: Middle +ReflowComments: false +SortIncludes: true +IndentPPDirectives: BeforeHash + +IncludeCategories: + - Regex: '^"' + Priority: 1 + - Regex: "^`_) +* mcap_storage: handle update_metadata call (`#83 `_) +* Update clang-format rules to fit ROS 2 style guide (`#80 `_) +* Revert "read_order: throw exception from set_read_order for unsupported orders" + This reverts commit aef9b9a65293f9e5d80a858ef84e485a8655a0c0. +* read_order: throw exception from set_read_order for unsupported orders +* Fix compile flags to work on rosbag_storage:0.17.x (`#78 `_) + This fixes the compile flags for rolling, which has two versions -- one that does not support read order (0.17.x) and one that does support read order (0.18.x). +* Fix Windows build (`#73 `_) + Update mcap version to newest windows-compatible release. + Add visibility macros for tests. + Add clang-format preprocessor indentation for visibility_control to be readable. +* Contributors: Andrew Symington, Emerson Knapp, James Smith, james-rms + +0.5.0 (2022-11-02) +------------------ +* set defaults for SQLite plugin parity (`#68 `_) +* rosbag2_storage_mcap: add storage preset profiles (`#57 `_) +* rename test_fixture_interfaces package to testdata (`#64 `_) +* Switch to using the vendored zstd library. (`#59 `_) +* Add set_read_order reader API (`#54 `_) +* Contributors: Chris Lalancette, Emerson Knapp, James Smith + +0.4.0 (2022-10-06) +------------------ +* Some minor improvements in rosbag2_storage_mcap after review (`#58 `_) + 1. Fixed some findings from Clang-Tidy + 1. Some renames according to the ROS2 coding style + 1. Add default initializations for member variables + 1. Moved code responsible for adding schema and channel from write(msg) + to create_topic(topic) method to reduce performance burden on first + message write and in lieu to preparation for moving schema collection + process to upper SequentialWriter layer. +* Revert "rosbag2_storage_mcap: add storage preset profiles" + This reverts commit 38830add3935b978968fe2703d3180b413ccc8c2. +* rosbag2_storage_mcap: add storage preset profiles +* Contributors: James Smith, Michael Orlov + +0.3.0 (2022-09-09) +------------------ +* Store IDL message definitions in Schema records when no MSG definition is available + (`#43 `_) +* Contributors: James Smith + +0.2.0 (2022-09-08) +------------------ +* Support timestamp-ordered playback (`#50 `_) +* Support regex topic filtering +* Contributors: James Smith + +0.1.7 (2022-08-15) +------------------ +* Add all lz4 sources to fix undefined symbols at runtime (`#46 `_) +* Contributors: Emerson Knapp + +0.1.6 (2022-07-22) +------------------ +* Upgrade mcap to fix LZ4 error and segfault (`#42 `_) + Incorporates fixes from https://github.com/foxglove/mcap/pull/478 and https://github.com/foxglove/mcap/pull/482 +* Contributors: Jacob Bandes-Storch + +0.1.5 (2022-04-25) +------------------ +* Fix build for Foxy (`#34 `_) +* Contributors: Jacob Bandes-Storch + +0.1.4 (2022-04-21) +------------------ +* fix: minor issues (`#31 `_) + * remove unnecessary block + * use target_link_libraries instead of ament_target_dependencies + * remove ros environment + * add prefix to compile definition +* Update email address for Foxglove maintainers (`#32 `_) +* Contributors: Daisuke Nishimatsu, Jacob Bandes-Storch + +0.1.3 (2022-04-20) +------------------ + +0.1.2 (2022-04-20) +------------------ +* Added mcap_vendor package. Updated CMakeLists.txt to fetch dependencies with FetchContent rather than Conan. +* Contributors: Jacob Bandes-Storch + +0.1.1 (2022-04-01) +------------------ +* CMake build script will now execute pip install conan automatically. +* Contributors: Daisuke Nishimatsu + +0.1.0 (2022-03-24) +------------------ +* [1.0.0] Use Summary section for get_metadata() and seek(), implement remaining methods (`#17 `_) +* feat: add play impl (`#16 `_) +* chore: refine package.xml (`#15 `_) +* Don't throw when READ_WRITE mode is used; add .mcap file extension to recorded files (`#14 `_) + I may be missing something, but from a cursory glance at [this code](https://github.com/ros2/rosbag2/blob/342d8ed3c1c4ae0411a4a92b60e79a728b8974b8/rosbag2_storage/src/rosbag2_storage/impl/storage_factory_impl.hpp#L108-L135), it appears that the `APPEND` mode is never used. This means we need to support `READ_WRITE`. + This also adds a `.mcap` extension to recorded file names. +* Add dynamic message definition lookup (`#13 `_) + Currently, an exception will be thrown if lookup fails. +* Switch C++ formatter to clang-format (`#12 `_) + Remove uncrustify linter in favor of clang-format, which is easier to configure for use in VS Code format-on-save. +* Merge pull request `#7 `_ from ros-tooling/jhurliman/reader-writer + Reader and writer implementation +* uninitialized struct +* lint +* lint +* lint +* Reader and writer implementation +* Merge pull request `#6 `_ from wep21/add-metadata-impl + feat: add metadata impl +* feat: add metadata impl +* Merge pull request `#5 `_ from wep21/mcap-storage-impl + feat: mcap storage impl +* chore: update cmake minimum version +* chore: install mcap header +* chore: include mcap header +* fix: move fetch content into rosbag2 storage mcap +* Merge pull request `#3 `_ from ros-tooling/emersonknapp/mcap_plugin_skeleton + Add mcap storage plugin skeleton and CI +* Add rosbag2_storage_mcap skeleton +* Contributors: Daisuke Nishimatsu, Emerson Knapp, Jacob Bandes-Storch, John Hurliman, wep21 diff --git a/rosbag2_storage_mcap/CMakeLists.txt b/rosbag2_storage_mcap/CMakeLists.txt new file mode 100644 index 0000000000..b93c53dfee --- /dev/null +++ b/rosbag2_storage_mcap/CMakeLists.txt @@ -0,0 +1,116 @@ +cmake_minimum_required(VERSION 3.14) +project(rosbag2_storage_mcap) + +# Set Release build if no build type was specified +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING + "Build type for the build. Possible values are: Debug, Release, RelWithDebInfo, MinSizeRel" + FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS + "Debug" "Release" "RelWithDebInfo" "MinSizeRel") +endif() + +# Enable additional warnings and warnings as errors +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wall -Wextra -Wpedantic) +endif() + +# Get the ROS_DISTRO environment variable +set(ROS_DISTRO $ENV{ROS_DISTRO}) + +find_package(ament_cmake REQUIRED) +find_package(ament_index_cpp REQUIRED) +find_package(mcap_vendor REQUIRED) +find_package(pluginlib REQUIRED) +find_package(rcutils REQUIRED) +find_package(rosbag2_storage REQUIRED) + +add_library(${PROJECT_NAME} SHARED + src/mcap_storage.cpp + src/message_definition_cache.cpp +) +target_include_directories(${PROJECT_NAME} PUBLIC + $ + $ +) +target_compile_features(${PROJECT_NAME} PUBLIC c_std_99 cxx_std_17) +target_compile_definitions(${PROJECT_NAME} PRIVATE "ROSBAG2_STORAGE_MCAP_BUILDING_DLL") +ament_target_dependencies(${PROJECT_NAME} + mcap_vendor + pluginlib + rcutils + rosbag2_storage) + +set(MCAP_COMPILE_DEFS) +# COMPATIBILITY(foxy) - 0.3.x is the Foxy release +if(${rosbag2_storage_VERSION} VERSION_GREATER_EQUAL 0.4.0) + list(APPEND MCAP_COMPILE_DEFS ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS) + list(APPEND MCAP_COMPILE_DEFS ROSBAG2_STORAGE_MCAP_WRITER_CREATES_DIRECTORY) +endif() +# COMPATIBILITY(galactic) - 0.9.x is the Galactic release +if(${rosbag2_storage_VERSION} VERSION_GREATER_EQUAL 0.10.0) + list(APPEND MCAP_COMPILE_DEFS ROSBAG2_STORAGE_MCAP_OVERRIDE_SEEK_METHOD) +endif() +# COMPATIBILITY(foxy, galactic) - 0.15.x is the Humble release +if(${rosbag2_storage_VERSION} VERSION_GREATER_EQUAL 0.15.0) + list(APPEND MCAP_COMPILE_DEFS ROSBAG2_STORAGE_MCAP_HAS_YAML_HPP) +endif() +# COMPATIBILITY(foxy, galactic, humble, rolling:0.17.x) +if(${rosbag2_storage_VERSION} VERSION_GREATER_EQUAL 0.17.0) + list(APPEND MCAP_COMPILE_DEFS ROSBAG2_STORAGE_MCAP_HAS_STORAGE_FILTER_TOPIC_REGEX) +endif() +# COMPATIBILITY(foxy, galactic, humble, rolling:0.17.x, rolling:0.18.x) +if(${rosbag2_storage_VERSION} VERSION_GREATER_EQUAL 0.18.0) + list(APPEND MCAP_COMPILE_DEFS ROSBAG2_STORAGE_MCAP_HAS_SET_READ_ORDER) + list(APPEND MCAP_COMPILE_DEFS ROSBAG2_STORAGE_MCAP_HAS_UPDATE_METADATA) +endif() + +target_compile_definitions(${PROJECT_NAME} PRIVATE ${MCAP_COMPILE_DEFS}) + +# Causes the visibility macros to use dllexport rather than dllimport, +# which is appropriate when building the dll but not consuming it. +target_compile_definitions(${PROJECT_NAME} PRIVATE "ROSBAG2_STORAGE_MCAP_BUILDING_LIBRARY") + +pluginlib_export_plugin_description_file(rosbag2_storage plugin_description.xml) + +install( + TARGETS ${PROJECT_NAME} + EXPORT export_${PROJECT_NAME} + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin +) + +if(BUILD_TESTING) + find_package(ament_cmake_gmock REQUIRED) + find_package(ament_lint_auto REQUIRED) + find_package(rcpputils REQUIRED) + find_package(rosbag2_cpp REQUIRED) + find_package(rosbag2_test_common REQUIRED) + find_package(std_msgs REQUIRED) + + add_definitions(-D_TEST_RESOURCES_DIR_PATH="${CMAKE_CURRENT_SOURCE_DIR}/test/${PROJECT_NAME}") + + set(ament_cmake_clang_format_CONFIG_FILE .clang-format) + list(APPEND AMENT_LINT_AUTO_EXCLUDE ament_cmake_uncrustify) + if(WIN32) + # clang-format is not easily exposed to Windows, this linter enforced regardless via Linux/OSX builds + list(APPEND AMENT_LINT_AUTO_EXCLUDE ament_cmake_clang_format) + endif() + ament_lint_auto_find_test_dependencies() + + ament_add_gmock(test_mcap_storage test/rosbag2_storage_mcap/test_mcap_storage.cpp) + target_link_libraries(test_mcap_storage ${PROJECT_NAME}) + ament_target_dependencies(test_mcap_storage rosbag2_storage rosbag2_cpp rosbag2_test_common std_msgs) + target_compile_definitions(test_mcap_storage PRIVATE ${MCAP_COMPILE_DEFS}) + + ament_add_gmock(test_message_definition_cache test/rosbag2_storage_mcap/test_message_definition_cache.cpp) + target_link_libraries(test_message_definition_cache ${PROJECT_NAME}) +endif() + + +ament_export_libraries(${PROJECT_NAME}) +ament_export_targets(export_${PROJECT_NAME}) +ament_export_dependencies(rosbag2_storage rcutils) + +ament_package() diff --git a/rosbag2_storage_mcap/CPPLINT.cfg b/rosbag2_storage_mcap/CPPLINT.cfg new file mode 100644 index 0000000000..7b4f580de2 --- /dev/null +++ b/rosbag2_storage_mcap/CPPLINT.cfg @@ -0,0 +1,2 @@ +# We manage include order with clang-format +filter=-build/include_order diff --git a/rosbag2_storage_mcap/LICENSE b/rosbag2_storage_mcap/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/rosbag2_storage_mcap/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/rosbag2_storage_mcap/README.md b/rosbag2_storage_mcap/README.md new file mode 100644 index 0000000000..17fa209d29 --- /dev/null +++ b/rosbag2_storage_mcap/README.md @@ -0,0 +1,146 @@ +# rosbag2_storage_mcap + +This package provides a [storage plugin](https://github.com/ros2/rosbag2#storage-format-plugin-architecture) for rosbag2 which extends it with support for the [MCAP](https://mcap.dev) file format. + + +[![ROS Foxy version](https://img.shields.io/ros/v/foxy/rosbag2_storage_mcap)](https://index.ros.org/p/rosbag2_storage_mcap/github-ros2-rosbag2/#foxy) +[![ROS Galactic version](https://img.shields.io/ros/v/galactic/rosbag2_storage_mcap)](https://index.ros.org/p/rosbag2_storage_mcap/github-ros2-rosbag2/#galactic) +[![ROS Humble version](https://img.shields.io/ros/v/humble/rosbag2_storage_mcap)](https://index.ros.org/p/rosbag2_storage_mcap/github-ros2-rosbag2/#humble) +[![ROS Rolling version](https://img.shields.io/ros/v/rolling/rosbag2_storage_mcap)](https://index.ros.org/p/rosbag2_storage_mcap/github-ros2-rosbag2/#rolling) + +## ⚠️ Move from `ros-tooling` organization + +This package was recently merged into [https://github.com/ros2/rosbag2](ros2/rosbag2) from [https://github.com/ros-tooling/rosbag2_storage_mcap](ros-tooling/rosbag2_storage_mcap). To view historical pull requests and git history of this package, check [here](https://github.com/ros-tooling/rosbag2_storage_mcap/commits/main). + + +## Installation + +rosbag2_storage_mcap is available as part of the [current ROS 2 distributions](https://docs.ros.org/en/rolling/Releases.html). On Ubuntu, after following the [ROS 2 installation instructions](https://docs.ros.org/en/humble/Installation.html), you can use: + +```bash +# Replace "rolling" with your ROS distro (`echo $ROS_DISTRO`) +$ sudo apt install ros-rolling-rosbag2-storage-mcap +``` + +## Usage + +Use MCAP files with regular [`ros2 bag` commands](https://docs.ros.org/en/humble/Tutorials/Beginner-CLI-Tools/Recording-And-Playing-Back-Data/Recording-And-Playing-Back-Data.html) by adding the `--storage mcap` option (abbreviated as `-s mcap`): + +```bash +$ ros2 bag record -s mcap /topic1 /topic2 ... + +$ ros2 bag play -s mcap path/to/your_recording.mcap + +$ ros2 bag info -s mcap path/to/your_recording.mcap +``` + + +### Writer Configuration + +To configure details of the MCAP writer for `ros2 bag record`, use the `--storage-config-file` options to provide a YAML file describing `mcap::McapWriterOptions`. Field descriptions below copied from [McapWriterOptions declaration](https://github.com/foxglove/mcap/blob/main/cpp/mcap/include/mcap/writer.hpp#L18) + +| Field | Type / Values | Description | +| ----- | ------------- | ----------- | +| noChunkCRC | bool | Disable CRC calculation for Chunks. Ignored if `noChunking=true`. | +| noAttachmentCRC | bool | Disable CRC calculation for Attachments. | +| enableDataCRC | bool | Enables CRC calculation for the entire Data section. Useful when `noChunking=True`. | +| noSummaryCRC | bool | Disable CRC calculation for the Summary section. | +| noChunking | bool | Do not write Chunks to the file, instead writing Schema, Channel, and Message records directly into the Data section. | +| noMessageIndex | bool | Do not write Message Index records to the file. If `noSummary=true` and `noChunkIndex=false`, Chunk Index records will still be written to the Summary section, providing a coarse message index. | +| noSummary | bool | Do not write Summary or Summary Offset sections to the file, placing the Footer record immediately after DataEnd. This can provide some speed boost to file writing and produce smaller files, at the expense of requiring a conversion process later if fast summarization or indexed access is desired. | +| chunkSize | unsigned int | Target uncompressed Chunk payload size in bytes. Once a Chunk's uncompressed data meets or exceeds this size, the Chunk will be compressed (if compression is enabled) and written to disk. Note that smaller Chunks may be written, such as the last Chunk in the Data section. This option is ignored if `noChunking=true`. | +| compression | "None", "Lz4", "Zstd" | Compression algorithm to use when writing Chunks. This option is ignored if `noChunking=true`. | +| compressionLevel | "Fastest", "Fast", "Default", "Slow", "Slowest" | Compression level to use when writing Chunks. Slower generally produces smaller files, at the expense of more CPU time. These levels map to different internal settings for each compression algorithm. | +| forceCompression | bool | By default, Chunks that do not benefit from compression will be written uncompressed. This option can be used to force compression on all Chunks. This option is ignored if `noChunking=true`. | +| noRepeatedSchemas | bool | Advanced option. | +| noRepeatedChannels | bool | Advanced option. | +| noMetadataIndex | bool | Advanced option. | +| noChunkIndex | bool | Advanced option. | +| noStatistics | bool | Advanced option. | +| noSummaryOffsets | bool | Advanced option. | + + +Example: + +``` +# mcap_writer_options.yml +noChunkCRC: false +noChunking: false +noMessageIndex: false +noSummary: false +chunkSize: 786432 +compression: "Zstd" +compressionLevel: "Fast" +forceCompression: false +``` + +``` +$ ros2 bag record -s mcap -o my_bag --all --storage-config-file mcap_writer_options.yml +``` + +### Storage Preset Profiles + +You can also use one of the preset profiles described below, for example: + +``` +$ ros2 bag record -s mcap -o my_bag --all --storage-preset-profile fastwrite +``` + +#### `fastwrite` + +Configures the MCAP writer for the highest possible write throughput and lowest resource utilization. This preset does not calculate CRCs for integrity checking, and does not write a message index. This preset profile is useful for resource-constrained robots. + +Equivalent to this storage configuration: +```yaml +noChunking: true +noSummaryCRC: true +``` + +Using MCAPs written with `fastwrite` as a long-term storage format is not recommended. Some features will not work when reading MCAP files without a message index, such as reading messages from a subset of topics or seeking. When recording MCAPs on your robot with `fastwrite`, it is a good idea to post-process these files afterwards, to restore the message index and also save storage space: + +```bash +# Using the MCAP CLI https://github.com/foxglove/mcap/tree/main/go/cli/mcap +$ mcap compress fast.mcap -o compressed.mcap +# Using `ros2 bag convert` +$ cat << EOF > convert.yaml +output_bags: + - uri: compressed + storage_id: mcap + storage_preset_profile: zstd_small +EOF +$ ros2 bag convert -i fast.mcap -o convert.yaml +``` + +Equivalent to this storage configuration: +```yaml +noChunking: true +noSummaryCRC: true +``` + +#### `zstd_fast` + +Configures the MCAP writer to use chunk compression with [zstd](http://facebook.github.io/zstd/). Chunk compression yields file sizes comparable to bags compressed with file-level compression, but allows tools to efficiently read messages without decompressing the entire bag. This preset uses the lowest compression ratio and disables CRC calculation, to achieve high throughput while conserving disk space. + +Equivalent to this storage configuration: + +```yaml +compression: "Zstd" +compressionLevel: "Fastest" +noChunkCRC: true +``` + +#### `zstd_small` + +Configures the MCAP writer to write 4MB chunks, compressed with zstd using its highest compression ratio. This produces very small bags, but can be resource-intensive to write. This preset also calculates chunk CRCs, which allow a reader to determine if a chunk is corrupted. This preset is useful when using `ros2 bag convert` as a post-processing step. + +Equivalent to this storage configuration: + +```yaml +compression: "Zstd" +compressionLevel: "Slowest" +chunkSize: 4194304 # 4 * 1024 * 1024 +``` + +### ROS 2 Distro maintenance + +Whenever a ROS 2 distribution reaches EOL, search for comments marked COMPATIBILITY - which may no longer be needed when no new releases will be made for that distro. diff --git a/rosbag2_storage_mcap/include/rosbag2_storage_mcap/message_definition_cache.hpp b/rosbag2_storage_mcap/include/rosbag2_storage_mcap/message_definition_cache.hpp new file mode 100644 index 0000000000..024c6641a6 --- /dev/null +++ b/rosbag2_storage_mcap/include/rosbag2_storage_mcap/message_definition_cache.hpp @@ -0,0 +1,110 @@ +// Copyright 2022, Foxglove Technologies. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ROSBAG2_STORAGE_MCAP__MESSAGE_DEFINITION_CACHE_HPP_ +#define ROSBAG2_STORAGE_MCAP__MESSAGE_DEFINITION_CACHE_HPP_ + +#include "visibility_control.hpp" + +#include +#include +#include +#include +#include + +namespace rosbag2_storage_mcap::internal +{ +enum struct Format +{ + IDL, + MSG, +}; + +struct MessageSpec +{ + MessageSpec(Format format, std::string text, const std::string & package_context); + const std::set dependencies; + const std::string text; + Format format; +}; + +struct DefinitionIdentifier +{ + Format format; + std::string package_resource_name; + + bool operator==(const DefinitionIdentifier & di) const + { + return (format == di.format) && (package_resource_name == di.package_resource_name); + } +}; + +class DefinitionNotFoundError : public std::exception +{ +private: + std::string name_; + +public: + explicit DefinitionNotFoundError(std::string name) + : name_(std::move(name)) + { + } + + const char * what() const noexcept override + { + return name_.c_str(); + } +}; + +class MessageDefinitionCache final +{ +public: + /** + * Concatenate the message definition with its dependencies into a self-contained schema. + * The format is different for MSG and IDL definitions, and is described fully here: + * [MSG](https://mcap.dev/specification/appendix.html#ros2msg-data-format) + * [IDL](https://mcap.dev/specification/appendix.html#ros2idl-data-format) + * Throws DefinitionNotFoundError if one or more definition files are missing for the given + * package resource name. + */ + ROSBAG2_STORAGE_MCAP_PUBLIC + std::pair get_full_text(const std::string & package_resource_name); + +private: + struct DefinitionIdentifierHash + { + std::size_t operator()(const DefinitionIdentifier & di) const + { + std::size_t h1 = std::hash()(di.format); + std::size_t h2 = std::hash()(di.package_resource_name); + return h1 ^ h2; + } + }; + /** + * Load and parse the message file referenced by the given datatype, or return it from + * msg_specs_by_datatype + */ + const MessageSpec & load_message_spec(const DefinitionIdentifier & definition_identifier); + + std::unordered_map + msg_specs_by_definition_identifier_; +}; + +ROSBAG2_STORAGE_MCAP_PUBLIC +std::set parse_dependencies(Format format, const std::string & text, + const std::string & package_context); + +} // namespace rosbag2_storage_mcap::internal + +#endif // ROSBAG2_STORAGE_MCAP__MESSAGE_DEFINITION_CACHE_HPP_ diff --git a/rosbag2_storage_mcap/include/rosbag2_storage_mcap/visibility_control.hpp b/rosbag2_storage_mcap/include/rosbag2_storage_mcap/visibility_control.hpp new file mode 100644 index 0000000000..57bf150937 --- /dev/null +++ b/rosbag2_storage_mcap/include/rosbag2_storage_mcap/visibility_control.hpp @@ -0,0 +1,49 @@ +// Copyright 2022 Foxglove Technologies Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* This header must be included by this library's headers which declare symbols + * that are defined separately. The contents of this header change the visibility + * of certain symbols, especially for Windows DLL usage. + */ + +#ifndef ROSBAG2_STORAGE_MCAP__VISIBILITY_CONTROL_HPP_ +#define ROSBAG2_STORAGE_MCAP__VISIBILITY_CONTROL_HPP_ + +// This logic was borrowed (then namespaced) from the examples on the gcc wiki: +// https://gcc.gnu.org/wiki/Visibility + +#if defined _WIN32 || defined __CYGWIN__ + #ifdef __GNUC__ + #define ROSBAG2_STORAGE_MCAP_EXPORT __attribute__((dllexport)) + #define ROSBAG2_STORAGE_MCAP_IMPORT __attribute__((dllimport)) + #else + #define ROSBAG2_STORAGE_MCAP_EXPORT __declspec(dllexport) + #define ROSBAG2_STORAGE_MCAP_IMPORT __declspec(dllimport) + #endif + #ifdef ROSBAG2_STORAGE_MCAP_BUILDING_DLL + #define ROSBAG2_STORAGE_MCAP_PUBLIC ROSBAG2_STORAGE_MCAP_EXPORT + #else + #define ROSBAG2_STORAGE_MCAP_PUBLIC ROSBAG2_STORAGE_MCAP_IMPORT + #endif +#else + #define ROSBAG2_STORAGE_MCAP_EXPORT __attribute__((visibility("default"))) + #define ROSBAG2_STORAGE_MCAP_IMPORT + #if __GNUC__ >= 4 + #define ROSBAG2_STORAGE_MCAP_PUBLIC __attribute__((visibility("default"))) + #else + #define ROSBAG2_STORAGE_MCAP_PUBLIC + #endif +#endif + +#endif // ROSBAG2_STORAGE_MCAP__VISIBILITY_CONTROL_HPP_ diff --git a/rosbag2_storage_mcap/package.xml b/rosbag2_storage_mcap/package.xml new file mode 100644 index 0000000000..84df948d5e --- /dev/null +++ b/rosbag2_storage_mcap/package.xml @@ -0,0 +1,32 @@ + + + + rosbag2_storage_mcap + 0.6.0 + rosbag2 storage plugin using the MCAP file format + Foxglove + ROS Tooling Working Group + Apache-2.0 + + ament_cmake + + ament_index_cpp + mcap_vendor + pluginlib + rcutils + rosbag2_storage + + ament_cmake_clang_format + ament_cmake_gmock + ament_lint_auto + ament_lint_common + rcpputils + rosbag2_cpp + rosbag2_test_common + std_msgs + rosbag2_storage_mcap_testdata + + + ament_cmake + + diff --git a/rosbag2_storage_mcap/plugin_description.xml b/rosbag2_storage_mcap/plugin_description.xml new file mode 100644 index 0000000000..dd7d45ec0d --- /dev/null +++ b/rosbag2_storage_mcap/plugin_description.xml @@ -0,0 +1,9 @@ + + + rosbag2 storage plugin using the MCAP file format + + diff --git a/rosbag2_storage_mcap/src/mcap_storage.cpp b/rosbag2_storage_mcap/src/mcap_storage.cpp new file mode 100644 index 0000000000..9836a88c23 --- /dev/null +++ b/rosbag2_storage_mcap/src/mcap_storage.cpp @@ -0,0 +1,730 @@ +// Copyright 2022, Amazon.com Inc or its Affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rcutils/logging_macros.h" +#include "rosbag2_storage/metadata_io.hpp" +#include "rosbag2_storage/ros_helper.hpp" +#include "rosbag2_storage/storage_interfaces/read_write_interface.hpp" +#include "rosbag2_storage_mcap/message_definition_cache.hpp" + +#ifdef ROSBAG2_STORAGE_MCAP_HAS_YAML_HPP + #include "rosbag2_storage/yaml.hpp" +#else + // COMPATIBILITY(foxy, galactic) - this block is available in rosbag2_storage/yaml.hpp in H + #ifdef _WIN32 + // This is necessary because of a bug in yaml-cpp's cmake + #define YAML_CPP_DLL + // This is necessary because yaml-cpp does not always use dllimport/dllexport consistently + #pragma warning(push) + #pragma warning(disable : 4251) + #pragma warning(disable : 4275) + #endif + #include "yaml-cpp/yaml.h" + #ifdef _WIN32 + #pragma warning(pop) + #endif +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_FILTER_TOPIC_REGEX + #include +#endif + +#define DECLARE_YAML_VALUE_MAP(KEY_TYPE, VALUE_TYPE, ...) \ + template <> \ + struct convert \ + { \ + static Node encode(const KEY_TYPE & e) \ + { \ + static const std::pair mapping[] = __VA_ARGS__; \ + for (const auto & m : mapping) { \ + if (m.first == e) { \ + return Node(m.second); \ + } \ + } \ + return Node(""); \ + } \ + \ + static bool decode(const Node & node, KEY_TYPE & e) \ + { \ + static const std::pair mapping[] = __VA_ARGS__; \ + const auto val = node.as(); \ + for (const auto & m : mapping) { \ + if (m.second == val) { \ + e = m.first; \ + return true; \ + } \ + } \ + return false; \ + } \ + } + +namespace +{ +// Simple wrapper with default constructor for use by YAML +struct McapWriterOptions : mcap::McapWriterOptions +{ + McapWriterOptions() + : mcap::McapWriterOptions("ros2") + { + } +}; +} // namespace + +namespace YAML +{ +#ifndef ROSBAG2_STORAGE_MCAP_HAS_YAML_HPP +template +void optional_assign(const Node & node, std::string field, T & assign_to) +{ + if (node[field]) { + assign_to = node[field].as(); + } +} +#endif + +DECLARE_YAML_VALUE_MAP(mcap::Compression, std::string, + {{mcap::Compression::None, "None"}, + {mcap::Compression::Lz4, "Lz4"}, + {mcap::Compression::Zstd, "Zstd"}}); + +DECLARE_YAML_VALUE_MAP(mcap::CompressionLevel, std::string, + {{mcap::CompressionLevel::Fastest, "Fastest"}, + {mcap::CompressionLevel::Fast, "Fast"}, + {mcap::CompressionLevel::Default, "Default"}, + {mcap::CompressionLevel::Slow, "Slow"}, + {mcap::CompressionLevel::Slowest, "Slowest"}}); + +template <> +struct convert +{ + // NOTE: when updating this struct, also update documentation in README.md + static bool decode(const Node & node, McapWriterOptions & o) + { + optional_assign(node, "noChunkCRC", o.noChunkCRC); + optional_assign(node, "noAttachmentCRC", o.noAttachmentCRC); + optional_assign(node, "enableDataCRC", o.enableDataCRC); + optional_assign(node, "noSummaryCRC", o.noSummaryCRC); + optional_assign(node, "noChunking", o.noChunking); + optional_assign(node, "noMessageIndex", o.noMessageIndex); + optional_assign(node, "noSummary", o.noSummary); + optional_assign(node, "chunkSize", o.chunkSize); + optional_assign(node, "compression", o.compression); + optional_assign(node, "compressionLevel", o.compressionLevel); + optional_assign(node, "forceCompression", o.forceCompression); + // Intentionally omitting "profile" and "library" + optional_assign(node, "noRepeatedSchemas", o.noRepeatedSchemas); + optional_assign(node, "noRepeatedChannels", o.noRepeatedChannels); + optional_assign(node, "noAttachmentIndex", o.noAttachmentIndex); + optional_assign(node, "noMetadataIndex", o.noMetadataIndex); + optional_assign(node, "noChunkIndex", o.noChunkIndex); + optional_assign(node, "noStatistics", o.noStatistics); + optional_assign(node, "noSummaryOffsets", o.noSummaryOffsets); + return true; + } +}; +} // namespace YAML + +namespace rosbag2_storage_plugins +{ +using mcap::ByteOffset; +using time_point = std::chrono::time_point; +static const char FILE_EXTENSION[] = ".mcap"; +static const char LOG_NAME[] = "rosbag2_storage_mcap"; + +static void OnProblem(const mcap::Status & status) +{ + RCUTILS_LOG_ERROR_NAMED(LOG_NAME, "%s", status.message.c_str()); +} + +/** + * A storage implementation for the MCAP file format. + */ +class MCAPStorage : public rosbag2_storage::storage_interfaces::ReadWriteInterface +{ +public: + MCAPStorage(); + ~MCAPStorage() override; + + /** BaseIOInterface **/ +#ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS + void open(const rosbag2_storage::StorageOptions & storage_options, + rosbag2_storage::storage_interfaces::IOFlag io_flag = + rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) override; + void open(const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag io_flag = + rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE); +#else + void open(const std::string & uri, + rosbag2_storage::storage_interfaces::IOFlag io_flag = + rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) override; +#endif + + /** BaseInfoInterface **/ + rosbag2_storage::BagMetadata get_metadata() override; + std::string get_relative_file_path() const override; + uint64_t get_bagfile_size() const override; + std::string get_storage_identifier() const override; + + /** BaseReadInterface **/ +#ifdef ROSBAG2_STORAGE_MCAP_HAS_SET_READ_ORDER + void set_read_order(const rosbag2_storage::ReadOrder &) override; +#endif + bool has_next() override; + std::shared_ptr read_next() override; + std::vector get_all_topics_and_types() override; + + /** ReadOnlyInterface **/ + void set_filter(const rosbag2_storage::StorageFilter & storage_filter) override; + void reset_filter() override; +#ifdef ROSBAG2_STORAGE_MCAP_OVERRIDE_SEEK_METHOD + void seek(const rcutils_time_point_value_t & time_stamp) override; +#else + void seek(const rcutils_time_point_value_t & timestamp); +#endif + + /** ReadWriteInterface **/ + uint64_t get_minimum_split_file_size() const override; + + /** BaseWriteInterface **/ + void write(std::shared_ptr msg) override; + void write( + const std::vector> & msg) override; + void create_topic(const rosbag2_storage::TopicMetadata & topic) override; + void remove_topic(const rosbag2_storage::TopicMetadata & topic) override; +#ifdef ROSBAG2_STORAGE_MCAP_HAS_UPDATE_METADATA + void update_metadata(const rosbag2_storage::BagMetadata &) override; +#endif + +private: + void open_impl(const std::string & uri, const std::string & preset_profile, + rosbag2_storage::storage_interfaces::IOFlag io_flag, + const std::string & storage_config_uri); + + void reset_iterator(rcutils_time_point_value_t start_time = 0); + bool read_and_enqueue_message(); + void ensure_summary_read(); + + std::optional opened_as_; + std::string relative_path_; + + std::shared_ptr next_; + + rosbag2_storage::BagMetadata metadata_{}; + std::unordered_map topics_; + std::unordered_map schema_ids_; // datatype -> schema_id + std::unordered_map channel_ids_; // topic -> channel_id + rosbag2_storage::StorageFilter storage_filter_{}; + mcap::ReadMessageOptions::ReadOrder read_order_ = + mcap::ReadMessageOptions::ReadOrder::LogTimeOrder; + + std::unique_ptr input_; + std::unique_ptr data_source_; + std::unique_ptr mcap_reader_; + std::unique_ptr linear_view_; + std::unique_ptr linear_iterator_; + + std::unique_ptr mcap_writer_; + rosbag2_storage_mcap::internal::MessageDefinitionCache msgdef_cache_{}; + + bool has_read_summary_ = false; +}; + +MCAPStorage::MCAPStorage() +{ + metadata_.storage_identifier = get_storage_identifier(); + metadata_.message_count = 0; +} + +MCAPStorage::~MCAPStorage() +{ + if (mcap_reader_) { + mcap_reader_->close(); + } + if (input_) { + input_->close(); + } + if (mcap_writer_) { + mcap_writer_->close(); + } +} + +/** BaseIOInterface **/ +#ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS +void MCAPStorage::open(const rosbag2_storage::StorageOptions & storage_options, + rosbag2_storage::storage_interfaces::IOFlag io_flag) +{ + open_impl(storage_options.uri, storage_options.storage_preset_profile, io_flag, + storage_options.storage_config_uri); +} +#endif + +void MCAPStorage::open(const std::string & uri, rosbag2_storage::storage_interfaces::IOFlag io_flag) +{ + open_impl(uri, "", io_flag, ""); +} + +static void SetOptionsForPreset(const std::string & preset_profile, McapWriterOptions & options) +{ + if (preset_profile == "fastwrite") { + options.noChunking = true; + options.noSummaryCRC = true; + } else if (preset_profile == "zstd_fast") { + options.compression = mcap::Compression::Zstd; + options.compressionLevel = mcap::CompressionLevel::Fastest; + options.noChunkCRC = true; + } else if (preset_profile == "zstd_small") { + options.compression = mcap::Compression::Zstd; + options.compressionLevel = mcap::CompressionLevel::Slowest; + options.chunkSize = 4 * 1024 * 1024; + } else if (preset_profile != "none") { + throw std::runtime_error( + "unknown MCAP storage preset profile " + "(valid options are 'none', 'fastwrite', 'zstd_fast', 'zstd_small'): " + + preset_profile); + } +} + +void MCAPStorage::open_impl(const std::string & uri, const std::string & preset_profile, + rosbag2_storage::storage_interfaces::IOFlag io_flag, + const std::string & storage_config_uri) +{ + switch (io_flag) { + case rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY: { + relative_path_ = uri; + input_ = std::make_unique(relative_path_, std::ios::binary); + data_source_ = std::make_unique(*input_); + mcap_reader_ = std::make_unique(); + auto status = mcap_reader_->open(*data_source_); + if (!status.ok()) { + throw std::runtime_error(status.message); + } + reset_iterator(); + break; + } + case rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE: + case rosbag2_storage::storage_interfaces::IOFlag::APPEND: { + // APPEND does not seem to be used; treat it the same as READ_WRITE + io_flag = rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE; + relative_path_ = uri + FILE_EXTENSION; + + mcap_writer_ = std::make_unique(); + McapWriterOptions options; + // Set defaults for the rosbag2 storage plugin specifically. + options.noChunkCRC = true; + options.compression = mcap::Compression::None; + // Set options from preset profile first + if (!preset_profile.empty()) { + SetOptionsForPreset(preset_profile, options); + } + // If both preset profile and storage config are specified, + // options from the storage config are overlaid on the options from the preset profile. + if (!storage_config_uri.empty()) { + YAML::Node yaml_node = YAML::LoadFile(storage_config_uri); + YAML::convert::decode(yaml_node, options); + } + + auto status = mcap_writer_->open(relative_path_, options); + if (!status.ok()) { + throw std::runtime_error(status.message); + } + break; + } + } + opened_as_ = io_flag; + metadata_.relative_file_paths = {get_relative_file_path()}; +} + +/** BaseInfoInterface **/ +rosbag2_storage::BagMetadata MCAPStorage::get_metadata() +{ + ensure_summary_read(); + + metadata_.version = 2; + metadata_.storage_identifier = get_storage_identifier(); + metadata_.bag_size = get_bagfile_size(); + metadata_.relative_file_paths = {get_relative_file_path()}; + + // Fill out summary metadata from the Statistics record + const mcap::Statistics & stats = mcap_reader_->statistics().value(); + metadata_.message_count = stats.messageCount; + metadata_.duration = std::chrono::nanoseconds(stats.messageEndTime - stats.messageStartTime); + metadata_.starting_time = time_point(std::chrono::nanoseconds(stats.messageStartTime)); + + // Build a list of topic information along with per-topic message counts + metadata_.topics_with_message_count.clear(); + for (const auto & [channel_id, channel_ptr] : mcap_reader_->channels()) { + const mcap::Channel & channel = *channel_ptr; + + // Look up the Schema for this topic + const auto schema_ptr = mcap_reader_->schema(channel.schemaId); + if (!schema_ptr) { + throw std::runtime_error("Could not find schema for topic " + channel.topic); + } + + // Create a TopicInformation for this topic + rosbag2_storage::TopicInformation topic_info{}; + topic_info.topic_metadata.name = channel.topic; + topic_info.topic_metadata.serialization_format = channel.messageEncoding; + topic_info.topic_metadata.type = schema_ptr->name; + + // Look up the offered_qos_profiles metadata entry + const auto metadata_it = channel.metadata.find("offered_qos_profiles"); + if (metadata_it != channel.metadata.end()) { + topic_info.topic_metadata.offered_qos_profiles = metadata_it->second; + } + + // Look up the message count for this Channel + const auto message_count_it = stats.channelMessageCounts.find(channel_id); + if (message_count_it != stats.channelMessageCounts.end()) { + topic_info.message_count = message_count_it->second; + } else { + topic_info.message_count = 0; + } + + metadata_.topics_with_message_count.push_back(topic_info); + } + + return metadata_; +} + +std::string MCAPStorage::get_relative_file_path() const +{ + return relative_path_; +} + +uint64_t MCAPStorage::get_bagfile_size() const +{ + if (opened_as_ == rosbag2_storage::storage_interfaces::IOFlag::READ_ONLY) { + return data_source_ ? data_source_->size() : 0; + } else { + if (!mcap_writer_) { + return 0; + } + const auto * data_sink = mcap_writer_->dataSink(); + return data_sink ? data_sink->size() : 0; + } +} + +std::string MCAPStorage::get_storage_identifier() const +{ + return "mcap"; +} + +/** BaseReadInterface **/ +bool MCAPStorage::read_and_enqueue_message() +{ + // The recording has not been opened. + if (!linear_iterator_) { + return false; + } + // Already have popped and queued the next message. + if (next_ != nullptr) { + return true; + } + + auto & it = *linear_iterator_; + + // At the end of the recording + if (it == linear_view_->end()) { + return false; + } + + const auto & messageView = *it; + auto msg = std::make_shared(); + msg->time_stamp = rcutils_time_point_value_t(messageView.message.logTime); + msg->topic_name = messageView.channel->topic; + msg->serialized_data = rosbag2_storage::make_serialized_message(messageView.message.data, + messageView.message.dataSize); + + // enqueue this message to be used + next_ = msg; + + ++it; + return true; +} + +void MCAPStorage::reset_iterator(rcutils_time_point_value_t start_time) +{ + ensure_summary_read(); + mcap::ReadMessageOptions options; + options.startTime = mcap::Timestamp(start_time); + options.readOrder = read_order_; + if (!storage_filter_.topics.empty()) { + options.topicFilter = [this](std::string_view topic) { + for (const auto & match_topic : storage_filter_.topics) { + if (match_topic == topic) { + return true; + } + } + return false; + }; + } +#ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_FILTER_TOPIC_REGEX + if (!storage_filter_.topics_regex.empty()) { + options.topicFilter = [this](std::string_view topic) { + std::smatch m; + std::string topic_string(topic); + std::regex re(storage_filter_.topics_regex); + return std::regex_match(topic_string, m, re); + }; + } +#endif + linear_view_ = + std::make_unique(mcap_reader_->readMessages(OnProblem, options)); + linear_iterator_ = std::make_unique(linear_view_->begin()); +} + +void MCAPStorage::ensure_summary_read() +{ + if (!has_read_summary_) { + const auto status = mcap_reader_->readSummary(mcap::ReadSummaryMethod::AllowFallbackScan); + + if (!status.ok()) { + throw std::runtime_error(status.message); + } + // check if message indexes are present, if not, read in file order. + bool message_indexes_found = false; + for (const auto & ci : mcap_reader_->chunkIndexes()) { + if (ci.messageIndexLength > 0) { + message_indexes_found = true; + break; + } + } + if (!message_indexes_found) { + RCUTILS_LOG_WARN_NAMED(LOG_NAME, + "no message indices found, falling back to reading in file order"); + read_order_ = mcap::ReadMessageOptions::ReadOrder::FileOrder; + } + has_read_summary_ = true; + } +} + +#ifdef ROSBAG2_STORAGE_MCAP_HAS_SET_READ_ORDER +void MCAPStorage::set_read_order(const rosbag2_storage::ReadOrder & read_order) +{ + auto next_read_order = read_order_; + switch (read_order.sort_by) { + case rosbag2_storage::ReadOrder::ReceivedTimestamp: + if (read_order.reverse) { + next_read_order = mcap::ReadMessageOptions::ReadOrder::ReverseLogTimeOrder; + } else { + next_read_order = mcap::ReadMessageOptions::ReadOrder::LogTimeOrder; + } + break; + case rosbag2_storage::ReadOrder::File: + if (!read_order.reverse) { + next_read_order = mcap::ReadMessageOptions::ReadOrder::FileOrder; + } else { + throw std::runtime_error("Reverse file order reading not implemented."); + } + break; + case rosbag2_storage::ReadOrder::PublishedTimestamp: + throw std::runtime_error("PublishedTimestamp read order not yet implemented in ROS 2"); + break; + } + if (next_read_order != read_order_) { + read_order_ = next_read_order; + reset_iterator(); + } +} +#endif + +bool MCAPStorage::has_next() +{ + if (!linear_iterator_) { + return false; + } + // Have already verified next message and enqueued it for use. + if (next_) { + return true; + } + + return read_and_enqueue_message(); +} + +std::shared_ptr MCAPStorage::read_next() +{ + if (!has_next()) { + throw std::runtime_error{"No next message is available."}; + } + // Importantly, clear next_ via move so that a next message can be read. + return std::move(next_); +} + +std::vector MCAPStorage::get_all_topics_and_types() +{ + auto metadata = get_metadata(); + std::vector out; + for (const auto & topic : metadata.topics_with_message_count) { + out.push_back(topic.topic_metadata); + } + return out; +} + +/** ReadOnlyInterface **/ +void MCAPStorage::set_filter(const rosbag2_storage::StorageFilter & storage_filter) +{ + storage_filter_ = storage_filter; + reset_iterator(); +} + +void MCAPStorage::reset_filter() +{ + set_filter(rosbag2_storage::StorageFilter()); +} + +void MCAPStorage::seek(const rcutils_time_point_value_t & time_stamp) +{ + reset_iterator(time_stamp); +} + +/** ReadWriteInterface **/ +uint64_t MCAPStorage::get_minimum_split_file_size() const +{ + return 1024; +} + +/** BaseWriteInterface **/ +void MCAPStorage::write(std::shared_ptr msg) +{ + const auto topic_it = topics_.find(msg->topic_name); + if (topic_it == topics_.end()) { + throw std::runtime_error{"Unknown message topic \"" + msg->topic_name + "\""}; + } + + // Get Channel reference + const auto channel_it = channel_ids_.find(msg->topic_name); + if (channel_it == channel_ids_.end()) { + // This should never happen since we adding channel on topic creation + throw std::runtime_error{"Channel reference not found for topic: \"" + msg->topic_name + "\""}; + } + + mcap::Message mcap_msg; + mcap_msg.channelId = channel_it->second; + mcap_msg.sequence = 0; + if (msg->time_stamp < 0) { + RCUTILS_LOG_WARN_NAMED(LOG_NAME, "Invalid message timestamp %ld", msg->time_stamp); + } + mcap_msg.logTime = mcap::Timestamp(msg->time_stamp); + mcap_msg.publishTime = mcap_msg.logTime; + mcap_msg.dataSize = msg->serialized_data->buffer_length; + mcap_msg.data = reinterpret_cast(msg->serialized_data->buffer); + const auto status = mcap_writer_->write(mcap_msg); + if (!status.ok()) { + throw std::runtime_error{std::string{"Failed to write "} + + std::to_string(msg->serialized_data->buffer_length) + + " byte message to MCAP file: " + status.message}; + } + + /// Update metadata + // Increment individual topic message count + topic_it->second.message_count++; + // Increment global message count + metadata_.message_count++; + // Determine recording duration + const auto message_time = time_point(std::chrono::nanoseconds(msg->time_stamp)); + metadata_.duration = std::max(metadata_.duration, message_time - metadata_.starting_time); +} + +void MCAPStorage::write( + const std::vector> & msgs) +{ + for (const auto & msg : msgs) { + write(msg); + } +} + +void MCAPStorage::create_topic(const rosbag2_storage::TopicMetadata & topic) +{ + auto topic_info = rosbag2_storage::TopicInformation{topic, 0}; + const auto topic_it = topics_.find(topic.name); + if (topic_it == topics_.end()) { + topics_.emplace(topic.name, topic_info); + } else { + RCUTILS_LOG_WARN_NAMED(LOG_NAME, "Topic with name: %s already exist!", topic.name.c_str()); + return; + } + + // Create Schema for topic if it doesn't exist yet + const auto & datatype = topic_info.topic_metadata.type; + const auto schema_it = schema_ids_.find(datatype); + mcap::SchemaId schema_id; + if (schema_it == schema_ids_.end()) { + mcap::Schema schema; + schema.name = datatype; + try { + auto [format, full_text] = msgdef_cache_.get_full_text(datatype); + if (format == rosbag2_storage_mcap::internal::Format::MSG) { + schema.encoding = "ros2msg"; + } else { + schema.encoding = "ros2idl"; + } + schema.data.assign(reinterpret_cast(full_text.data()), + reinterpret_cast(full_text.data() + full_text.size())); + } catch (rosbag2_storage_mcap::internal::DefinitionNotFoundError & err) { + RCUTILS_LOG_ERROR_NAMED(LOG_NAME, "definition file(s) missing for %s: missing %s", + datatype.c_str(), err.what()); + schema.encoding = ""; + } + mcap_writer_->addSchema(schema); + schema_ids_.emplace(datatype, schema.id); + schema_id = schema.id; + } else { + schema_id = schema_it->second; + } + + // Create Channel for topic if it doesn't exist yet + const auto channel_it = channel_ids_.find(topic.name); + if (channel_it == channel_ids_.end()) { + mcap::Channel channel; + channel.topic = topic.name; + channel.messageEncoding = topic_info.topic_metadata.serialization_format; + channel.schemaId = schema_id; + channel.metadata.emplace("offered_qos_profiles", + topic_info.topic_metadata.offered_qos_profiles); + mcap_writer_->addChannel(channel); + channel_ids_.emplace(topic.name, channel.id); + } +} + +void MCAPStorage::remove_topic(const rosbag2_storage::TopicMetadata & topic) +{ + topics_.erase(topic.name); +} + +#ifdef ROSBAG2_STORAGE_MCAP_HAS_UPDATE_METADATA +void MCAPStorage::update_metadata(const rosbag2_storage::BagMetadata & bag_metadata) +{ + if (bag_metadata.compression_mode == "message") { + throw std::runtime_error( + "MCAP storage plugin does not support message compression, " + "consider using chunk compression by setting `compression: 'Zstd'` in storage config"); + } +} +#endif + +} // namespace rosbag2_storage_plugins + +#include "pluginlib/class_list_macros.hpp" // NOLINT +PLUGINLIB_EXPORT_CLASS(rosbag2_storage_plugins::MCAPStorage, + rosbag2_storage::storage_interfaces::ReadWriteInterface) diff --git a/rosbag2_storage_mcap/src/message_definition_cache.cpp b/rosbag2_storage_mcap/src/message_definition_cache.cpp new file mode 100644 index 0000000000..5cfbebac81 --- /dev/null +++ b/rosbag2_storage_mcap/src/message_definition_cache.cpp @@ -0,0 +1,197 @@ +// Copyright 2022, Foxglove Technologies. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rosbag2_storage_mcap/message_definition_cache.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rosbag2_storage_mcap::internal +{ +// Match datatype names (foo_msgs/Bar or foo_msgs/msg/Bar) +static const std::regex PACKAGE_TYPENAME_REGEX{R"(^([a-zA-Z0-9_]+)/(?:msg/)?([a-zA-Z0-9_]+)$)"}; + +// Match field types from .msg definitions ("foo_msgs/Bar" in "foo_msgs/Bar[] bar") +static const std::regex MSG_FIELD_TYPE_REGEX{R"((?:^|\n)\s*([a-zA-Z0-9_/]+)(?:\[[^\]]*\])?\s+)"}; + +// match field types from `.idl` definitions ("foo_msgs/msg/bar" in #include ) +static const std::regex IDL_FIELD_TYPE_REGEX{ + R"((?:^|\n)#include\s+(?:"|<)([a-zA-Z0-9_/]+)\.idl(?:"|>))"}; + +static const std::unordered_set PRIMITIVE_TYPES{ + "bool", "byte", "char", "float32", "float64", "int8", "uint8", + "int16", "uint16", "int32", "uint32", "int64", "uint64", "string"}; + +static std::set parse_msg_dependencies(const std::string & text, + const std::string & package_context) +{ + std::set dependencies; + + for (std::sregex_iterator iter(text.begin(), text.end(), MSG_FIELD_TYPE_REGEX); + iter != std::sregex_iterator(); ++iter) { + std::string type = (*iter)[1]; + if (PRIMITIVE_TYPES.find(type) != PRIMITIVE_TYPES.end()) { + continue; + } + if (type.find('/') == std::string::npos) { + dependencies.insert(package_context + '/' + std::move(type)); + } else { + dependencies.insert(std::move(type)); + } + } + return dependencies; +} + +static std::set parse_idl_dependencies(const std::string & text) +{ + std::set dependencies; + + for (std::sregex_iterator iter(text.begin(), text.end(), IDL_FIELD_TYPE_REGEX); + iter != std::sregex_iterator(); ++iter) { + dependencies.insert((*iter)[1]); + } + return dependencies; +} + +std::set parse_dependencies(Format format, const std::string & text, + const std::string & package_context) +{ + switch (format) { + case Format::MSG: + return parse_msg_dependencies(text, package_context); + case Format::IDL: + return parse_idl_dependencies(text); + default: + throw std::runtime_error("switch is not exhaustive"); + } +} + +static const char * extension_for_format(Format format) +{ + switch (format) { + case Format::MSG: + return ".msg"; + case Format::IDL: + return ".idl"; + default: + throw std::runtime_error("switch is not exhaustive"); + } +} + +static std::string delimiter(const DefinitionIdentifier & definition_identifier) +{ + std::string result = + "================================================================================\n"; + switch (definition_identifier.format) { + case Format::MSG: + result += "MSG: "; + break; + case Format::IDL: + result += "IDL: "; + break; + default: + throw std::runtime_error("switch is not exhaustive"); + } + result += definition_identifier.package_resource_name; + result += "\n"; + return result; +} + +MessageSpec::MessageSpec(Format format, std::string text, const std::string & package_context) + : dependencies(parse_dependencies(format, text, package_context)) + , text(std::move(text)) + , format(format) +{ +} + +const MessageSpec & MessageDefinitionCache::load_message_spec( + const DefinitionIdentifier & definition_identifier) +{ + if (auto it = msg_specs_by_definition_identifier_.find(definition_identifier); + it != msg_specs_by_definition_identifier_.end()) { + return it->second; + } + std::smatch match; + if (!std::regex_match(definition_identifier.package_resource_name, match, + PACKAGE_TYPENAME_REGEX)) { + throw std::invalid_argument("Invalid package resource name: " + + definition_identifier.package_resource_name); + } + std::string package = match[1]; + std::string share_dir = ament_index_cpp::get_package_share_directory(package); + std::ifstream file{share_dir + "/msg/" + match[2].str() + + extension_for_format(definition_identifier.format)}; + if (!file.good()) { + throw DefinitionNotFoundError(definition_identifier.package_resource_name); + } + + std::string contents{std::istreambuf_iterator(file), {}}; + const MessageSpec & spec = + msg_specs_by_definition_identifier_ + .emplace(definition_identifier, + MessageSpec(definition_identifier.format, std::move(contents), package)) + .first->second; + + // "References and pointers to data stored in the container are only invalidated by erasing that + // element, even when the corresponding iterator is invalidated." + return spec; +} + +std::pair MessageDefinitionCache::get_full_text( + const std::string & root_package_resource_name) +{ + std::unordered_set seen_deps; + + std::function append_recursive = + [&](const DefinitionIdentifier & definition_identifier) { + const MessageSpec & spec = load_message_spec(definition_identifier); + std::string result = spec.text; + for (const auto & dep_name : spec.dependencies) { + DefinitionIdentifier dep{definition_identifier.format, dep_name}; + bool inserted = seen_deps.insert(dep).second; + if (inserted) { + result += "\n"; + result += delimiter(dep); + result += append_recursive(dep); + } + } + return result; + }; + + std::string result; + Format format = Format::MSG; + try { + result = append_recursive(DefinitionIdentifier{format, root_package_resource_name}); + } catch (const DefinitionNotFoundError & err) { + // log that we've fallen back + RCUTILS_LOG_WARN_NAMED("rosbag2_storage_mcap", "no .msg definition for %s, falling back to IDL", + err.what()); + format = Format::IDL; + DefinitionIdentifier root_definition_identifier{format, root_package_resource_name}; + result = delimiter(root_definition_identifier) + append_recursive(root_definition_identifier); + } + return std::make_pair(format, result); +} +} // namespace rosbag2_storage_mcap::internal diff --git a/rosbag2_storage_mcap/test/rosbag2_storage_mcap/mcap_writer_options_zstd.yaml b/rosbag2_storage_mcap/test/rosbag2_storage_mcap/mcap_writer_options_zstd.yaml new file mode 100644 index 0000000000..cc6a970cd9 --- /dev/null +++ b/rosbag2_storage_mcap/test/rosbag2_storage_mcap/mcap_writer_options_zstd.yaml @@ -0,0 +1,8 @@ +noCRC: false +noChunking: false +noMessageIndex: false +noSummary: false +chunkSize: 786432 +compression: "Zstd" +compressionLevel: "Fast" +forceCompression: true \ No newline at end of file diff --git a/rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_mcap_storage.cpp b/rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_mcap_storage.cpp new file mode 100644 index 0000000000..c2f5665e51 --- /dev/null +++ b/rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_mcap_storage.cpp @@ -0,0 +1,172 @@ +// Copyright 2022, Foxglove Technologies. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rclcpp/serialization.hpp" +#include "rclcpp/serialized_message.hpp" +#include "rcpputils/filesystem_helper.hpp" +#include "rosbag2_cpp/reader.hpp" +#include "rosbag2_cpp/readers/sequential_reader.hpp" +#include "rosbag2_cpp/writer.hpp" +#include "rosbag2_cpp/writers/sequential_writer.hpp" +#ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS + #include "rosbag2_storage/storage_options.hpp" +using StorageOptions = rosbag2_storage::StorageOptions; +#else + #include "rosbag2_cpp/storage_options.hpp" +using StorageOptions = rosbag2_cpp::StorageOptions; +#endif +#include "rosbag2_test_common/temporary_directory_fixture.hpp" +#include "std_msgs/msg/string.hpp" + +#include + +#include +#include + +using namespace ::testing; // NOLINT +using TemporaryDirectoryFixture = rosbag2_test_common::TemporaryDirectoryFixture; + +TEST_F(TemporaryDirectoryFixture, can_write_and_read_basic_mcap_file) +{ + auto uri = rcpputils::fs::path(temporary_dir_path_) / "bag"; + auto expected_bag = uri / "bag_0.mcap"; + const int64_t timestamp_nanos = 100; // arbitrary value + rcutils_time_point_value_t time_stamp{timestamp_nanos}; + const std::string topic_name = "test_topic"; + const std::string message_data = "Test Message 1"; + const std::string storage_id = "mcap"; + // COMPATIBILITY(foxy) + // using verbose APIs for Foxy compatibility which did not yet provide plain-message API + rclcpp::Serialization serialization; + + { + StorageOptions options; + options.uri = uri.string(); + options.storage_id = storage_id; + rosbag2_storage::TopicMetadata topic_metadata; + topic_metadata.name = topic_name; + topic_metadata.type = "std_msgs/msg/String"; + + std_msgs::msg::String msg; + msg.data = message_data; + + rosbag2_cpp::Writer writer{std::make_unique()}; +#ifndef ROSBAG2_STORAGE_MCAP_WRITER_CREATES_DIRECTORY + rcpputils::fs::create_directories(uri); +#endif + writer.open(options, rosbag2_cpp::ConverterOptions{}); + writer.create_topic(topic_metadata); + + auto serialized_msg = std::make_shared(); + serialization.serialize_message(&msg, serialized_msg.get()); + + // This is really kludgy, it's due to a mismatch between types in the rclcpp serialization API + // and the historical Foxy serialized APIs. Prevents the hacked shared ptr from deleting the + // data that `serialized_bag_msg` should reasonably expect to continue existing. + // For this example it wouldn't matter, but in case anybody extends this test, it's for safety. + auto serialized_bag_msg = std::make_shared(); + serialized_bag_msg->serialized_data = std::shared_ptr( + const_cast(&serialized_msg->get_rcl_serialized_message()), + [](rcutils_uint8_array_t * /* data */) {}); + serialized_bag_msg->time_stamp = time_stamp; + serialized_bag_msg->topic_name = topic_name; + writer.write(serialized_bag_msg); + EXPECT_TRUE(expected_bag.is_regular_file()); + } + { + StorageOptions options; + options.uri = expected_bag.string(); + options.storage_id = storage_id; + + rosbag2_cpp::Reader reader{std::make_unique()}; + reader.open(options, rosbag2_cpp::ConverterOptions{}); + EXPECT_TRUE(reader.has_next()); + + std_msgs::msg::String msg; + auto serialized_bag_msg = reader.read_next(); + rclcpp::SerializedMessage extracted_serialized_msg(*serialized_bag_msg->serialized_data); + serialization.deserialize_message(&extracted_serialized_msg, &msg); + EXPECT_EQ(msg.data, message_data); + } +} + +#ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS +// This test disabled on Foxy since StorageOptions doesn't have storage_config_uri field on it +TEST_F(TemporaryDirectoryFixture, can_write_mcap_with_zstd_configured_from_yaml) +{ + auto uri = rcpputils::fs::path(temporary_dir_path_) / "bag"; + auto expected_bag = uri / "bag_0.mcap"; + const int64_t timestamp_nanos = 100; // arbitrary value + rcutils_time_point_value_t time_stamp{timestamp_nanos}; + const std::string topic_name = "test_topic"; + const std::string message_data = "Test Message 1"; + const std::string storage_id = "mcap"; + const std::string config_path = _TEST_RESOURCES_DIR_PATH; + // COMPATIBILITY(foxy) + // using verbose APIs for Foxy compatibility which did not yet provide plain-message API + rclcpp::Serialization serialization; + + { + StorageOptions options; + options.uri = uri.string(); + options.storage_id = storage_id; + options.storage_config_uri = config_path + "/mcap_writer_options_zstd.yaml"; + rosbag2_storage::TopicMetadata topic_metadata; + topic_metadata.name = topic_name; + topic_metadata.type = "std_msgs/msg/String"; + + std_msgs::msg::String msg; + msg.data = message_data; + + rosbag2_cpp::Writer writer{std::make_unique()}; + #ifndef ROSBAG2_STORAGE_MCAP_WRITER_CREATES_DIRECTORY + rcpputils::fs::create_directories(uri); + #endif + writer.open(options, rosbag2_cpp::ConverterOptions{}); + writer.create_topic(topic_metadata); + + auto serialized_msg = std::make_shared(); + serialization.serialize_message(&msg, serialized_msg.get()); + + // This is really kludgy, it's due to a mismatch between types in the rclcpp serialization API + // and the historical Foxy serialized APIs. Prevents the hacked shared ptr from deleting the + // data that `serialized_bag_msg` should reasonably expect to continue existing. + // For this example it wouldn't matter, but in case anybody extends this test, it's for safety. + auto serialized_bag_msg = std::make_shared(); + serialized_bag_msg->serialized_data = std::shared_ptr( + const_cast(&serialized_msg->get_rcl_serialized_message()), + [](rcutils_uint8_array_t * /* data */) {}); + serialized_bag_msg->time_stamp = time_stamp; + serialized_bag_msg->topic_name = topic_name; + writer.write(serialized_bag_msg); + writer.write(serialized_bag_msg); + EXPECT_TRUE(expected_bag.is_regular_file()); + } + { + StorageOptions options; + options.uri = expected_bag.string(); + options.storage_id = storage_id; + + rosbag2_cpp::Reader reader{std::make_unique()}; + reader.open(options, rosbag2_cpp::ConverterOptions{}); + EXPECT_TRUE(reader.has_next()); + + std_msgs::msg::String msg; + auto serialized_bag_msg = reader.read_next(); + rclcpp::SerializedMessage extracted_serialized_msg(*serialized_bag_msg->serialized_data); + serialization.deserialize_message(&extracted_serialized_msg, &msg); + EXPECT_EQ(msg.data, message_data); + } +} +#endif // #ifdef ROSBAG2_STORAGE_MCAP_HAS_STORAGE_OPTIONS diff --git a/rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_message_definition_cache.cpp b/rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_message_definition_cache.cpp new file mode 100644 index 0000000000..d0cbe70f4d --- /dev/null +++ b/rosbag2_storage_mcap/test/rosbag2_storage_mcap/test_message_definition_cache.cpp @@ -0,0 +1,126 @@ +// Copyright 2022, Foxglove Technologies. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gmock/gmock.h" +#include "rosbag2_storage_mcap/message_definition_cache.hpp" + +#include +#include + +using rosbag2_storage_mcap::internal::Format; +using rosbag2_storage_mcap::internal::MessageDefinitionCache; +using rosbag2_storage_mcap::internal::parse_dependencies; +using ::testing::UnorderedElementsAre; + +TEST(test_message_definition_cache, can_find_idl_includes) +{ + const char sample[] = + R"r( +#include "rosbag2_storage_mcap_testdata/msg/BasicIdlA.idl" + +#include + +module rosbag2_storage_mcap_testdata { + module msg { + struct ComplexIdl { + rosbag2_storage_mcap_testdata::msg::BasicIdlA a; + rosbag2_storage_mcap_testdata::msg::BasicIdlB b; + }; + }; +}; + +)r"; + std::set dependencies = parse_dependencies(Format::IDL, sample, ""); + ASSERT_THAT(dependencies, UnorderedElementsAre("rosbag2_storage_mcap_testdata/msg/BasicIdlA", + "rosbag2_storage_mcap_testdata/msg/BasicIdlB")); +} + +TEST(test_message_definition_cache, can_find_msg_deps) +{ + MessageDefinitionCache cache; + auto [format, content] = cache.get_full_text("rosbag2_storage_mcap_testdata/ComplexMsg"); + ASSERT_EQ(format, Format::MSG); + ASSERT_EQ(content, + R"r(rosbag2_storage_mcap_testdata/BasicMsg b + +================================================================================ +MSG: rosbag2_storage_mcap_testdata/BasicMsg +float32 c +)r"); +} + +TEST(test_message_definition_cache, can_find_idl_deps) +{ + MessageDefinitionCache cache; + auto [format, content] = cache.get_full_text("rosbag2_storage_mcap_testdata/msg/ComplexIdl"); + EXPECT_EQ(format, Format::IDL); + EXPECT_EQ(content, + R"r(================================================================================ +IDL: rosbag2_storage_mcap_testdata/msg/ComplexIdl +#include "rosbag2_storage_mcap_testdata/msg/BasicIdl.idl" + +module rosbag2_storage_mcap_testdata { + module msg { + struct ComplexIdl { + rosbag2_storage_mcap_testdata::msg::BasicIdl a; + }; + }; +}; + +================================================================================ +IDL: rosbag2_storage_mcap_testdata/msg/BasicIdl +module rosbag2_storage_mcap_testdata { + module msg { + struct BasicIdl { + float x; + }; + }; +}; +)r"); +} + +TEST(test_message_definition_cache, can_resolve_msg_with_idl_deps) +{ + MessageDefinitionCache cache; + auto [format, content] = + cache.get_full_text("rosbag2_storage_mcap_testdata/msg/ComplexMsgDependsOnIdl"); + EXPECT_EQ(format, Format::IDL); + EXPECT_EQ(content, + R"r(================================================================================ +IDL: rosbag2_storage_mcap_testdata/msg/ComplexMsgDependsOnIdl +// generated from rosidl_adapter/resource/msg.idl.em +// with input from rosbag2_storage_mcap_testdata/msg/ComplexMsgDependsOnIdl.msg +// generated code does not contain a copyright notice + +#include "rosbag2_storage_mcap_testdata/msg/BasicIdl.idl" + +module rosbag2_storage_mcap_testdata { + module msg { + struct ComplexMsgDependsOnIdl { + rosbag2_storage_mcap_testdata::msg::BasicIdl a; + }; + }; +}; + +================================================================================ +IDL: rosbag2_storage_mcap_testdata/msg/BasicIdl +module rosbag2_storage_mcap_testdata { + module msg { + struct BasicIdl { + float x; + }; + }; +}; +)r"); +} diff --git a/rosbag2_storage_mcap_testdata/CHANGELOG.rst b/rosbag2_storage_mcap_testdata/CHANGELOG.rst new file mode 100644 index 0000000000..951bda065a --- /dev/null +++ b/rosbag2_storage_mcap_testdata/CHANGELOG.rst @@ -0,0 +1,14 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package rosbag2_storage_mcap_testdata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.6.0 (2022-11-28) +------------------ + +0.5.0 (2022-11-02) +------------------ +* rename test_fixture_interfaces package to testdata (`#64 `_) +* Contributors: James Smith + +0.4.0 (2022-10-06) +------------------ diff --git a/rosbag2_storage_mcap_testdata/CMakeLists.txt b/rosbag2_storage_mcap_testdata/CMakeLists.txt new file mode 100644 index 0000000000..d41a600298 --- /dev/null +++ b/rosbag2_storage_mcap_testdata/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.8) +project(rosbag2_storage_mcap_testdata) + +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wall -Wextra -Wpedantic) +endif() + +# find dependencies +find_package(ament_cmake REQUIRED) +find_package(rosidl_default_generators REQUIRED) +rosidl_generate_interfaces(${PROJECT_NAME} + "msg/ComplexIdl.idl" + "msg/BasicIdl.idl" + "msg/BasicMsg.msg" + "msg/ComplexMsg.msg" + "msg/ComplexMsgDependsOnIdl.msg" + ADD_LINTER_TESTS +) + +ament_package() diff --git a/rosbag2_storage_mcap_testdata/msg/BasicIdl.idl b/rosbag2_storage_mcap_testdata/msg/BasicIdl.idl new file mode 100644 index 0000000000..2bb2e93fe8 --- /dev/null +++ b/rosbag2_storage_mcap_testdata/msg/BasicIdl.idl @@ -0,0 +1,7 @@ +module rosbag2_storage_mcap_testdata { + module msg { + struct BasicIdl { + float x; + }; + }; +}; diff --git a/rosbag2_storage_mcap_testdata/msg/BasicMsg.msg b/rosbag2_storage_mcap_testdata/msg/BasicMsg.msg new file mode 100644 index 0000000000..54f231c265 --- /dev/null +++ b/rosbag2_storage_mcap_testdata/msg/BasicMsg.msg @@ -0,0 +1 @@ +float32 c diff --git a/rosbag2_storage_mcap_testdata/msg/ComplexIdl.idl b/rosbag2_storage_mcap_testdata/msg/ComplexIdl.idl new file mode 100644 index 0000000000..4c5e7ae9eb --- /dev/null +++ b/rosbag2_storage_mcap_testdata/msg/ComplexIdl.idl @@ -0,0 +1,9 @@ +#include "rosbag2_storage_mcap_testdata/msg/BasicIdl.idl" + +module rosbag2_storage_mcap_testdata { + module msg { + struct ComplexIdl { + rosbag2_storage_mcap_testdata::msg::BasicIdl a; + }; + }; +}; diff --git a/rosbag2_storage_mcap_testdata/msg/ComplexMsg.msg b/rosbag2_storage_mcap_testdata/msg/ComplexMsg.msg new file mode 100644 index 0000000000..89f48028e2 --- /dev/null +++ b/rosbag2_storage_mcap_testdata/msg/ComplexMsg.msg @@ -0,0 +1 @@ +rosbag2_storage_mcap_testdata/BasicMsg b diff --git a/rosbag2_storage_mcap_testdata/msg/ComplexMsgDependsOnIdl.msg b/rosbag2_storage_mcap_testdata/msg/ComplexMsgDependsOnIdl.msg new file mode 100644 index 0000000000..9b8376a5af --- /dev/null +++ b/rosbag2_storage_mcap_testdata/msg/ComplexMsgDependsOnIdl.msg @@ -0,0 +1 @@ +rosbag2_storage_mcap_testdata/BasicIdl a diff --git a/rosbag2_storage_mcap_testdata/package.xml b/rosbag2_storage_mcap_testdata/package.xml new file mode 100644 index 0000000000..b9ce8f24b4 --- /dev/null +++ b/rosbag2_storage_mcap_testdata/package.xml @@ -0,0 +1,23 @@ + + + + rosbag2_storage_mcap_testdata + 0.6.0 + message definition test fixtures for MCAP schema recording + Foxglove + ROS Tooling Working Group + Apache-2.0 + + ament_cmake + rosidl_default_generators + + ament_lint_auto + ament_lint_common + + + ament_cmake + + + rosidl_interface_packages + + diff --git a/zstd_vendor/no_internal_headers.patch b/zstd_vendor/no_internal_headers.patch index 10793892e4..e091fb4b1e 100644 --- a/zstd_vendor/no_internal_headers.patch +++ b/zstd_vendor/no_internal_headers.patch @@ -1,28 +1,27 @@ -From f1153100884e580dce331e8dd56b260ffb2f04d5 Mon Sep 17 00:00:00 2001 -From: Emerson Knapp -Date: Tue, 2 Feb 2021 17:47:46 -0800 +From e85114a0200c06dc5bb523a6b5764e938b0e8047 Mon Sep 17 00:00:00 2001 +From: James Smith +Date: Thu, 1 Dec 2022 09:57:11 +1100 Subject: [PATCH] Don't install internal headers -Signed-off-by: Emerson Knapp +Signed-off-by: James Smith --- - build/cmake/lib/CMakeLists.txt | 4 ---- - 1 file changed, 4 deletions(-) + build/cmake/lib/CMakeLists.txt | 3 --- + 1 file changed, 3 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt -index 7adca875..0c2d777e 100644 +index 7adca875..c66380b9 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt -@@ -147,10 +147,6 @@ endif () +@@ -147,9 +147,6 @@ endif () # install target install(FILES ${LIBRARY_DIR}/zstd.h - ${LIBRARY_DIR}/deprecated/zbuff.h - ${LIBRARY_DIR}/dictBuilder/zdict.h - ${LIBRARY_DIR}/dictBuilder/cover.h -- ${LIBRARY_DIR}/common/zstd_errors.h + ${LIBRARY_DIR}/common/zstd_errors.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") - if (ZSTD_BUILD_SHARED) -- -2.17.1 +2.34.1