From bcd9f453881403b137ca807e8f4f16671208d8c2 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:49:51 -0700 Subject: [PATCH] Fix streaming (#65) * add openpilot * remove opendbc and cereal * symlinks * use uv! * source venv * fix FindCereal.cmake * bump openpilot * add common too * not needed * fix build * fix macos * test * test * need op too * portaudio --- .github/workflows/openpilot.yaml | 13 ++++++++----- .gitmodules | 9 +++------ 3rdparty/cereal | 2 +- 3rdparty/common | 1 + 3rdparty/msgq | 1 + 3rdparty/opendbc | 2 +- 3rdparty/opendbc_repo | 1 - 3rdparty/openpilot | 1 + Dockerfile | 21 ++++++++++----------- build.sh | 2 +- cmake/FindCereal.cmake | 19 ++++++++++++------- 11 files changed, 39 insertions(+), 33 deletions(-) mode change 160000 => 120000 3rdparty/cereal create mode 120000 3rdparty/common create mode 120000 3rdparty/msgq delete mode 160000 3rdparty/opendbc_repo create mode 160000 3rdparty/openpilot diff --git a/.github/workflows/openpilot.yaml b/.github/workflows/openpilot.yaml index e247d65b9..4ec0d7f35 100644 --- a/.github/workflows/openpilot.yaml +++ b/.github/workflows/openpilot.yaml @@ -16,12 +16,12 @@ jobs: arch: ${{ fromJson( ((github.repository == 'commaai/plotjuggler') && ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/plotjuggler'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} + (github.event.pull_request.head.repo.full_name == 'commaai/plotjuggler'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-4vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }} steps: - uses: actions/checkout@v2 with: - submodules: true + submodules: recursive - name: build run: ./build.sh - name: zip plugins @@ -42,18 +42,21 @@ jobs: steps: - uses: actions/checkout@v2 with: - submodules: true + submodules: recursive - uses: actions/setup-python@v2 with: python-version: '3.11.x' - name: setup dependencies run: | - brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar - sudo pip3 install --no-cache-dir -r 3rdparty/opendbc_repo/requirements.txt + brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar portaudio + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv + uv pip install 3rdparty/openpilot 3rdparty/openpilot/opendbc_repo - name: build run: | export PYTHONPATH=$PWD/3rdparty # force usage of protobuf@3 instead of protobuf + source .venv/bin/activate mkdir -p build && cd build && cmake -DCMAKE_PREFIX_PATH="$(brew --prefix)/opt/protobuf@3" .. && make -j8 - name: zip plugins run: eval "$ZIP" diff --git a/.gitmodules b/.gitmodules index 5a50669ae..0e1918218 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "3rdparty/cereal"] - path = 3rdparty/cereal - url = ../../commaai/cereal.git -[submodule "3rdparty/opendbc"] - path = 3rdparty/opendbc_repo - url = ../../commaai/opendbc.git +[submodule "3rdparty/openpilot"] + path = 3rdparty/openpilot + url = https://github.com/commaai/openpilot diff --git a/3rdparty/cereal b/3rdparty/cereal deleted file mode 160000 index 8ae5c51a6..000000000 --- a/3rdparty/cereal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8ae5c51a6515c171f3a82c9511a18d79a9e55c07 diff --git a/3rdparty/cereal b/3rdparty/cereal new file mode 120000 index 000000000..19c42a96d --- /dev/null +++ b/3rdparty/cereal @@ -0,0 +1 @@ +openpilot/cereal \ No newline at end of file diff --git a/3rdparty/common b/3rdparty/common new file mode 120000 index 000000000..47b9db83f --- /dev/null +++ b/3rdparty/common @@ -0,0 +1 @@ +openpilot/common \ No newline at end of file diff --git a/3rdparty/msgq b/3rdparty/msgq new file mode 120000 index 000000000..65a23eec0 --- /dev/null +++ b/3rdparty/msgq @@ -0,0 +1 @@ +openpilot/msgq_repo/msgq \ No newline at end of file diff --git a/3rdparty/opendbc b/3rdparty/opendbc index 7cd9a5bd1..e09a6047b 120000 --- a/3rdparty/opendbc +++ b/3rdparty/opendbc @@ -1 +1 @@ -opendbc_repo/opendbc \ No newline at end of file +openpilot/opendbc_repo/opendbc \ No newline at end of file diff --git a/3rdparty/opendbc_repo b/3rdparty/opendbc_repo deleted file mode 160000 index a40652d34..000000000 --- a/3rdparty/opendbc_repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a40652d341ea3161a6a10001fa74284d9d650f8c diff --git a/3rdparty/openpilot b/3rdparty/openpilot new file mode 160000 index 000000000..756d64e2c --- /dev/null +++ b/3rdparty/openpilot @@ -0,0 +1 @@ +Subproject commit 756d64e2c3a38983caa56cefd55e58fb663332c6 diff --git a/Dockerfile b/Dockerfile index d75107027..0d7fe098d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,17 +35,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libsqlite3-dev \ clang \ ocl-icd-opencl-dev \ - opencl-headers + opencl-headers \ + portaudio19-dev -RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash -ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}" -RUN pyenv install 3.11.4 && \ - pyenv global 3.11.4 && \ - pyenv rehash +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/root/.cargo/bin:${PATH}" +RUN uv venv --python 3.11.4 +RUN uv pip install pkgconfig jinja2 Cython -RUN pip3 install pkgconfig jinja2 - -# installs scons, pycapnp, cython, etc. ENV PYTHONPATH /tmp/plotjuggler/3rdparty -COPY 3rdparty/opendbc_repo/requirements.txt /tmp/ -RUN pip3 install Cython && pip3 install --no-cache-dir -r /tmp/requirements.txt +COPY 3rdparty/openpilot/opendbc_repo/pyproject.toml /tmp/opendbc_repo/pyproject.toml +COPY 3rdparty/openpilot/pyproject.toml /tmp/openpilot/pyproject.toml +RUN uv pip install --no-cache-dir /tmp/opendbc_repo +RUN uv pip install --no-cache-dir /tmp/openpilot diff --git a/build.sh b/build.sh index 81dc6e6b0..ddb4a4e0b 100755 --- a/build.sh +++ b/build.sh @@ -6,4 +6,4 @@ docker run \ --rm \ --volume $PWD:/tmp/plotjuggler \ --workdir /tmp/plotjuggler plotjuggler:latest \ - /bin/bash -c "mkdir -p build && cd build && cmake .. && make -j$(nproc)" + /bin/bash -c "source /.venv/bin/activate && mkdir -p build && cd build && cmake .. && make -j$(nproc)" diff --git a/cmake/FindCereal.cmake b/cmake/FindCereal.cmake index 3d9dd1579..4f2f8af3f 100644 --- a/cmake/FindCereal.cmake +++ b/cmake/FindCereal.cmake @@ -5,8 +5,13 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) # build cereal before we search for required Cereal plugin libs below execute_process( - COMMAND scons -j8 --minimal - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/cereal + COMMAND scons -j8 --minimal -u + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/cereal + ) + + execute_process( + COMMAND scons -j8 --minimal -u + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/msgq ) find_path(Cereal_INCLUDE_DIRS @@ -29,14 +34,14 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) NAMES cereal PATHS - ${CMAKE_SOURCE_DIR}/3rdparty/cereal + ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/cereal ) - find_library(Cereal_LIBRARY_MESSAGING + find_library(Cereal_LIBRARY_MSGQ NAMES - messaging + msgq PATHS - ${CMAKE_SOURCE_DIR}/3rdparty/cereal + ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/msgq_repo ) find_library(Cereal_LIBRARY_ZMQ @@ -47,7 +52,7 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) set(Cereal_LIBRARIES ${Cereal_LIBRARIES} ${Cereal_LIBRARY_CEREAL} - ${Cereal_LIBRARY_MESSAGING} + ${Cereal_LIBRARY_MSGQ} ${Cereal_LIBRARY_ZMQ} )