Skip to content

Commit

Permalink
Switch back to internal builds of igraph and HDF5. (#89)
Browse files Browse the repository at this point in the history
This avoids headaches from guessing ABI compatibility across different
Emscripten versions and build flags. With suitable caching, we can avoid
achieve prebuilt binaries but with more suitable integration with our own CI.

We also abandon the dedicated scran.js Docker image in favor of the generic
emsdk + CMake image. This reduces the complexity of the CI and avoids stale
object files in the CMake build. We use more caching to save the Wasm binary so
that it doesn't need to be rebuilt if the C++ code hasn't changed.

Also got rid of preamble.js as it's no longer needed in the new toolchain.
  • Loading branch information
LTLA authored Oct 2, 2024
1 parent bb498e0 commit ecd2b28
Show file tree
Hide file tree
Showing 10 changed files with 186 additions and 44 deletions.
89 changes: 71 additions & 18 deletions .github/workflows/run-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,57 +6,110 @@ on:

name: Test JS bindings

env:
IMAGE_VERSION: master

jobs:
prepare-version:
runs-on: ubuntu-latest
steps:
- run: echo "null"
outputs:
version: ${{ env.IMAGE_VERSION }}

# Building the RDS files for the various RDS-reading utilities.
build_rds:
create_rds:
runs-on: ubuntu-latest
container: rocker/r-base:latest

steps:
- name: Checkout repo
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Build RDS files
run: |
cd tests/rds
R -f generate.R
- name: Upload RDS files
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: rds-files
path: tests/rds/*.rds

check:
build_deps:
runs-on: ubuntu-latest
needs: build_rds
container: ghcr.io/kanaverse/scran.js-docker/builder:master
defaults:
run:
working-directory: /scran.js
needs: [prepare-version]
container: ghcr.io/kanaverse/emcmake-docker/builder:${{ needs.prepare-version.outputs.version }}

steps:
- name: Get to the right branch
- name: Checkout repo
uses: actions/checkout@v4

- name: Cache installed
id: installed
uses: actions/cache@v4
with:
path: extern/installed
key: deps-${{ hashFiles('**/extern/**/build.sh') }}-${{ env.IMAGE_VERSION }}

- name: Build HDF5
if: steps.installed.outputs.cache-hit != 'true'
run: |
cd extern/hdf5
./build.sh
- name: Build igraph
if: steps.installed.outputs.cache-hit != 'true'
run: |
git fetch --all
git checkout $GITHUB_SHA
cd extern/igraph
./build.sh
test:
runs-on: ubuntu-latest
needs: [ create_rds, build_deps, prepare-version]
container: ghcr.io/kanaverse/emcmake-docker/builder:${{ needs.prepare-version.outputs.version }}

steps:
- name: Checkout repo
uses: actions/checkout@v4

- name: Set up Node
uses: actions/setup-node@v3
uses: actions/setup-node@v4
with:
node-version: 16
node-version: 18

- name: Update node build
run: bash build.sh main
- name: Cache Modules
uses: actions/cache@v4
with:
path: '**/node_modules'
key: npm-${{ hashFiles('**/package.json') }}

- name: Update NPM packages
run: npm i --include=dev

- name: Cache installed
uses: actions/cache@v4
with:
path: extern/installed
key: deps-${{ hashFiles('**/extern/**/build.sh') }}-${{ env.IMAGE_VERSION }}

- name: Cache node build
id: wasm-build
uses: actions/cache@v4
with:
path: js/wasm
key: wasm-${{ hashFiles('**/build.sh', '**/CMakeLists.txt', '**/src/*') }}-${{ env.IMAGE_VERSION }}

- name: Update node build
if: steps.wasm-build.outputs.cache-hit != 'true'
run: bash build.sh main

- name: Download RDS files
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
with:
name: rds-files
path: /scran.js/tests/rds
path: tests/rds

- name: Run tests
run: |
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
extern/installed
build_*
docs/html/
docs/latex/
Expand Down
9 changes: 7 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ project(scran_wasm

set(CMAKE_CXX_STANDARD 17)

# Need the CMAKE_FIND_ROOT_PATH_BOTH to override Emscripten's overrides (see emscripten/issues#6595)
find_package(igraph REQUIRED CONFIG CMAKE_FIND_ROOT_PATH_BOTH)
find_package(HDF5 REQUIRED COMPONENTS C CXX CONFIG CMAKE_FIND_ROOT_PATH_BOTH)
find_package(ZLIB REQUIRED)
add_subdirectory(extern)

add_executable(
Expand Down Expand Up @@ -66,8 +70,9 @@ target_link_libraries(
mnncorrect
qdtsne
umappp
hdf5-wasm-cpp
igraph
hdf5-static
hdf5_cpp-static
igraph::igraph
singlepp
rds2cpp
)
Expand Down
7 changes: 6 additions & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,12 @@ if [ ! -e $builddir ]
then
mkdir $builddir
echo "{}" > $builddir/package.json # avoid assuming ES6 syntax for igraph config scripts.
emcmake cmake -S . -B $builddir -DCOMPILE_NODE=${node_flag} -DCMAKE_BUILD_TYPE=Release
emcmake cmake \
-S . \
-B $builddir \
-DCOMPILE_NODE=${node_flag} \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH=extern/installed
fi

cd $builddir
Expand Down
17 changes: 0 additions & 17 deletions extern/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ endif()

include(FetchContent)

## TATAMI RELATED CONTENT ###
FetchContent_Declare(
tatami
GIT_REPOSITORY https://github.com/tatami-inc/tatami
Expand All @@ -40,20 +39,6 @@ FetchContent_Declare(
GIT_TAG 20ec1fb64e0419907d3094827b5d9134854a0119
)

## Prebuilt static libraries ###
FetchContent_Declare(
igraph
URL https://github.com/kanaverse/igraph-wasm/releases/download/v0.2.1-pthreads_3.1.43/igraph-0.10.6-wasm.tar.gz # need the thread-aware version.
URL_HASH SHA256=a40595001e0b39ef715f4a84bb83f3bb668ee8eed3ac55bd206212d45d7c53f1
)

FetchContent_Declare(
h5wasm
URL https://github.com/kanaverse/libhdf5-wasm/releases/download/v0.3.0_3.1.43/libhdf5-1_12_2-wasm.tar.gz
URL_HASH SHA256=b2e58b102fe98326c77f1a160025c47a062c1a205818568680d5f5ed78df801c
)

## Other stuff ###
FetchContent_Declare(
irlba
GIT_REPOSITORY https://github.com/LTLA/CppIrlba
Expand Down Expand Up @@ -106,8 +91,6 @@ FetchContent_MakeAvailable(tatami)
FetchContent_MakeAvailable(tatami_layered)
FetchContent_MakeAvailable(tatami_mtx)
FetchContent_MakeAvailable(tatami_hdf5)
FetchContent_MakeAvailable(igraph)
FetchContent_MakeAvailable(h5wasm)
FetchContent_MakeAvailable(irlba)
FetchContent_MakeAvailable(umappp)
FetchContent_MakeAvailable(scran)
Expand Down
3 changes: 3 additions & 0 deletions extern/hdf5/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
hdf5*
package.json
build-*
55 changes: 55 additions & 0 deletions extern/hdf5/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#!/bin/bash

set -e
set -u

HDF5_VERSION=1.14.5
HDF5_HASH=ec2e13c52e60f9a01491bb3158cb3778c985697131fc6a342262d32a26e58e44
SOURCE_DIR=hdf5-${HDF5_VERSION}

if [[ ! -e ${SOURCE_DIR} ]]
then
wget -q https://github.com/HDFGroup/hdf5/releases/download/hdf5_${HDF5_VERSION}/hdf5-${HDF5_VERSION}.tar.gz -O hdf5.tar.gz
OBSERVED_HASH=($(shasum -a 256 hdf5.tar.gz))
if [[ ${OBSERVED_HASH} != ${HDF5_HASH} ]]
then
echo "hash mismatch for ${HDF5_VERSION} (got ${OBSERVED_HASH})"
exit 1
fi
tar -xf hdf5.tar.gz

# Some source-editing shenanigans are required to deal with the lack of
# FE_INVALID in Emscripten, see emscripten-core/emscripten#22005. Hey,
# I don't make the rules.
offender=${SOURCE_DIR}/src/H5Tinit_float.c
cat ${offender} | sed "s/feclearexcept(FE_INVALID)/0/" > tmp
mv tmp ${offender}
fi

BUILD_DIR=build-${HDF5_VERSION}
if [ ! -e ${BUILD_DIR} ]
then
mkdir -p ../installed
coreflags="-pthread" # propagating compile flags from the root scran.js CMakeLists.txt.
echo "{}" > package.json # avoid assuming ES6 module syntax from the root scran.js package.json.
emcmake cmake \
-S ${SOURCE_DIR} \
-B ${BUILD_DIR} \
-DCMAKE_C_FLAGS="${coreflags}" \
-DCMAKE_CXX_FLAGS="${coreflags}" \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$(pwd)/../installed \
-DBUILD_SHARED_LIBS=OFF \
-DBUILD_TESTING=OFF \
-DHDF5_BUILD_EXAMPLES=OFF \
-DHDF5_BUILD_TOOLS=OFF \
-DHDF5_BUILD_UTILS=OFF \
-DHDF5_BUILD_CPP_LIB=ON \
-DHDF5_ENABLE_Z_LIB_SUPPORT=ON \
-DZLIB_USE_EXTERNAL=OFF \
-DHDF5_ENABLE_SZIP_SUPPORT=OFF
fi

cd ${BUILD_DIR}
emmake make
emmake make install
3 changes: 3 additions & 0 deletions extern/igraph/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
igraph*
package.json
build-*
40 changes: 40 additions & 0 deletions extern/igraph/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/bash

set -e
set -u

IGRAPH_VERSION=0.10.13
IGRAPH_HASH=c6dc44324f61f52c098bedb81f6a602365d39d692d5068ca4fc3734b2a15e64c
SOURCE_DIR=igraph-${IGRAPH_VERSION}

if [[ ! -e ${SOURCE_DIR} ]]
then
wget -q https://github.com/igraph/igraph/releases/download/${IGRAPH_VERSION}/igraph-${IGRAPH_VERSION}.tar.gz -O igraph.tar.gz
OBSERVED_HASH=($(shasum -a 256 igraph.tar.gz))
if [[ ${OBSERVED_HASH} != ${IGRAPH_HASH} ]]
then
echo "hash mismatch for ${IGRAPH_VERSION} (got ${OBSERVED_HASH})"
exit 1
fi
tar -xf igraph.tar.gz
fi

BUILD_DIR=build-${IGRAPH_VERSION}
if [ ! -e ${BUILD_DIR} ]
then
mkdir -p ../installed
coreflags="-pthread" # propagating compile flags from the root scran.js CMakeLists.txt.
echo "{}" > package.json # avoid assuming ES6 module syntax from the root scran.js package.json.
emcmake cmake \
-S ${SOURCE_DIR} \
-B ${BUILD_DIR} \
-DCMAKE_C_FLAGS="${coreflags}" \
-DCMAKE_CXX_FLAGS="${coreflags}" \
-DIGRAPH_WARNINGS_AS_ERRORS=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$(pwd)/../installed
fi

cd ${BUILD_DIR}
emmake make
emmake make install
6 changes: 0 additions & 6 deletions misc/preamble.js

This file was deleted.

0 comments on commit ecd2b28

Please sign in to comment.