Skip to content

Commit

Permalink
feature: Publish cohort SDK NPMs with bundled rust src (#85)
Browse files Browse the repository at this point in the history
* feat: Publish sdk to GH Actions Packages

* feat: Added CI build step for Initiator JS

* feat: Added CI build step for Replicator JS

* fix: Use latest rdkafka

* chore: Rename build job

* chore: Add README.md content to initiator and replicator JS

* fix: Review comments, update readme.

* chore: Update readme.

* chore: Update comments.

* chore: Add comments
  • Loading branch information
fmarek-kindred authored Oct 11, 2023
1 parent 86e1454 commit c809e90
Show file tree
Hide file tree
Showing 54 changed files with 1,680 additions and 4,035 deletions.
34 changes: 14 additions & 20 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -21,50 +21,44 @@ PG_HOST=$LOCAL_HOST_IP
PG_PORT=5432
PG_USER=admin
PG_PASSWORD=admin
PG_DATABASE=talos_certifier
PG_DATABASE=talos-certifier-dev
ADMIN_PG_USER=admin
ADMIN_PG_PASSWORD=admin
ADMIN_PG_DATABASE=postgres

# Mock

# CONFIG
RUST_LOG="warn"
DB_MOCK=false
CERTIFIER_MOCK=false

# CONFIG
RUST_LOG=debug

# Cohort configs
AGENT_NAME="Sample Cohort"
COHORT_NAME="Sample Cohort"
COHORT_WORKLOAD_DURATION=30

AGENT_BUFFER_SIZE=10000
AGENT_TIMEOUT_MS=5000

KAFKA_FETCH_WAIT_MAX_MS=6000
KAFKA_MESSAGE_TIMEOUT_MS=15000
KAFKA_ENQUEUE_TIMEOUT_MS=10
AGENT_BUFFER_SIZE=10000000
# Timeout used by synchronous rdkafka publisher.
AGENT_KAFKA_PRODUCER_SEND_TIMEOUT_MS=10
AGENT_TIMEOUT_MS=20000
KAFKA_LOG_LEVEL=info

COHORT_PG_HOST=127.0.0.1
COHORT_PG_PORT=5432
COHORT_PG_USER=postgres
COHORT_PG_PASSWORD=admin
COHORT_PG_DATABASE=talos-sample-cohort-dev
COHORT_PG_POOL_SIZE=10
COHORT_ADMIN_PG_DATABASE=postgres
COHORT_PG_POOL_SIZE=100

# Replicator and Statemap Installer Services
REPLICATOR_COMMIT_FREQ_MS=10000
REPLICATOR_ENABLE_STATS=false
REPLICATOR_KAFKA_COMMIT_FREQ_MS=10000
REPLICATOR_ENABLE_STATS=true
REPLICATOR_CHANNEL_SIZE=100000

REPLICATOR_SUFFIX_CAPACITY=100000
REPLICATOR_SUFFIX_CAPACITY=800000
REPLICATOR_SUFFIX_PRUNE_THRESHOLD=1

STATEMAP_QUEUE_ENABLE_STATS=true
STATEMAP_QUEUE_CLEANUP_FREQUENCY_MS=10000
STATEMAP_INSTALLER_THREAD_POOL=10

STATEMAP_INSTALLER_THREAD_POOL=100

# For example banking cohort
BANK_REPLICATOR_KAFKA_GROUP_ID="talos-replicator-dev"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ jobs:
- run: rustup component add rustfmt clippy
- name: cargo build
run: cargo build --release
- run: scripts/pre-commit-checks.sh
- run: scripts/pre-commit-checks.sh
4 changes: 0 additions & 4 deletions .github/workflows/master.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,3 @@ jobs:
uses: ./.github/workflows/build.yml
coverage:
uses: ./.github/workflows/coverage.yml




120 changes: 120 additions & 0 deletions .github/workflows/npm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
name: cohort_sdk_js
on:
workflow_call:
#push:
env:
DEBUG: "napi:*"
OPENSSL_DIR: /usr/local/Cellar/openssl@1.1/1.1.1w
SASL2_DIR: /usr/local/Cellar/cyrus-sasl/2.1.28_2
LDFLAGS: "-L/usr/local/Cellar/openssl@1.1/1.1.1w/lib -L/usr/local/opt/cyrus-sasl/lib"
CPPFLAGS: "-I/usr/local/Cellar/openssl@1.1/1.1.1w/include -I/usr/local/opt/cyrus-sasl/include"
jobs:
npm:
strategy:
fail-fast: false
matrix:
settings:
- host: macos-latest
target: x86_64-apple-darwin
jsPackageHome: packages/cohort_sdk_js
finalPackageName: packages/cohort_sdk_js/cohort_sdk_js.darwin-x64.node
customiseHostCommand: echo 'No customizations are needed'
buildCommand: npm run build -- --target x86_64-apple-darwin

- host: macos-latest
target: aarch64-apple-darwin
jsPackageHome: packages/cohort_sdk_js
finalPackageName: packages/cohort_sdk_js/cohort_sdk_js.darwin-arm64.node
customiseHostCommand: |-
touch ~/.cargo/config
cat <<EOT >> ~/.cargo/config
[target.aarch64-apple-darwin]
rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
EOT
echo "~/.cargo/config"
cat ~/.cargo/config
buildCommand: |-
npm run build -- --target aarch64-apple-darwin
- host: windows-latest
target: x86_64-pc-windows-msvc
jsPackageHome: packages\\cohort_sdk_js
finalPackageName: packages\cohort_sdk_js\cohort_sdk_js.*.node
customiseHostCommand: ECHO 'No customizations are needed'
# customiseHostCommand: |-
# choco install openssl --version=1.1.1.2100
# ECHO "C:\Program Files\OpenSSL\="
# ls -l "C:\\Program Files\\OpenSSL"
# setx OPENSSL_DIR "C:\\Program Files\\OpenSSL"
buildCommand: npm run build -- --target x86_64-pc-windows-msvc

- host: ubuntu-latest
target: x86_64-unknown-linux-gnu
jsPackageHome: packages/cohort_sdk_js
finalPackageName: packages/cohort_sdk_js/cohort_sdk_js.linux-x64-gnu.node
customiseHostCommand: sudo apt-get update -y && sudo apt-get install -y libsasl2-dev
buildCommand: npm run build -- --target x86_64-unknown-linux-gnu

name: stable - ${{ matrix.settings.target }} - node@16
runs-on: ${{ matrix.settings.host }}
steps:
- uses: actions/checkout@v3

- name: Setup node
uses: actions/setup-node@v3
with:
node-version: 16
check-latest: true

- name: Install
uses: actions-rs/toolchain@v1
with:
profile: minimal
override: true
toolchain: stable
target: ${{ matrix.settings.target }}

- name: Cache cargo
uses: actions/cache@v3
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
.cargo-cache
target/
key: ${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}

- name: Customise host image
shell: bash
run: ${{ matrix.settings.customiseHostCommand }}

- name: Show current directory name
shell: bash
run: pwd

- name: List content before build
shell: bash
run: ls -l && ls -l ${{ matrix.settings.jsPackageHome }}

# - name: Cache cargo TODO deal with caching
- name: Install node dependencies
run: cd ${{ matrix.settings.jsPackageHome }} && pwd && ls -l . && npm install -verbose

# - name: Cargo build
# run: cargo build

- name: NAPI-RS build
shell: bash
run: cd ${{ matrix.settings.jsPackageHome }} && ${{ matrix.settings.buildCommand }}

- name: List content after build
shell: bash
run: ls -l && ls -l ${{ matrix.settings.jsPackageHome }}

- name: Upload built binary
uses: actions/upload-artifact@v3
with:
name: bindings-${{ matrix.settings.target }}
path: ${{ matrix.settings.finalPackageName }}
if-no-files-found: error
132 changes: 132 additions & 0 deletions .github/workflows/npm_publish_light.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Creates and publishes @kindredgroup/cohort_sdk_js, @kindredgroup/cohort_sdk_client
# NPM packages. The "cohort_sdk_js" is bundled together with Talos sources so that users
# could build it locally.
# Performs a basic build test by installing "cohort_sdk_client" into
# "cohort_banking_initiator_js", "cohort_banking_replicator_js" apps and then builds apps.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

name: Build and Publish Cohort SDK NPMs
on:
#workflow_call:
push:
env:
SDK_JS_PACKAGE_NAME: "@kindredgroup/cohort_sdk_js"
SDK_CLIENT_PACKAGE_NAME: "@kindredgroup/cohort_sdk_client"
jobs:
npm:
strategy:
fail-fast: false
matrix:
settings:
- host: macos-latest
dirSdkJs: packages/cohort_sdk_js
dirSdkJsClient: cohort_sdk_client
dirCohortInitiator: cohort_banking_initiator_js
dirCohortReplicator: cohort_banking_replicator_js

name: Publish SDK NPS with bundled Rust for node@16
runs-on: ${{ matrix.settings.host }}
steps:
- uses: actions/checkout@v3

- name: Setup node
uses: actions/setup-node@v3
with:
node-version: 16
check-latest: true
registry-url: "https://npm.pkg.github.com"

- name: Customise host image
shell: bash
run: |-
echo "NPM_CONFIG_USERCONFIG=$NPM_CONFIG_USERCONFIG"
cat $NPM_CONFIG_USERCONFIG
echo ""
- name: Setup version for ${{ matrix.settings.dirSdkJs }} NPM
id: sdk-js-version
shell: bash
run: |-
echo "Building SDK JS Library"
cd ${{ matrix.settings.dirSdkJs }}
pwd
ls -l
npm ci --foreground-scripts
../../scripts/github-actions-ci/set-npm-dev-version.sh ${{ env.SDK_JS_PACKAGE_NAME }} "SDK_JS_PACKAGE_VERSION"
echo ""
- name: Build and Publish ${{ matrix.settings.dirSdkJs }}
shell: bash
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |-
cd ${{ matrix.settings.dirSdkJs }}
pwd
ls -l
echo "Current version is"
npm version
npm publish --foreground-scripts
echo ""
- name: Setup version for ${{ matrix.settings.dirSdkJsClient }} NPM
id: sdk-client-version
shell: bash
env:
SDK_JS_PACKAGE_VERSION: ${{ steps.sdk-js-version.outputs.SDK_JS_PACKAGE_VERSION }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |-
echo "Building SDK Client"
cd ${{ matrix.settings.dirSdkJsClient }}
pwd
ls -lah
rm package-lock.json || true
echo "D: npm install $SDK_JS_PACKAGE_NAME@$SDK_JS_PACKAGE_VERSION --foreground-scripts"
npm install "$SDK_JS_PACKAGE_NAME@$SDK_JS_PACKAGE_VERSION" --foreground-scripts
../scripts/github-actions-ci/set-npm-dev-version.sh ${{ env.SDK_CLIENT_PACKAGE_NAME }} "SDK_CLIENT_PACKAGE_VERSION"
echo ""
- name: Build and Publish ${{ matrix.settings.dirSdkJsClient }} NPM
shell: bash
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |-
cd ${{ matrix.settings.dirSdkJsClient }}
pwd
ls -l
echo "Current version is"
npm version
npm ci --foreground-scripts
npm run build --foreground-scripts
npm publish --foreground-scripts
echo ""
- name: Build ${{ matrix.settings.dirCohortInitiator }}
env:
SDK_CLIENT_PACKAGE_VERSION: ${{ steps.sdk-client-version.outputs.SDK_CLIENT_PACKAGE_VERSION }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SKIP_NAPI_RS_STEP: true # this will re-use already bundled native lib
shell: bash
run: |-
cd ${{ matrix.settings.dirCohortInitiator }}
pwd
ls -l
rm package-lock.json || true
echo "D: npm install $SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION --foreground-scripts"
npm install "$SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION" --foreground-scripts
npm run build --foreground-scripts
- name: Build ${{ matrix.settings.dirCohortReplicator }}
env:
SDK_CLIENT_PACKAGE_VERSION: ${{ steps.sdk-client-version.outputs.SDK_CLIENT_PACKAGE_VERSION }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SKIP_NAPI_RS_STEP: true # this will re-use already bundled native lib
shell: bash
run: |-
cd ${{ matrix.settings.dirCohortInitiator }}
pwd
ls -l
rm package-lock.json || true
echo "D: npm install $SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION --foreground-scripts"
npm install "$SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION" --foreground-scripts
npm run build --foreground-scripts
Loading

0 comments on commit c809e90

Please sign in to comment.