diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..00a51aff5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# +# https://help.github.com/articles/dealing-with-line-endings/ +# +# These are explicitly windows files and should use crlf +*.bat text eol=crlf + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index efef46bd0..359fa543c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,162 +1,104 @@ name: KBase Workspace tests on: - pull_request: - types: - - opened - - reopened - - synchronize - - ready_for_review - push: - # run workflow when merging to main or develop - branches: - - main - - master - - develop + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + # run workflow when merging to main or develop + branches: + - main + - master + - develop # TODO find out what minio ver we're using in CI / appdev / prod and use that # TODO split tests into mongo related & non mongo related. Run the former once. jobs: - - workspace_container_tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - - name: Install dependencies and set up test config - shell: bash - run: | - # set up python dependencies - pip install pytest requests - - - name: Run tests - shell: bash - run: | - sh scripts/run_tests.sh - - workspace_deluxe_tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - java: '11' - mongo: 'mongodb-linux-x86_64-3.6.23' - minio: '2019-05-23T00-29-34Z' - wired_tiger: 'true' - ant_test: 'test_quick_coverage' - # the current production setup - - java: '11' - mongo: 'mongodb-linux-x86_64-3.6.13' - minio: '2019-05-23T00-29-34Z' - wired_tiger: 'false' - ant_test: 'test_quick_coverage' - steps: - - uses: actions/checkout@v3 - - - name: Set up java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{matrix.java}} - - - name: Set up Python 3.7 # sample service is on 3.7, handle service is on sdkbase2.latest - uses: actions/setup-python@v3 - with: - python-version: "3.7" - - - name: Install dependencies and set up test config - shell: bash - run: | - export HOMEDIR=`pwd` - - # set up python dependencies - cd python_dependencies - pip install pipenv - pipenv sync --system - cd .. - - # move to parent dir of homedir to install binaries etc - cd .. - - # set up jars - git clone https://github.com/kbase/jars - export JARSDIR=$(pwd)/jars/lib/jars/ - - # set up arango - export ARANGODB_VER=3.9.1 - export ARANGODB_V=39 - curl -O https://download.arangodb.com/arangodb$ARANGODB_V/Community/Linux/arangodb3-linux-$ARANGODB_VER.tar.gz - tar -xf arangodb3-linux-$ARANGODB_VER.tar.gz - export ARANGO_EXE=$(pwd)/arangodb3-linux-$ARANGODB_VER/bin/arangod - export ARANGO_JS=$(pwd)/arangodb3-linux-$ARANGODB_VER/usr/share/arangodb3/js/ - - # set up handle service - export HS_COMMIT=08e18379817e16db920501b66ba62b66598f506c - export LOG_COMMIT=b549c557e3c519e0a55eadf7863a93db25cd6806 - git clone https://github.com/kbase/handle_service2.git - cd handle_service2/ - git checkout $HS_COMMIT - sudo chmod -R 777 . - cd lib/ - mkdir biokbase - cd biokbase/ - wget https://raw.githubusercontent.com/kbase/sdkbase2/$LOG_COMMIT/log.py - cd .. - export HSDIR=`pwd` - cd ../.. - - # set up sample service - export SAMPLE_COMMIT=6813fb148e95db2b11db6eea04f4d1d45cbb7119 - git clone https://github.com/kbase/sample_service.git - cd sample_service - git checkout $SAMPLE_COMMIT - cd lib - export SAMPLE_DIR=`pwd` - cd ../.. - - # set up blobstore - wget -q -O blobstore https://github.com/kbase/blobstore/releases/download/v0.1.2/blobstore_linux_amd64 - chmod a+x blobstore - export BLOBEXE=$(pwd)/blobstore - - # set up mongo - wget -q http://fastdl.mongodb.org/linux/${{matrix.mongo}}.tgz - tar xfz ${{matrix.mongo}}.tgz - export MONGOD=`pwd`/${{matrix.mongo}}/bin/mongod - - # set up minio - wget -q https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.${{matrix.minio}} -O minio - chmod a+x minio - export MINIOD=`pwd`/minio - - # set up test config - cd $HOMEDIR - cp -n test.cfg.example test.cfg - sed -i "s#^test.temp.dir =.*#test.temp.dir=temp_test_dir#" test.cfg - sed -i "s#^test.mongo.exe.*#test.mongo.exe=$MONGOD#" test.cfg - sed -i "s#^test.minio.exe.*#test.minio.exe=$MINIOD#" test.cfg - sed -i "s#^test.mongo.useWiredTiger.*#test.mongo.useWiredTiger=${{matrix.wired_tiger}}#" test.cfg - sed -i "s#^test.jars.dir.*#test.jars.dir=$JARSDIR#" test.cfg - sed -i "s#^test.blobstore.exe.*#test.blobstore.exe=$BLOBEXE#" test.cfg - sed -i "s#^test.handleservice.dir.*#test.handleservice.dir=$HSDIR#" test.cfg - sed -i "s#^test.sampleservice.dir.*#test.sampleservice.dir=$SAMPLE_DIR#" test.cfg - sed -i "s#^test.arango.exe.*#test.arango.exe=$ARANGO_EXE#" test.cfg - sed -i "s#^test.arango.js.*#test.arango.js=$ARANGO_JS#" test.cfg - cat test.cfg - - - name: Run tests - shell: bash - run: | - ant javadoc - ant ${{matrix.ant_test}} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - fail_ci_if_error: true + workspace_container_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Install dependencies and set up test config + shell: bash + run: | + # set up python dependencies + pip install pytest requests sphinx + + - name: Run tests + shell: bash + run: | + sh scripts/run_tests.sh + + workspace_deluxe_tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04] #, ubuntu-20.04, ubuntu-18.04] + mongo: ["3.6.13", "3.6.23"] + test_cmd: ["LongTests", "NoLongTests", "test_quick_coverage"] + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up java + uses: actions/setup-java@v3 + with: + distribution: "temurin" + java-version: 11 + + - name: Set up Python 3.7 # sample service is on 3.7, handle service is on sdkbase2.latest + uses: actions/setup-python@v3 + with: + python-version: "3.7" + + - name: set up environment + uses: ./.github/workflows/test_set_up.yml + with: + cache-token: ${{ secrets.GITHUB_TOKEN }} + os: ${{ matrix.os }} + mongo: ${{ matrix.mongo }} + + - name: restore dependencies from cache + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }} + key: deps-${{ matrix.os }}-${{ matrix.mongo }}-${{ hashFiles('.github/workflows/test_set_up.yml') }} + + - name: build the war file (requires compileJava, javadoc) + run: ./gradlew war --info + + - name: run the ${{matrix.test_cmd}} tests and generate coverage + run: | + mkdir test-reports + if [ "${{matrix.test_cmd}}" = "test_quick_coverage" ]; then + ant "${{matrix.test_cmd}}" + else + ./gradlew "coverage${{matrix.test_cmd}}" --info + fi + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + if: matrix.test_cmd == 'test_quick_coverage' + with: + fail_ci_if_error: true + + - name: upload jacoco report + uses: actions/upload-artifact@v3 + if: matrix.test_cmd != 'test_quick_coverage' + with: + name: coverage${{matrix.test_cmd}}-${{matrix.mongo}}.exec + path: build/ + if-no-files-found: warn diff --git a/.github/workflows/test_set_up.yml b/.github/workflows/test_set_up.yml new file mode 100644 index 000000000..d16e7c35d --- /dev/null +++ b/.github/workflows/test_set_up.yml @@ -0,0 +1,172 @@ +name: Setup Dependencies + +on: + workflow_call: + inputs: + os: + required: true + type: string + mongo: + required: true + type: string + outputs: + cache-key: + description: "Cache key" + value: ${{ jobs.setup.outputs.cache-key }} + secrets: + cache-token: + required: true + +jobs: + setup: + runs-on: ${{ inputs.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Compute cache key + id: compute-key + run: echo "cache-key=deps-${{ inputs.os }}-${{ inputs.mongo }}-${{ hashFiles('.github/workflows/test_set_up.yml') }}" >> $GITHUB_OUTPUT + + - name: Cache dependencies + id: cache-deps + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }} + key: ${{ steps.compute-key.outputs.cache-key }} + + - name: Install dependencies, set up env vars and test cfg + shell: bash + if: steps.cache-deps.outputs.cache-hit != 'true' + run: | + set -x + export HOMEDIR=$(pwd) + echo "HOMEDIR=$(pwd)" >> env-vars.txt + cd .. + export BASEDIR=$(pwd) + echo "BASEDIR=$(pwd)" >> env-vars.txt + export MINIO_VER='2019-05-23T00-29-34Z' + echo "MINIO_VER='2019-05-23T00-29-34Z'" >> env-vars.txt + export MONGO_VER='mongodb-linux-x86_64-${{inputs.mongo}}' + echo "MONGO_VER='mongodb-linux-x86_64-${{inputs.mongo}}'" >> env-vars.txt + # 3.6.13 has wired_tiger set to 'false', 3.6.23 has wired_tiger 'true' + if [ "${{inputs.mongo}}" = "3.6.13" ]; then + export WIRED_TIGER="false" + echo "WIRED_TIGER="false"" >> env-vars.txt + else + export WIRED_TIGER="true" + echo "WIRED_TIGER="true"" >> env-vars.txt + fi + export ARANGODB_VER=3.9.1 + echo "ARANGODB_VER=3.9.1" >> env-vars.txt + export ARANGODB_V=39 + echo "ARANGODB_V=39" >> env-vars.txt + export HS_COMMIT=08e18379817e16db920501b66ba62b66598f506c + echo "HS_COMMIT=08e18379817e16db920501b66ba62b66598f506c" >> env-vars.txt + export LOG_COMMIT=b549c557e3c519e0a55eadf7863a93db25cd6806 + echo "LOG_COMMIT=b549c557e3c519e0a55eadf7863a93db25cd6806" >> env-vars.txt + export SAMPLE_COMMIT=6813fb148e95db2b11db6eea04f4d1d45cbb7119 + echo "SAMPLE_COMMIT=6813fb148e95db2b11db6eea04f4d1d45cbb7119" >> env-vars.txt + + # set up python dependencies + cd $HOMEDIR/python_dependencies + pip install pipenv + pipenv sync --system + # install sphinx for generating documentation + pip install sphinx + + # move to parent dir of homedir to install binaries etc + cd $BASEDIR + # if [ "${{ inputs.test_cmd }}" = "test_quick_coverage" ]; then + # set up jars + git clone https://github.com/kbase/jars + + export JARSDIR=$BASEDIR/jars/lib/jars/ + echo "JARSDIR=$BASEDIR/jars/lib/jars/" >> env-vars.txt + # else + # export JARSDIR=$BASEDIR/build/download/ + # echo "JARSDIR=$HOMEDIR/build/download/" >> env-vars.txt + # fi + + # set up arango + curl -O https://download.arangodb.com/arangodb$ARANGODB_V/Community/Linux/arangodb3-linux-$ARANGODB_VER.tar.gz + tar -xf arangodb3-linux-$ARANGODB_VER.tar.gz + + export ARANGO_EXE=$BASEDIR/arangodb3-linux-$ARANGODB_VER/bin/arangod + echo "ARANGO_EXE=$BASEDIR/arangodb3-linux-$ARANGODB_VER/bin/arangod" >> env-vars.txt + export ARANGO_JS=$BASEDIR/arangodb3-linux-$ARANGODB_VER/usr/share/arangodb3/js/ + echo "ARANGO_JS=$BASEDIR/arangodb3-linux-$ARANGODB_VER/usr/share/arangodb3/js/" >> env-vars.txt + + # set up handle service + git clone https://github.com/kbase/handle_service2.git + cd handle_service2/ + # $BASEDIR/handle_service2 + git checkout $HS_COMMIT + sudo chmod -R 777 . + mkdir lib/biokbase + cd lib/biokbase/ + # $BASEDIR/handle_service2/lib/biokbase + wget https://raw.githubusercontent.com/kbase/sdkbase2/$LOG_COMMIT/log.py + + export HSDIR=$BASEDIR/handle_service2/lib + echo "HSDIR=$BASEDIR/handle_service2/lib" >> env-vars.txt + + # set up sample service + cd $BASEDIR + git clone https://github.com/kbase/sample_service.git + cd sample_service + # $BASEDIR/sample_service + git checkout $SAMPLE_COMMIT + + export SAMPLE_DIR=$BASEDIR/sample_service/lib + echo "SAMPLE_DIR=$BASEDIR/sample_service/lib" >> env-vars.txt + + # set up blobstore + cd $BASEDIR + wget -q -O blobstore https://github.com/kbase/blobstore/releases/download/v0.1.2/blobstore_linux_amd64 || { echo "Failed to wget the blobstore"; exit 1; } + chmod a+x blobstore + + export BLOBEXE=$BASEDIR/blobstore + echo "BLOBEXE=$BASEDIR/blobstore" >> env-vars.txt + + # set up mongo + wget -q http://fastdl.mongodb.org/linux/$MONGO_VER.tgz + tar xfz $MONGO_VER.tgz + + export MONGOD=$BASEDIR/$MONGO_VER/bin/mongod + echo "MONGOD=$BASEDIR/$MONGO_VER/bin/mongod" >> env-vars.txt + + # set up minio + # wget -q https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.$MINIO_VER -O minio + cp $HOMEDIR/ws_deps/minio.RELEASE.2019-05-23T00-29-34Z minio + chmod a+x minio + + export MINIOD=$BASEDIR/minio + echo "MINIOD=$BASEDIR/minio" >> env-vars.txt + + # write the test config + set -a + source env-vars.txt + set +a + env + + # set up test config + cd $HOMEDIR + cp -n test.cfg.example test.cfg + sed -i "s#^test.temp.dir =.*#test.temp.dir=temp_test_dir#" test.cfg + sed -i "s#^test.mongo.exe.*#test.mongo.exe=$MONGOD#" test.cfg + sed -i "s#^test.minio.exe.*#test.minio.exe=$MINIOD#" test.cfg + sed -i "s#^test.mongo.useWiredTiger.*#test.mongo.useWiredTiger=$WIRED_TIGER#" test.cfg + sed -i "s#^test.jars.dir.*#test.jars.dir=$JARSDIR#" test.cfg + sed -i "s#^test.blobstore.exe.*#test.blobstore.exe=$BLOBEXE#" test.cfg + sed -i "s#^test.handleservice.dir.*#test.handleservice.dir=$HSDIR#" test.cfg + sed -i "s#^test.sampleservice.dir.*#test.sampleservice.dir=$SAMPLE_DIR#" test.cfg + sed -i "s#^test.arango.exe.*#test.arango.exe=$ARANGO_EXE#" test.cfg + sed -i "s#^test.arango.js.*#test.arango.js=$ARANGO_JS#" test.cfg + cat test.cfg + + - name: Set cache-key output + id: set-cache-key-output + run: echo "cache-key=${{ steps.compute-key.outputs.cache-key }}" >> "$GITHUB_OUTPUT" diff --git a/Dockerfile b/Dockerfile index 3c6a15f24..da7d3e6ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,15 @@ -FROM eclipse-temurin:11-jdk as build +FROM eclipse-temurin:11-jdk AS build -COPY . /tmp/workspace_deluxe -WORKDIR /tmp RUN apt-get update -y && \ - apt-get install -y ant git ca-certificates python3-sphinx && \ - git clone https://github.com/kbase/jars && \ - cd workspace_deluxe && \ - make docker_deps + apt-get install -y git ca-certificates python3-sphinx + +COPY . /tmp/workspace_deluxe +WORKDIR /tmp/workspace_deluxe +RUN make docs && \ + ./gradlew war && \ + mkdir -p deployment/services/workspace/ && \ + cp dist/WorkspaceService.war deployment/services/workspace/ + # updated/slimmed down version of what's in kbase/kb_jre FROM ubuntu:18.04 diff --git a/Makefile b/Makefile index 17db74f06..89b8a0171 100644 --- a/Makefile +++ b/Makefile @@ -1,51 +1,26 @@ #port is now set in deploy.cfg SERVICE = workspace -SERVICE_CAPS = Workspace CLIENT_JAR = WorkspaceClient.jar WAR = WorkspaceService.war URL = https://kbase.us/services/ws/ - -#End of user defined variables -TARGET ?= /kb/deployment - -GITCOMMIT := $(shell git rev-parse --short HEAD) -#TODO use --points-at when git 1.7.10 available -TAGS := $(shell git tag --contains $(GITCOMMIT)) - -DEPLOY_RUNTIME ?= /kb/runtime -JAVA_HOME ?= $(DEPLOY_RUNTIME)/java -SERVICE_DIR ?= $(TARGET)/services/$(SERVICE) -GLASSFISH_HOME ?= $(DEPLOY_RUNTIME)/glassfish3 -SERVICE_USER ?= kbase - -ASADMIN = $(GLASSFISH_HOME)/glassfish/bin/asadmin - -ANT = ant +GRADLE = ./gradlew # make sure our make test works .PHONY : test -default: build-libs build-docs +default: compile -build-libs: - @#TODO at some point make dependent on compile - checked in for now. - $(ANT) compile - -build-docs: +sdk_docs: -rm -r docs - $(ANT) javadoc - pod2html --infile=lib/Bio/KBase/$(SERVICE)/Client.pm --outfile=docs/$(SERVICE)_perl.html - rm -f pod2htm?.tmp sphinx-build docsource/ docs cp $(SERVICE).spec docs/. cp docshtml/* docs/. + pod2html --infile=lib/Bio/KBase/$(SERVICE)/Client.pm --outfile=docs/$(SERVICE)_perl.html + rm -f pod2htm?.tmp -docker_deps: build-libs build-docs - $(ANT) buildwar - # cp server_scripts/glassfish_administer_service.py deployment/bin - # chmod 755 deployment/bin/glassfish_administer_service.py - mkdir -p deployment/services/workspace/ - cp dist/$(WAR) deployment/services/workspace/ +docs: sdk_docs + $(GRADLE) javadoc + cp -r build/docs/javadoc docs/ compile: compile-typespec compile-typespec-java compile-html @@ -69,63 +44,13 @@ compile-typespec: test: test-service test-service: - $(ANT) test + $(GRADLE) test test-quick: - $(ANT) test_quick - -deploy: deploy-client deploy-service - -deploy-client: deploy-client-libs deploy-docs - -deploy-client-libs: - mkdir -p $(TARGET)/lib/ - cp dist/client/$(CLIENT_JAR) $(TARGET)/lib/ - cp -rv lib/* $(TARGET)/lib/ - echo $(GITCOMMIT) > $(TARGET)/lib/$(SERVICE).clientdist - echo $(TAGS) >> $(TARGET)/lib/$(SERVICE).clientdist - -deploy-docs: - mkdir -p $(SERVICE_DIR)/webroot - cp -r docs/* $(SERVICE_DIR)/webroot/. - -deploy-service: deploy-service-libs deploy-service-scripts - -deploy-service-libs: - $(ANT) buildwar - mkdir -p $(SERVICE_DIR) - cp dist/$(WAR) $(SERVICE_DIR) - echo $(GITCOMMIT) > $(SERVICE_DIR)/$(SERVICE).serverdist - echo $(TAGS) >> $(SERVICE_DIR)/$(SERVICE).serverdist - -deploy-service-scripts: - cp server_scripts/glassfish_administer_service.py $(SERVICE_DIR) - server_scripts/build_server_control_scripts.py $(SERVICE_DIR) $(WAR)\ - $(TARGET) $(JAVA_HOME) deploy.cfg $(ASADMIN) $(SERVICE_CAPS) - -deploy-upstart: - echo "# $(SERVICE) service" > /etc/init/$(SERVICE).conf - echo "# NOTE: stop $(SERVICE) does not work" >> /etc/init/$(SERVICE).conf - echo "# Use the standard stop_service script as the $(SERVICE_USER) user" >> /etc/init/$(SERVICE).conf - echo "#" >> /etc/init/$(SERVICE).conf - echo "# Make sure to set up the $(SERVICE_USER) user account" >> /etc/init/$(SERVICE).conf - echo "# shell> groupadd kbase" >> /etc/init/$(SERVICE).conf - echo "# shell> useradd -r -g $(SERVICE_USER) $(SERVICE_USER)" >> /etc/init/$(SERVICE).conf - echo "#" >> /etc/init/$(SERVICE).conf - echo "start on runlevel [23] and started shock" >> /etc/init/$(SERVICE).conf - echo "stop on runlevel [!23]" >> /etc/init/$(SERVICE).conf - echo "pre-start exec chown -R $(SERVICE_USER) $(TARGET)/services/$(SERVICE)" >> /etc/init/$(SERVICE).conf - echo "exec su kbase -c '$(TARGET)/services/$(SERVICE)/start_service'" >> /etc/init/$(SERVICE).conf - -undeploy: - -rm -rf $(SERVICE_DIR) - -rm -rfv $(TARGET)/lib/Bio/KBase/$(SERVICE) - -rm -rfv $(TARGET)/lib/biokbase/$(SERVICE) - -rm -rfv $(TARGET)/lib/javascript/$(SERVICE) - -rm -rfv $(TARGET)/lib/$(CLIENT_JAR) + $(GRADLE) testNoLongTests clean: - $(ANT) clean + $(GRADLE) clean -rm -rf docs -rm -rf bin -rm -rf deployment/services/workspace/* diff --git a/build.gradle b/build.gradle new file mode 100644 index 000000000..6c31d10e4 --- /dev/null +++ b/build.gradle @@ -0,0 +1,340 @@ +/* + * Gradle file for the KBase workspace_deluxe. + */ + +plugins { + id "java" + id "war" + id "jacoco" +} + +group = "workspace service" +version = "0.14.1" + +ext { + clientJarFile = "WorkspaceClient.jar" + serviceJarFile = "WorkspaceService.jar" + warFile = "WorkspaceService.war" + warXml = "war/web.xml" + // directories + src = "src" + dist = "dist" + docs = "docs" + downloadDir = "$buildDir/download" + tempunpack = "$buildDir/tmp/unpackedjars" + // other config + kbaseCommonJar = "kbase-common-0.1.1.jar" + testConfig = "test.cfg" +} + +repositories { + mavenCentral() + maven { + name = "Clojars" + url = "https://repo.clojars.org/" + } +} + +configurations { + implementation + testImplementation + runtimeDeps { + extendsFrom implementation + } + allDeps { + extendsFrom implementation, testImplementation + } +} + +dependencies { + // downloaded jars + implementation fileTree(dir: "$downloadDir", include: ["*.jar"]) + + implementation "ch.qos.logback:logback-classic:1.1.2" + implementation "com.fasterxml.jackson.core:jackson-annotations:2.9.9" + implementation "com.fasterxml.jackson.core:jackson-core:2.9.9" + implementation "com.fasterxml.jackson.core:jackson-databind:2.9.9" + implementation "com.github.ben-manes.caffeine:caffeine:2.9.3" + implementation "com.google.guava:guava:14.0.1" + implementation "commons-codec:commons-codec:1.8" + implementation "commons-io:commons-io:2.4" + implementation "info.picocli:picocli:4.6.1" + implementation "javax.annotation:javax.annotation-api:1.3.2" + implementation "javax.servlet:servlet-api:2.5" + implementation "joda-time:joda-time:2.2" + implementation "net.java.dev.jna:jna:3.4.0" + implementation "org.apache.commons:commons-lang3:3.1" + implementation "org.apache.httpcomponents:httpclient:4.5.9" + implementation "org.apache.httpcomponents:httpmime:4.5.8" + implementation "org.apache.kafka:kafka-clients:2.1.0" + implementation "org.eclipse.jetty.aggregate:jetty-all:7.0.0.v20091005" + implementation "org.ini4j:ini4j:0.5.2" + implementation "org.mongodb:mongo-java-driver:3.12.10" + implementation "org.slf4j:slf4j-api:1.7.30" + implementation "org.syslog4j:syslog4j:0.9.46" + implementation("software.amazon.awssdk:s3:2.17.214") { + exclude module: "apache-client" + exclude module: "netty-nio-client" + } + implementation "software.amazon.awssdk:url-connection-client:2.17.214" + + // Test deps + testImplementation "com.arangodb:arangodb-java-driver:6.7.2" + testImplementation "com.github.zafarkhaja:java-semver:0.9.0" + testImplementation "junit:junit:4.12" + testImplementation "nl.jqno.equalsverifier:equalsverifier:3.1.10" + testImplementation "org.hamcrest:hamcrest-core:1.3" + testImplementation "org.mockito:mockito-core:3.0.0" +} + +// download the KBase jars if they aren't already present +task downloadJars { + doLast { + def dir = file(downloadDir) + if (!dir.exists()) { + dir.mkdirs() + } + def url = "https://github.com/kbase/jars/raw/master/lib/jars/kbase" + def jars = [ + "auth/kbase-auth-0.4.4.jar", + "auth2/kbase-auth2test-0.2.4.jar", + "common/$kbaseCommonJar", + "handle/AbstractHandleClient-1.0.0.jar", + "kidl/kbase-kidl-parser-1409261812-7863aef.jar", + "sample/SampleServiceClient-0.1.1.jar", + "shock/shock-client-0.1.0.jar" + ] + jars.each { jar -> + def jarName = jar.split("/")[-1] + def file = file("$downloadDir/$jarName") + if (!file.exists()) { + ant.get(src: "$url/$jar", dest: "$downloadDir/$jarName") + } + } + } +} + +// Custom java project layout +sourceSets { + main { + java { + srcDir src + exclude "**/test/**" + } + } + test { + java { + srcDir src + } + resources { + srcDir src + include "**/*.properties" + include "**/*.cfg" + include "**/*.spec" + include "**/*.instance.*" + include "**/instance.*" + include "**/*.instance" + include "**/*.html" + include "**/*.css" + include "**/*.gif" + include "**/*.js" + include "**/*.png" + include "**/*.txt" + include "**/*.weirdsuffix" + } + } +} + +tasks.withType(JavaCompile) { + dependsOn(downloadJars) + classpath = configurations.allDeps + options.compilerArgs = ["-source", "11", "-target", "11", "-g"] +} + +task clientJar(type: Jar, dependsOn: compileJava) { + from sourceSets.main.output + include "us/kbase/workspace/*.class" + exclude "us/kbase/workspace/WorkspaceServer.class" + include "us/kbase/common/service/*.class" + exclude "us/kbase/common/service/JsonServer*" + archiveFileName = clientJarFile + destinationDirectory = file("$dist/client") +} + +task serviceJar(type: Jar, dependsOn: compileJava) { + from sourceSets.main.output + archiveFileName = serviceJarFile + destinationDirectory = file("$dist") +} + +task buildJars { + dependsOn serviceJar, clientJar +} + +task unzipJar(type: Copy, dependsOn: [downloadJars]) { + from zipTree("$downloadDir/$kbaseCommonJar") + into "$tempunpack" +} + +// by default, the files go into build/docs/javadoc +tasks.javadoc { + dependsOn unzipJar + classpath = configurations.allDeps + failOnError = false + options { + author = false + noDeprecated = false + noDeprecatedList = false + noIndex = false + noNavBar = false + noTree = false + splitIndex = true + use = true + version = true + links = [ + "http://download.oracle.com/javase/11/docs/api/", + "https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-core/2.9.9/", + "https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-databind/2.9.9/" + ] + } + source = fileTree(dir: "$src/us/kbase", + include: [ + "workspace/*.java", + "common/service/*.java" + ], + exclude: [ + "workspace/WorkspaceServer.java", + "common/service/JsonServer*", + "common/service/JacksonTupleModule.java", + "common/service/JsonClientCaller.java", + "common/service/JsonTreeTraversingParser.java", + "common/service/KBaseJsonParser.java" + ]) + source += fileTree(dir: "$tempunpack/us/kbase/common/service", include: [ + "UObject.java", + "JsonTokenStream.java", + "*Exception.java" + ]) + + doLast { + delete "$tempunpack" + } +} + +task buildDocs(type: Exec) { + description "Run sphinx-build and pod2html" + group "Documentation" + // Set the command to run sphinx-build and pod2html + commandLine "make", "docs" +} + +tasks.war { + dependsOn javadoc + dependsOn buildDocs + description = "Build the WAR file. Assumes compilation has been run." + archiveFileName = warFile + destinationDirectory = file("$dist") + + // location of the web.xml file + webXml = file("$warXml") + + duplicatesStrategy "exclude" + + // add the dependencies to lib + into("WEB-INF/lib") { + from sourceSets.main.compileClasspath + } + // copy the compiled classes over + into("WEB-INF/classes") { + from sourceSets.main.output.classesDirs + } + + // copy over the non-java documentation + into("WEB-INF/classes/server_docs") { + from docs + } + + // copy over the javadoc + into("WEB-INF/classes/server_docs/javadoc") { + from javadoc.destinationDir + } + +} + +task expandWar(type: Copy) { + from zipTree("$dist/$warFile") + into "war_contents" +} + +// test reports are saved to build/reports/tests/ +tasks.withType(Test) { + dependsOn compileTestJava + useJUnit() + systemProperty "test.cfg", testConfig + maxHeapSize = "3G" + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "short" + showStandardStreams = true + showExceptions = true + showCauses = true + } + filter { + // gradle thinks that classes annotated with + // @RunWith are tests + excludeTestsMatching "*Tester" + } +} + +task testNoLongTests(type: Test) { + description = "run all tests except the slow tests" + systemProperty "test.cfg", testConfig + forkEvery 1 + filter { + excludeTestsMatching "*LongTest" + } +} + +task testLongTestsOnly(type: Test) { + description = "run the slow tests" + systemProperty "test.cfg", testConfig + forkEvery 1 + filter { + includeTestsMatching "*LongTest" + } +} + + +// output from coverage tasks is saved to build/reports/jacoco/ +tasks.withType(JacocoReport) { + reports { + html.required = true + xml.required = true + csv.required = true + } + sourceSets sourceSets.main +} + +task coverageNoLongTests(type: JacocoReport, dependsOn: testNoLongTests) { + group = "Reporting" + description = "Generate code coverage reports for all tests except the slow tests" + executionData(testNoLongTests) +} + +task coverageLongTests(type: JacocoReport, dependsOn: testLongTestsOnly) { + group = "Reporting" + description = "Generate code coverage reports for the slow tests" + executionData(testLongTestsOnly) +} + +task createScript { + dependsOn compileJava + description = "create cli script" + + def runtimeClasspath = configurations.runtimeDeps.files.join(":") + file("update_workspace_database_schema").write("#!/bin/sh\n" + + "java -cp ${project.rootDir.absolutePath}/$dist/$serviceJarFile:$runtimeClasspath us.kbase.workspace.kbase.SchemaUpdaterCLI \$@\n") + file("update_workspace_database_schema").setExecutable(true) + // always run the script + outputs.upToDateWhen { false } +} diff --git a/build.xml b/build.xml index 37b8e5219..ea8a6e8bf 100644 --- a/build.xml +++ b/build.xml @@ -75,7 +75,7 @@ - + @@ -242,7 +242,8 @@ + source="11" splitindex="true" use="true" Windowtitle="Workspace 0.14.1 API" + version="true"> diff --git a/deployment/conf/.templates/start_workspace.sh.templ b/deployment/conf/.templates/start_workspace.sh.templ deleted file mode 100644 index 83ab6f4e6..000000000 --- a/deployment/conf/.templates/start_workspace.sh.templ +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env sh -cd /kb/deployment - -# Values for port, threads, min/max memory are populated into source template from environment vars -# the defaults are based on -bin/glassfish_administer_service.py --verbose --admin $GLASSFISH/bin/asadmin --domain Workspace \ - --domain-dir /kb/deployment/services/workspace/glassfish_domain \ - --war /kb/deployment/services/workspace/WorkspaceService.war \ - --port {{ default .Env.service_port "7058" }} --instanceport {{ default .Env.instance_port "32768" }} \ - --threads {{ default .Env.server_threads "20" }} \ - --Xms {{ default .Env.min_memory "10000" }} --Xmx {{ default .Env.max_memory "15000" }} \ - --properties KB_DEPLOYMENT_CONFIG=$KB_DEPLOYMENT_CONFIG && \ -tail -n 500 -f /kb/deployment/services/workspace/glassfish_domain/Workspace/logs/server.log diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..41d9927a4 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..59bc51a20 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 000000000..1b6c78733 --- /dev/null +++ b/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/scripts/workspace_container_test.py b/scripts/workspace_container_test.py index 63acea5e7..7a9bb54b1 100644 --- a/scripts/workspace_container_test.py +++ b/scripts/workspace_container_test.py @@ -70,7 +70,26 @@ def create_read_workspace(token: str) -> None: def test_get_docs() -> None: """check that the workspace documentation can be accessed""" - response = requests.get(WS_URL + "/docs/") - assert response.status_code == 200 - assert response.text.find("KBase Workspace Service Manual") != -1 - assert response.text.find("KBase Workspace " + WORKSPACE_VERSION + " documentation") != -1 + url_and_content = { + "/": [ + "KBase Workspace Service Manual", + "KBase Workspace " + WORKSPACE_VERSION + " documentation" + ], + # from rst files + "/search.html": ['

Search

'], + # pod2html output + "/workspace_perl.html": ['$info = $obj->get_workspace_info($wsi)'], + # copied as-is + "/workspace.spec": ["module Workspace {"], + # javadoc output + "/javadoc/help-doc.html": [ + '', + 'API Help (Workspace ' + WORKSPACE_VERSION + ' API)' + ] + } + + for url in url_and_content.keys(): + response = requests.get(WS_URL + "/docs" + url) + assert response.status_code == 200 + for content in url_and_content[url]: + assert response.text.find(content) != -1 diff --git a/server_scripts/build_server_control_scripts.py b/server_scripts/build_server_control_scripts.py deleted file mode 100755 index 9bff4d34e..000000000 --- a/server_scripts/build_server_control_scripts.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -''' -Created on Mar 11, 2014 - -@author: gaprice@lbl.gov -''' -from __future__ import print_function -import sys -from configobj import ConfigObj -import os -import stat - -PORT = 'port' -THREADS = 'server-threads' -MINMEM = 'min-memory' -MAXMEM = 'max-memory' - - -def printerr(*objs): - print(*objs, file=sys.stderr) - sys.exit(1) - - -def make_executable(path): - st = os.stat(path) - os.chmod(path, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - -def getConfig(param, cfg, cfile): - if param not in cfg: - printerr('Missing expected parameter {} in config file {}' - .format(param, cfile)) - return cfg[param] - -if len(sys.argv) < 8: - printerr("Missing arguments to build_server_control_scripts") -if len(sys.argv) == 8: - _, serviceDir, war, target, javaHome, deployCfg, asadmin, serviceDomain =\ - sys.argv - port = None -else: - _, serviceDir, war, target, javaHome, deployCfg, asadmin, serviceDomain,\ - port = sys.argv - -if not os.path.isfile(deployCfg): - printerr('Configuration parameter is not a file: ' + deployCfg) -cfg = ConfigObj(deployCfg) -if serviceDomain not in cfg: - printerr('No {} section in config file {} - '.format( - serviceDomain, deployCfg)) -wscfg = cfg[serviceDomain] - -if port is None: - if PORT not in wscfg: - printerr("Port not provided as argument or in config") - port = wscfg[PORT] - -threads = getConfig(THREADS, wscfg, deployCfg) -minmem = getConfig(MINMEM, wscfg, deployCfg) -maxmem = getConfig(MAXMEM, wscfg, deployCfg) - -with open(os.path.join(serviceDir, 'start_service'), 'w') as ss: - ss.write('export JAVA_HOME={}\n'.format(javaHome)) - ss.write('export PATH=$JAVA_HOME/bin:$PATH\n') - ss.write('export CLASSPATH=\n') - ss.write('if [ -z "$KB_DEPLOYMENT_CONFIG" ]\n') - ss.write('then\n') - ss.write(' export KB_DEPLOYMENT_CONFIG={}/deployment.cfg\n' - .format(target)) - ss.write('fi\n') - ss.write(('{}/glassfish_administer_service.py --admin {} ' + - '--domain {} --domain-dir {}/glassfish_domain ' + - '--war {} --port {} --threads {} --Xms {} --Xmx {} ' + - '--properties KB_DEPLOYMENT_CONFIG=$KB_DEPLOYMENT_CONFIG\n') - .format(serviceDir, asadmin, serviceDomain, serviceDir, - os.path.join(serviceDir, war), port, threads, minmem, maxmem)) - -with open(os.path.join(serviceDir, 'stop_service'), 'w') as ss: - ss.write('export JAVA_HOME={}\n'.format(javaHome)) - ss.write('export PATH=$JAVA_HOME/bin:$PATH\n') - ss.write('export CLASSPATH=\n') - ss.write(('{}/glassfish_administer_service.py --admin {} ' + - '--domain {} --domain-dir {}/glassfish_domain --port {}\n') - .format(serviceDir, asadmin, serviceDomain, serviceDir, port)) - -make_executable(os.path.join(serviceDir, 'start_service')) -make_executable(os.path.join(serviceDir, 'stop_service')) diff --git a/server_scripts/glassfish_administer_service.py b/server_scripts/glassfish_administer_service.py deleted file mode 100755 index cd59e2347..000000000 --- a/server_scripts/glassfish_administer_service.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python -''' -Created on Dec 6, 2013 - -@author: gaprice@lbl.gov -''' -from __future__ import print_function -from argparse import ArgumentParser -import subprocess -import os -import xml.etree.ElementTree as ET -import urllib2 -from subprocess import CalledProcessError -import sys - -_PARALLEL_GC = "-XX:-UseParallelGC" -_PARALLEL_GC_ESC = "-XX\:-UseParallelGC" - - -def _parseArgs(): - parser = ArgumentParser(description='script to administer a Glassfish ' + - ' application.') - parser.add_argument('-w', '--war', - help='path to the application WAR file. If ' + - 'omitted, the service at the port and domain is ' + - 'stopped.') - parser.add_argument('-a', '--admin', required=True, - help='location of the Glassfish asadmin program.') - parser.add_argument('-d', '--domain', required=True, - help='name of the Glassfish domain where the ' + - 'application is or will be installed.') - parser.add_argument('-l', '--domain-dir', - help='directory where the glassfish domain ' + - 'information and logs will be stored. Defaults to ' + - 'glassfish/domains.') - parser.add_argument('-p', '--port', required=True, type=int, - help='the port where the application runs.') - parser.add_argument('-i', '--instanceport', type=int, default=8080, - help='port for glassfish --instanceport option in create-domain') - parser.add_argument('-t', '--threads', type=int, default=20, - help='the number of threads for the application.') - parser.add_argument('-s', '--Xms', type=int, - help='minimum memory for the domain in MB. ' + - 'This will cause a domain restart if changed.') - parser.add_argument('-x', '--Xmx', type=int, - help='maximum memory for the domain in MB. ' + - 'This will cause a domain restart if changed.') - parser.add_argument('-r', '--properties', nargs='*', - help='JVM system properties to add to the server.') - parser.add_argument('-g', '--noparallelgc', action='store_true', - help='turn off the parallel garbage ' + - ' collector and use the standard gc.') - parser.add_argument('-v', '--verbose', action='store_true', - help='Verbose logging of glassfish actions') - return parser.parse_args() - - -class CommandGlassfishDomain(object): - - def __init__(self, asadminpath, domain, domainpath, verbose, instanceport): - self.asadminpath = asadminpath - self.domain = domain - self.path = None - self.verbose = verbose - if (domainpath): - domaindir = os.path.abspath(os.path.expanduser(domainpath)) - if not os.path.isdir(domaindir): - if not os.path.exists(domaindir): - os.mkdir(domaindir) - else: - print('Domain path ' + domainpath + ' must be a directory') - sys.exit(1) - self.path = domaindir - p = (' at ' + self.path) if(self.path) else '' - if self.exists(): - print('Domain ' + self.domain + ' exists' + p + - ', skipping creation') - else: - print('Creating domain ' + self.domain + p) - print(self._run_local_command('create-domain', '--nopassword=true', - '--instanceport=' + str(instanceport), # move instanceport off 8080 - self.domain).rstrip()) - self.adminport = self.get_admin_port() - self.start_domain() - - def get_admin_port(self): - # the fact I have to do this is moronic - if (self.path): - domains = self.path - else: - bindir = os.path.dirname(self.asadminpath) - glassfish = os.path.join(bindir, "..") - domains = os.path.join(glassfish, "domains") - domain = os.path.join(domains, self.domain) - configfile = os.path.join(domain, "config/domain.xml") - xml = ET.parse(configfile) - root = xml.getroot() - config = root.findall("./configs/config[@name='server-config']")[0] - adminlist = config.findall( - "./network-config/network-listeners/network-listener[@protocol=" + - "'admin-listener']")[0] - return adminlist.attrib['port'] - - def start_domain(self): - if self.is_running(): - print ("Domain " + self.domain + " is already running on port " + - self.adminport) - else: - print("Starting domain " + self.domain) - print(self._run_local_command('start-domain', self.domain) - .rstrip()) - self.adminport = self.get_admin_port() - - def restart_domain(self): - if self.is_running(): - print("Restarting " + self.domain + ", please wait") - print(self._run_local_command('restart-domain', self.domain) - .rstrip()) - else: - self.start_domain() - - def exists(self): - return self.domain in self._list_domains() - - def is_running(self): - return self.domain + " running" in self._list_domains() - - def start_service(self, war, port, threads): - portstr = str(port) - threadstr = str(threads) - if 'server-' + portstr in self._run_remote_command('list-virtual-servers'): - print("Virtual server already exists") - else: - print(self._run_remote_command( - 'create-virtual-server', '--hosts', - '${com.sun.aas.hostName}', 'server-' + portstr).rstrip()) - if 'thread-pool-' + portstr in self._run_remote_command('list-threadpools', 'server'): - print("Threadpool already exists") - else: - print(self._run_remote_command('create-threadpool', '--maxthreadpoolsize=' + threadstr, - '--minthreadpoolsize=' + threadstr, - 'thread-pool-' + portstr).rstrip()) - if 'http-listener-' + portstr in self._run_remote_command('list-http-listeners'): - print('Http listener already exists') - else: - print(self._run_remote_command( - 'create-http-listener', '--listeneraddress', '0.0.0.0', - '--listenerport', portstr, - '--default-virtual-server', 'server-' + portstr, - '--securityEnabled=false', '--acceptorthreads=' + threadstr, - 'http-listener-' + portstr).rstrip()) - print(self._run_remote_command( - 'set', 'server.network-config.network-listeners.' + - 'network-listener.http-listener-' + portstr + - '.thread-pool=thread-pool-' + portstr).rstrip()) - print(self._run_remote_command( - 'set', 'server.network-config.protocols.protocol.' + - 'http-listener-' + portstr + '.http.timeout-seconds=1800') - .rstrip()) - if 'app-' + portstr in self._run_remote_command('list-applications'): - print(self._run_remote_command('undeploy', 'app-' + portstr) - .rstrip()) - print(self._run_remote_command( - 'deploy', '--virtualservers', 'server-' + portstr, - '--contextroot', '/', '--name', 'app-' + portstr, war).rstrip()) - try: - urllib2.urlopen('http://localhost:' + portstr) - except urllib2.HTTPError as h: - resp = h.read() - else: - print('Unexpected response from server - the server did not ' + - 'start up successfully. Please check the glassfish logs.') - return False - if '32603' in resp: - print('The server failed to start up successfully and is ' + - 'running in protected mode. Please check the system and ' + - 'glassfish logs.') - return False - elif '32300' in resp: - print('The server started successfully.') - return True - else: - print('The server failed to start up successfully and is not ' - + 'running. Please check the system and glassfish logs.') - return False - - def stop_service(self, port): - portstr = str(port) - if 'app-' + portstr in self._run_remote_command('list-applications'): - print(self._run_remote_command('undeploy', 'app-' + portstr) - .rstrip()) - if 'http-listener-' + portstr in self._run_remote_command('list-http-listeners'): - print(self._run_remote_command('delete-http-listener', - 'http-listener-' + portstr).rstrip()) - if 'http-listener-' + portstr in self._run_remote_command('list-protocols'): - print(self._run_remote_command('delete-protocol', - 'http-listener-' + portstr).rstrip()) - if 'thread-pool-' + portstr in self._run_remote_command('list-threadpools', 'server'): - print(self._run_remote_command( - 'delete-threadpool', 'thread-pool-' + portstr).rstrip()) - if 'server-' + portstr in self._run_remote_command('list-virtual-servers'): - print(self._run_remote_command( - 'delete-virtual-server', 'server-' + portstr).rstrip()) - - def set_min_max_memory(self, minm, maxm): - # will restart the domain if changes are necessary - xmx = [] - xms = [] - for o in self._run_remote_command('list-jvm-options').split('\n'): - if o.startswith('-Xmx'): - xmx.append(o) - if o.startswith('-Xms'): - xms.append(o) - if (len(xms) > 1 and minm is None): - print('WARNING: multiple Xms parameters set on service: ' + - str(xms)) - if (len(xmx) > 1 and maxm is None): - print('WARNING: multiple Xmx parameters set on service: ' + - str(xmx)) - changed = self._set_memory(None if minm is None else '-Xms' + - str(minm) + 'm', xms) - changed2 = self._set_memory(None if maxm is None else '-Xmx' - + str(maxm) + 'm', xmx) - if changed or changed2: - self.restart_domain() - - def reenable_parallel_gc(self): - if self.parallel_gc_is_disabled(): - self.delete_jvm_option(_PARALLEL_GC_ESC) - self.restart_domain() - - def parallel_gc_is_disabled(self): - for o in self._run_remote_command('list-jvm-options').split('\n'): - if o == _PARALLEL_GC: - return True - return False - - def stop_parallel_gc(self): - if not self.parallel_gc_is_disabled(): - self.create_jvm_option(_PARALLEL_GC_ESC) - self.restart_domain() - - def create_property(self, prop): - print('Creating property ' + prop) - print(self._run_remote_command('create-system-properties', prop) - .rstrip()) - - def create_jvm_option(self, prop): - print('Creating jvm property ' + prop) - print(self._run_remote_command('create-jvm-options', prop) - .rstrip()) - - def delete_jvm_option(self, prop): - print('Removing jvm property ' + prop) - print(self._run_remote_command('delete-jvm-options', prop).rstrip()) - - def _set_memory(self, memstr, memlist): - if (memstr is not None and [memstr] != memlist): - print("Removing options " + str(memlist)) - for o in memlist: - self._remove_option(o) - print("Setting option " + memstr) - self._set_option(memstr) - return True - else: - return False - - def _set_option(self, opt): - self._run_remote_command('create-jvm-options', opt) - - def _remove_option(self, opt): - self._run_remote_command('delete-jvm-options', opt) - - def _list_domains(self): - return self._run_local_command('list-domains') - - def _run_local_command(self, subcmd, *args): - if self.verbose: - print("Running local command:", subcmd, list(args), file=sys.stderr) - cmd = [self.asadminpath, subcmd] - if (self.path): - cmd.extend(['--domaindir', self.path]) - try: - return subprocess.check_output(cmd + list(args)) - except CalledProcessError as cpe: - print(cpe.output.rstrip()) - sys.exit(1) - - def _run_remote_command(self, *cmd): - if self.verbose: - print("Running remote command:", list(cmd), file=sys.stderr) - try: - return subprocess.check_output([self.asadminpath, '-p', - self.adminport] + list(cmd)) - except CalledProcessError as cpe: - print(cpe.output.rstrip()) - sys.exit(1) - - -if __name__ == '__main__': - args = _parseArgs() - gf = CommandGlassfishDomain(args.admin, args.domain, args.domain_dir, args.verbose, args.instanceport) - if args.war is None: - gf.stop_service(args.port) - else: - if (args.noparallelgc): - gf.stop_parallel_gc() - else: - gf.reenable_parallel_gc() - gf.set_min_max_memory(args.Xms, args.Xmx) - for p in args.properties: - gf.create_property(p) - success = gf.start_service(args.war, args.port, args.threads) - if not success: - sys.exit(1) diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 000000000..140bf4140 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,10 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user manual at https://docs.gradle.org/7.4.2/userguide/multi_project_builds.html + */ + +rootProject.name = 'Workspace' diff --git a/src/us/kbase/typedobj/db/KidlUtil.java b/src/us/kbase/typedobj/db/KidlUtil.java index 02bfc00a6..d90b675c2 100644 --- a/src/us/kbase/typedobj/db/KidlUtil.java +++ b/src/us/kbase/typedobj/db/KidlUtil.java @@ -1,8 +1,5 @@ package us.kbase.typedobj.db; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; @@ -37,30 +34,6 @@ public static boolean compareJson(Map parse1, Map parse2, String hea return ok; } - public static boolean compareJsonSchemas(Map> schemas1, - Map> schemas2, String header) throws IOException, - JsonParseException, JsonMappingException, JsonGenerationException, - Exception { - boolean ok = true; - assertThat(schemas1.keySet(), is(schemas2.keySet())); - for (String moduleName : schemas1.keySet()) { - assertThat(schemas1.get(moduleName).keySet(), is(schemas2.get(moduleName).keySet())); - for (Map.Entry entry : schemas1.get(moduleName).entrySet()) { - String schema1 = rewriteJson(entry.getValue()); - String schema2 = rewriteJson(schemas2.get(moduleName).get(entry.getKey())); - if (!schema1.equals(schema2)) { - ok = false; - System.out.println(header + " (" + moduleName + "." + entry.getKey() + "):"); - System.out.println("--------------------------------------------------------"); - showDiff(schema1, schema2); - System.out.println(); - System.out.println("*"); - } - } - } - return ok; - } - private static void showDiff(String origText, String newText) throws Exception { List origLn = getLines(origText); List newLn = getLines(newText); diff --git a/src/us/kbase/workspace/WorkspaceServer.java b/src/us/kbase/workspace/WorkspaceServer.java index 75cf6493f..57f051de6 100644 --- a/src/us/kbase/workspace/WorkspaceServer.java +++ b/src/us/kbase/workspace/WorkspaceServer.java @@ -51,8 +51,6 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; -//import org.apache.commons.lang3.builder.ToStringBuilder; - import us.kbase.typedobj.core.TempFilesManager; import us.kbase.typedobj.core.TypeDefId; import us.kbase.workspace.database.DependencyStatus; diff --git a/src/us/kbase/workspace/kbase/admin/AdministrationCommandSetInstaller.java b/src/us/kbase/workspace/kbase/admin/AdministrationCommandSetInstaller.java index b8b368e06..f152336c3 100644 --- a/src/us/kbase/workspace/kbase/admin/AdministrationCommandSetInstaller.java +++ b/src/us/kbase/workspace/kbase/admin/AdministrationCommandSetInstaller.java @@ -11,7 +11,7 @@ import java.util.Optional; import java.util.stream.Collectors; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,6 +84,7 @@ public class AdministrationCommandSetInstaller { private static final String CREATE_WORKSPACE = "createWorkspace"; private static final String DELETE_WS = "deleteWorkspace"; private static final String UNDELETE_WS = "undeleteWorkspace"; + private static final String UNDELETE_WS2 = "undeleteWorkspace"; private final static ObjectMapper MAPPER = new ObjectMapper() .registerModule(new JacksonTupleModule()); diff --git a/ws_deps/minio.RELEASE.2019-05-23T00-29-34Z b/ws_deps/minio.RELEASE.2019-05-23T00-29-34Z new file mode 100755 index 000000000..6b965260f Binary files /dev/null and b/ws_deps/minio.RELEASE.2019-05-23T00-29-34Z differ