diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index efef46bd..3e68fb4b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,18 +44,9 @@ jobs: strategy: fail-fast: false matrix: - include: - - java: '11' - mongo: 'mongodb-linux-x86_64-3.6.23' - minio: '2019-05-23T00-29-34Z' - wired_tiger: 'true' - ant_test: 'test_quick_coverage' - # the current production setup - - java: '11' - mongo: 'mongodb-linux-x86_64-3.6.13' - minio: '2019-05-23T00-29-34Z' - wired_tiger: 'false' - ant_test: 'test_quick_coverage' + mongo: ['3.6.13', '3.6.23'] + test_cmd: ['test_quick_coverage', 'NoLongTests'] + steps: - uses: actions/checkout@v3 @@ -63,7 +54,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: ${{matrix.java}} + java-version: 11 - name: Set up Python 3.7 # sample service is on 3.7, handle service is on sdkbase2.latest uses: actions/setup-python@v3 @@ -74,6 +65,19 @@ jobs: shell: bash run: | export HOMEDIR=`pwd` + export MINIO_VER='2019-05-23T00-29-34Z' + export MONGO_VER='mongodb-linux-x86_64-${{matrix.mongo}}' + # 3.6.13 has wired_tiger set to 'false', 3.6.23 has wired_tiger 'true' + if [ "${{matrix.mongo}}" = "3.6.13" ]; then + export WIRED_TIGER="false" + else + export WIRED_TIGER="true" + fi + export ARANGODB_VER=3.9.1 + export ARANGODB_V=39 + export HS_COMMIT=08e18379817e16db920501b66ba62b66598f506c + export LOG_COMMIT=b549c557e3c519e0a55eadf7863a93db25cd6806 + export SAMPLE_COMMIT=6813fb148e95db2b11db6eea04f4d1d45cbb7119 # set up python dependencies cd python_dependencies @@ -89,16 +93,12 @@ jobs: export JARSDIR=$(pwd)/jars/lib/jars/ # set up arango - export ARANGODB_VER=3.9.1 - export ARANGODB_V=39 curl -O https://download.arangodb.com/arangodb$ARANGODB_V/Community/Linux/arangodb3-linux-$ARANGODB_VER.tar.gz tar -xf arangodb3-linux-$ARANGODB_VER.tar.gz export ARANGO_EXE=$(pwd)/arangodb3-linux-$ARANGODB_VER/bin/arangod export ARANGO_JS=$(pwd)/arangodb3-linux-$ARANGODB_VER/usr/share/arangodb3/js/ # set up handle service - export HS_COMMIT=08e18379817e16db920501b66ba62b66598f506c - export LOG_COMMIT=b549c557e3c519e0a55eadf7863a93db25cd6806 git clone https://github.com/kbase/handle_service2.git cd handle_service2/ git checkout $HS_COMMIT @@ -112,7 +112,6 @@ jobs: cd ../.. # set up sample service - export SAMPLE_COMMIT=6813fb148e95db2b11db6eea04f4d1d45cbb7119 git clone https://github.com/kbase/sample_service.git cd sample_service git checkout $SAMPLE_COMMIT @@ -126,12 +125,12 @@ jobs: export BLOBEXE=$(pwd)/blobstore # set up mongo - wget -q http://fastdl.mongodb.org/linux/${{matrix.mongo}}.tgz - tar xfz ${{matrix.mongo}}.tgz - export MONGOD=`pwd`/${{matrix.mongo}}/bin/mongod + wget -q http://fastdl.mongodb.org/linux/$MONGO_VER.tgz + tar xfz $MONGO_VER.tgz + export MONGOD=`pwd`/$MONGO_VER/bin/mongod # set up minio - wget -q https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.${{matrix.minio}} -O minio + wget -q https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.$MINIO_VER -O minio chmod a+x minio export MINIOD=`pwd`/minio @@ -141,7 +140,7 @@ jobs: sed -i "s#^test.temp.dir =.*#test.temp.dir=temp_test_dir#" test.cfg sed -i "s#^test.mongo.exe.*#test.mongo.exe=$MONGOD#" test.cfg sed -i "s#^test.minio.exe.*#test.minio.exe=$MINIOD#" test.cfg - sed -i "s#^test.mongo.useWiredTiger.*#test.mongo.useWiredTiger=${{matrix.wired_tiger}}#" test.cfg + sed -i "s#^test.mongo.useWiredTiger.*#test.mongo.useWiredTiger=$WIRED_TIGER#" test.cfg sed -i "s#^test.jars.dir.*#test.jars.dir=$JARSDIR#" test.cfg sed -i "s#^test.blobstore.exe.*#test.blobstore.exe=$BLOBEXE#" test.cfg sed -i "s#^test.handleservice.dir.*#test.handleservice.dir=$HSDIR#" test.cfg @@ -150,13 +149,32 @@ jobs: sed -i "s#^test.arango.js.*#test.arango.js=$ARANGO_JS#" test.cfg cat test.cfg - - name: Run tests - shell: bash + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + gradle-version: 7.6 + + - name: build the war file (requires compileJava, javadoc) + run: gradle war --info + + - name: run the ${{matrix.test_cmd}} tests and generate coverage run: | - ant javadoc - ant ${{matrix.ant_test}} + mkdir test-reports + if [ "${{matrix.test_cmd}}" = "test_quick_coverage" ]; then + ant "${{matrix.test_cmd}}" + else + gradle "coverage${{matrix.test_cmd}}" --info + fi - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 + if: matrix.test_cmd == "test_quick_coverage" with: fail_ci_if_error: true + + - name: upload jacoco report + uses: actions/upload-artifact@v3 + if: matrix.test_cmd != "test_quick_coverage" + with: + name: coverage${{matrix.test_cmd}}-${{matrix.mongo}}.exec + path: build/jacoco/test${{matrix.test_cmd}}.exec diff --git a/build.gradle b/build.gradle index 0768e76e..1e032b2c 100644 --- a/build.gradle +++ b/build.gradle @@ -1,156 +1,326 @@ /* - * This file was generated by the Gradle 'init' task. - * - * This is a general purpose Gradle build. - * Learn more about Gradle by exploring our samples at https://docs.gradle.org/7.4.2/samples + * Gradle file for the KBase workspace_deluxe. */ plugins { - id 'java' - id 'war' - id 'jacoco' + id "java" + id "war" + id "jacoco" +} + +group = "workspace service" +version = "0.14.1" + +ext { + clientJarFile = "WorkspaceClient.jar" + serviceJarFile = "WorkspaceService.jar" + warFile = "WorkspaceService.war" + warXml = "war/web.xml" + // directories + src = "src" + dist = "dist" + jarDir = "../jars/lib/jars" + downloadDir = "$buildDir/download" + tempunpack = "$buildDir/tmp/unpackedjars" + // other config + kbaseCommonJar = "kbase-common-0.1.1.jar" + testConfig = "test.cfg" } repositories { mavenCentral() } -compileJava { - options.release = 8 +configurations { + implementation + testImplementation + runtimeDeps { + extendsFrom implementation + } + allDeps { + extendsFrom implementation, testImplementation + } } -test { - systemProperty "test.cfg", "./test.cfg" - maxHeapSize = "3G" +dependencies { + implementation files("$jarDir/kbase/common/$kbaseCommonJar") + implementation files("$jarDir/ini4j/ini4j-0.5.2.jar") + implementation files("$jarDir/jetty/jetty-all-7.0.0.jar") + implementation files("$jarDir/jna/jna-3.4.0.jar") + implementation files("$jarDir/servlet/servlet-api-2.5.jar") + implementation files("$jarDir/syslog4j/syslog4j-0.9.46.jar") + implementation files("$jarDir/annotation/javax.annotation-api-1.3.2.jar") + implementation files("$jarDir/junit/junit-4.12.jar") + implementation files("$jarDir/hamcrest/hamcrest-core-1.3.jar") + implementation files("$jarDir/kbase/common/kbase-common-0.1.1.jar") + implementation files("$jarDir/joda/joda-time-2.2.jar") + implementation files("$jarDir/kbase/auth/kbase-auth-0.4.4.jar") + implementation files("$jarDir/jackson/jackson-annotations-2.9.9.jar") + implementation files("$jarDir/jackson/jackson-core-2.9.9.jar") + implementation files("$jarDir/jackson/jackson-databind-2.9.9.jar") + implementation files("$jarDir/kbase/shock/shock-client-0.1.0.jar") + implementation files("$jarDir/apache_commons/commons-logging-1.1.1.jar") + implementation files("$jarDir/apache_commons/http/httpclient-4.5.9.jar") + implementation files("$jarDir/apache_commons/http/httpcore-4.4.5.jar") + implementation files("$jarDir/apache_commons/http/httpmime-4.5.8.jar") + implementation files("$jarDir/kbase/kidl/kbase-kidl-parser-1409261812-7863aef.jar") + implementation files("$jarDir/apache_commons/commons-codec-1.8.jar") + implementation files("$jarDir/apache_commons/commons-io-2.4.jar") + implementation files("$jarDir/apache_commons/commons-lang3-3.1.jar") + implementation files("$jarDir/picocli/picocli-4.6.1.jar") + implementation files("$jarDir/mongo/mongo-java-driver-3.12.10.jar") + implementation files("$jarDir/slf4j/slf4j-api-1.7.30.jar") + implementation files("$jarDir/logback/logback-core-1.1.2.jar") + implementation files("$jarDir/logback/logback-classic-1.1.2.jar") + implementation files("$jarDir/google/guava-14.0.1.jar") + implementation files("$jarDir/caffeinecache/caffeine-2.9.3.jar") + implementation files("$jarDir/kafka/kafka-clients-2.1.0.jar") + implementation files("$jarDir/kbase/handle/AbstractHandleClient-1.0.0.jar") + implementation files("$jarDir/kbase/sample/SampleServiceClient-0.1.1.jar") + implementation files("$jarDir/amazon/V2/annotations-2.17.214.jar") + implementation files("$jarDir/amazon/V2/arns-2.17.214.jar") + implementation files("$jarDir/amazon/V2/auth-2.17.214.jar") + implementation files("$jarDir/amazon/V2/aws-core-2.17.214.jar") + implementation files("$jarDir/amazon/V2/aws-query-protocol-2.17.214.jar") + implementation files("$jarDir/amazon/V2/aws-xml-protocol-2.17.214.jar") + implementation files("$jarDir/amazon/V2/eventstream-1.0.1.jar") + implementation files("$jarDir/amazon/V2/http-client-spi-2.17.214.jar") + implementation files("$jarDir/amazon/V2/json-utils-2.17.214.jar") + implementation files("$jarDir/amazon/V2/metrics-spi-2.17.214.jar") + implementation files("$jarDir/amazon/V2/profiles-2.17.214.jar") + implementation files("$jarDir/amazon/V2/protocol-core-2.17.214.jar") + implementation files("$jarDir/amazon/V2/regions-2.17.214.jar") + implementation files("$jarDir/amazon/V2/s3-2.17.214.jar") + implementation files("$jarDir/amazon/V2/sdk-core-2.17.214.jar") + implementation files("$jarDir/amazon/V2/third-party-jackson-core-2.17.214.jar") + implementation files("$jarDir/amazon/V2/url-connection-client-2.17.214.jar") + implementation files("$jarDir/amazon/V2/utils-2.17.214.jar") + implementation files("$jarDir/reactivestreams/reactive-streams-1.0.3.jar") + + // TEST + testImplementation files("$jarDir/apache_commons/commons-collections-3.2.1.jar") + testImplementation files("$jarDir/apache_commons/commons-lang-2.4.jar") + testImplementation files("$jarDir/apache_commons/velocity-1.7.jar") + testImplementation files("$jarDir/arangodb/arangodb-java-driver-6.7.2.jar") + testImplementation files("$jarDir/arangodb/velocypack-2.4.0.jar") + testImplementation files("$jarDir/bytebuddy/byte-buddy-1.9.10.jar") + testImplementation files("$jarDir/bytebuddy/byte-buddy-agent-1.9.10.jar") + testImplementation files("$jarDir/equalsverifier/equalsverifier-3.1.10.jar") + testImplementation files("$jarDir/jsemver/java-semver-0.9.0.jar") + testImplementation files("$jarDir/kbase/auth2/kbase-auth2test-0.2.4.jar") + testImplementation files("$jarDir/mockito/mockito-core-3.0.0.jar") + testImplementation files("$jarDir/objenesis/objenesis-2.6.jar") +} + +// download the KBase jars if they aren't already present +task downloadJars { + doLast { + def dir = file(downloadDir) + if (!dir.exists()) { + dir.mkdirs() + } + def url = "https://github.com/kbase/jars/raw/master/lib/jars/kbase" + def jars = [ + "auth/kbase-auth-0.4.4.jar", + "auth2/kbase-auth2test-0.2.4.jar", + "common/$kbaseCommonJar", + "handle/AbstractHandleClient-1.0.0.jar", + "kidl/kbase-kidl-parser-1409261812-7863aef.jar", + "sample/SampleServiceClient-0.1.1.jar", + "shock/shock-client-0.1.0.jar" + ] + jars.each { jar -> + def jarName = jar.split("/")[-1] + def file = file("$downloadDir/$jarName") + if (!file.exists()) { + ant.get(src: "$url/$jarName", dest: "$downloadDir/$jarName") + } + } + } } // Custom java project layout sourceSets { main { java { - srcDirs = ['src'] - exclude '**/test/**' + srcDir src + exclude "**/test/**" } } test { java { - srcDirs = ['src'] - include '**/test/**' - exclude '*Tester.java' + srcDir src + } + resources { + srcDir src + include "**/*.properties" + include "**/*.cfg" + include "**/*.spec" + include "**/*.instance.*" + include "**/instance.*" + include "**/*.instance" + include "**/*.html" + include "**/*.css" + include "**/*.gif" + include "**/*.js" + include "**/*.png" + include "**/*.txt" + include "**/*.weirdsuffix" } } } -war { - webXml = file('war/web.xml') +tasks.withType(JavaCompile) { + classpath = configurations.allDeps + options.compilerArgs = ["-source", "11", "-target", "11", "-g"] } -def fromURL = { url, name -> - File file = new File("$buildDir/download/${name}.jar") - file.parentFile.mkdirs() - if (!file.exists()) { - new URL(url).withInputStream { downloadStream -> - file.withOutputStream { fileOut -> - fileOut << downloadStream - } - } +task clientJar(type: Jar, dependsOn: compileJava) { + from sourceSets.main.output + include "us/kbase/workspace/*.class" + exclude "us/kbase/workspace/WorkspaceServer.class" + include "us/kbase/common/service/*.class" + exclude "us/kbase/common/service/JsonServer*" + archiveFileName = clientJarFile + destinationDirectory = file("$dist/client") +} + +task serviceJar(type: Jar, dependsOn: compileJava) { + from sourceSets.main.output + archiveFileName = serviceJarFile + destinationDirectory = file("$dist") +} + +task buildJars { + dependsOn serviceJar, clientJar +} + +task unzipJar(type: Copy) { + from zipTree("$jarDir/kbase/common/$kbaseCommonJar") + into "$tempunpack" +} + +// by default, the files go into build/docs/javadoc +tasks.javadoc { + dependsOn unzipJar + classpath = configurations.allDeps + failOnError = false + options { + author = false + noDeprecated = false + noDeprecatedList = false + noIndex = false + noNavBar = false + noTree = false + splitIndex = true + use = true + version = true + links = [ + "http://download.oracle.com/javase/11/docs/api/", + "https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-core/2.9.9/", + "https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-databind/2.9.9/" + ] + } + source = fileTree(dir: "$src/us/kbase", + include: [ + "workspace/*.java", + "common/service/*.java" + ], + exclude: [ + "workspace/WorkspaceServer.java", + "common/service/JsonServer*", + "common/service/JacksonTupleModule.java", + "common/service/JsonClientCaller.java", + "common/service/JsonTreeTraversingParser.java", + "common/service/KBaseJsonParser.java" + ]) + source += fileTree(dir: "$tempunpack/us/kbase/common/service", include: [ + "UObject.java", + "JsonTokenStream.java", + "*Exception.java" + ]) + + doLast { + delete "$tempunpack" } - files(file.absolutePath) } -dependencies { +tasks.war { + dependsOn javadoc + description = "Build the WAR file. Assumes compilation has been run." + archiveFileName = warFile + destinationDirectory = file("$dist") + + // location of the web.xml file + webXml = file("$warXml") + + duplicatesStrategy "exclude" - // ### General application dependencies ### - - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/auth/kbase-auth-0.4.4.jar', - 'kbase-auth-0.4.4' - ) - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/kidl/kbase-kidl-parser-1409261812-7863aef.jar', - 'kbase-kidl-parser-1409261812-7863aef' - ) - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.9.9' - implementation 'com.fasterxml.jackson.core:jackson-core:2.9.9' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.9.9' - implementation 'org.ini4j:ini4j:0.5.2' - implementation 'commons-io:commons-io:2.4' - implementation 'org.apache.commons:commons-lang3:3.1' - implementation 'commons-codec:commons-codec:1.8' - implementation 'info.picocli:picocli:4.6.1' - implementation 'org.mongodb:mongo-java-driver:3.12.10' - implementation 'org.slf4j:slf4j-api:1.7.30' - implementation 'ch.qos.logback:logback-classic:1.1.2' - implementation 'com.google.guava:guava:14.0.1' - implementation 'com.github.ben-manes.caffeine:caffeine:2.9.3' - implementation 'org.apache.kafka:kafka-clients:2.1.0' - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/handle/AbstractHandleClient-1.0.0.jar', - 'AbstractHandleClient-1.0.0' - ) - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/sample/SampleServiceClient-0.1.1.jar', - 'SampleServiceClient-0.1.1' - ) - - // ### Server dependencies, specifically for java_common JsonServerServlet ### - - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/common/kbase-common-0.1.0.jar', - 'kbase-common-0.1.0' - ) - // joda-time is only required for kbase-common. Would be nice to remove - implementation 'joda-time:joda-time:2.2' - // this is OOOOOOLD. But that probably means updating java_common - implementation 'org.eclipse.jetty.aggregate:jetty-all:7.0.0.v20091005' - implementation 'javax.servlet:servlet-api:2.5' - // Syslog4j 0.9.46 doesn't appear to be available on Maven. It apparently lives in - // a JetBrains artifact server, but that's too much trouble and there's only one version there - // anyway. - // https://mvnrepository.com/artifact/org.jetbrains/syslog4j/0.9.46 - // Need to rework the java common logger to not use syslog4j at all since it's abandonware - // and has a ton of CVEs, even in the newer versions. - // The old ant file used to specify a JNA jar which is listed as optional in Maven, and since - // the workspace doesn't actually talk to syslog anymore, is unnecessary. - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/syslog4j/syslog4j-0.9.46.jar', - 'syslog4j-0.9.46' - ) - implementation 'javax.annotation:javax.annotation-api:1.3.2' - - // ### Blobstore / Shock client and dependencies ### - - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/shock/shock-client-0.1.0.jar', - 'shock-client-0.1.0' - ) - implementation 'org.apache.httpcomponents:httpclient:4.5.9' - implementation 'org.apache.httpcomponents:httpmime:4.5.8' - - // ### Amazon S3 ### - - implementation('software.amazon.awssdk:s3:2.17.214') { - exclude module: 'apache-client' - exclude module: 'netty-nio-client' + // add the dependencies to lib + into("WEB-INF/lib") { + from sourceSets.main.compileClasspath } - implementation 'software.amazon.awssdk:url-connection-client:2.17.214' - - // ### Test ### - - testImplementation 'junit:junit:4.12' - testImplementation 'org.hamcrest:hamcrest-core:1.3' - testImplementation 'com.github.zafarkhaja:java-semver:0.9.0' - testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.1.10' - // TODO will want to upgrade auth2 to gradle at some point. - // How will the proper jars and templates zip file be built for tests here? - // Ideal is to no longer use the github jars repo anywhere. - // The auth2test code pulling in a gradle file and building seems a little nuts - // Maybe clone the auth repo, build, and put the build folder into test.cfg here? - testImplementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/auth2/kbase-auth2test-0.2.4.jar', - 'kbase-auth2test-0.2.4' - ) - testImplementation 'com.arangodb:arangodb-java-driver:6.7.2' - testImplementation 'org.mockito:mockito-core:3.0.0' + // copy the compiled classes over + into("WEB-INF/classes") { + from sourceSets.main.output.classesDirs + } + // copy in the documentation + into("WEB-INF/classes/server_docs/javadoc") { + from javadoc.destinationDir + } +} + +task expandWar(type: Copy) { + from zipTree("$dist/$warFile") + into "war_contents" } +// test reports are saved to build/reports/tests/ +tasks.withType(Test) { + dependsOn compileTestJava + useJUnit() + systemProperty "test.cfg", testConfig + maxHeapSize = "3G" + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "short" + showStandardStreams = true + showExceptions = true + showCauses = true + } +} + +task testNoLongTests(type: Test) { + description = "run all tests except the slow tests" + systemProperty "test.cfg", testConfig + forkEvery 1 + exclude "**/test/**/**LongTest.java" +} + +// output from coverage tasks is saved to build/reports/jacoco/ +tasks.withType(JacocoReport) { + reports { + html.required = true + xml.required = true + csv.required = true + } + sourceSets sourceSets.main +} + +task coverageNoLongTests(type: JacocoReport, dependsOn: testNoLongTests) { + group = "Reporting" + description = "Generate code coverage reports for all tests except the slow tests" + executionData(testNoLongTests) +} + +task createScript { + dependsOn compileJava + description = "create cli script" + + def runtimeClasspath = configurations.runtimeDeps.files.join(":") + file("update_workspace_database_schema").write("#!/bin/sh\n" + + "java -cp ${project.rootDir.absolutePath}/$dist/$serviceJarFile:$runtimeClasspath us.kbase.workspace.kbase.SchemaUpdaterCLI \$@\n") + file("update_workspace_database_schema").setExecutable(true) + // always run the script + outputs.upToDateWhen { false } +} diff --git a/build.xml b/build.xml index 37b8e521..ea8a6e8b 100644 --- a/build.xml +++ b/build.xml @@ -75,7 +75,7 @@ - + @@ -242,7 +242,8 @@ + source="11" splitindex="true" use="true" Windowtitle="Workspace 0.14.1 API" + version="true">