From 58020e4d625d5e96fae0f78259e2b8e8c8ccf4fe Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 6 Feb 2024 13:44:42 -0800 Subject: [PATCH 01/19] test Mongo v3 and v7 --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b26b691..28dbfa5f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,12 @@ jobs: minio: '2019-05-23T00-29-34Z' wired_tiger: 'false' ant_test: 'test_quick_coverage' + # Mongo7 setup + - java: '11' + mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' + minio: '2019-05-23T00-29-34Z' + wired_tiger: 'false' + ant_test: 'test_quick_coverage' steps: - uses: actions/checkout@v3 From 4b59fd45c7cc19d66f6b63272ca303c8d9949e3a Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 14:33:17 -0800 Subject: [PATCH 02/19] update build file, classpath file, and Mongo in main --- .classpath | 7 +++++-- build.xml | 7 +++++-- .../typedobj/db/test/MongoTypeStorageTest.java | 5 +++-- .../typedobj/db/test/TypeRegisteringTest.java | 5 +++-- .../kbase/workspace/kbase/InitWorkspaceServer.java | 14 ++++++++------ src/us/kbase/workspace/kbase/SchemaUpdaterCLI.java | 2 +- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.classpath b/.classpath index 1acd7683..7a0894eb 100644 --- a/.classpath +++ b/.classpath @@ -85,7 +85,10 @@ - + + + + @@ -105,6 +108,6 @@ - + diff --git a/build.xml b/build.xml index 37b8e521..463c0a75 100644 --- a/build.xml +++ b/build.xml @@ -24,7 +24,7 @@ - + @@ -62,7 +62,10 @@ - + + + + diff --git a/src/us/kbase/typedobj/db/test/MongoTypeStorageTest.java b/src/us/kbase/typedobj/db/test/MongoTypeStorageTest.java index 384abea4..e3a5afb0 100644 --- a/src/us/kbase/typedobj/db/test/MongoTypeStorageTest.java +++ b/src/us/kbase/typedobj/db/test/MongoTypeStorageTest.java @@ -12,7 +12,8 @@ import org.junit.BeforeClass; import org.junit.Test; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.common.test.TestCommon; @@ -43,7 +44,7 @@ public static void setUp() throws Exception { MONGO.getServerPort()); @SuppressWarnings("resource") - final MongoClient mc = new MongoClient("localhost:" + MONGO.getServerPort()); + final MongoClient mc = MongoClients.create("mongodb://localhost:" + MONGO.getServerPort()); MONGO_DB = mc.getDatabase("test_" + MongoTypeStorageTest.class.getSimpleName()); } diff --git a/src/us/kbase/typedobj/db/test/TypeRegisteringTest.java b/src/us/kbase/typedobj/db/test/TypeRegisteringTest.java index d51513fc..a71f7cd7 100644 --- a/src/us/kbase/typedobj/db/test/TypeRegisteringTest.java +++ b/src/us/kbase/typedobj/db/test/TypeRegisteringTest.java @@ -33,7 +33,8 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.common.test.TestCommon; @@ -144,7 +145,7 @@ public static MongoDatabase createMongoDbConnection() throws Exception { mongo.getTempDir()); } @SuppressWarnings("resource") - final MongoClient mcli = new MongoClient("localhost:" + mongo.getServerPort()); + final MongoClient mcli = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); final MongoDatabase mdb = mcli.getDatabase("TypeRegisteringTest"); TestCommon.destroyDB(mdb); return mdb; diff --git a/src/us/kbase/workspace/kbase/InitWorkspaceServer.java b/src/us/kbase/workspace/kbase/InitWorkspaceServer.java index 8c69fb9b..dae1b392 100644 --- a/src/us/kbase/workspace/kbase/InitWorkspaceServer.java +++ b/src/us/kbase/workspace/kbase/InitWorkspaceServer.java @@ -22,11 +22,11 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.slf4j.LoggerFactory; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; import com.mongodb.MongoCredential; import com.mongodb.MongoException; import com.mongodb.ServerAddress; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoDatabase; import us.kbase.abstracthandle.AbstractHandleClient; @@ -411,16 +411,18 @@ private static SampleIdHandlerFactory getSampleIdHandlerFactory( public static MongoClient buildMongo(final KBaseWorkspaceConfig c, final String dbName) throws WorkspaceInitException { //TODO ZLATER MONGO handle shards & replica sets - final MongoClientOptions opts = MongoClientOptions.builder() - .retryWrites(c.getMongoRetryWrites()).build(); + final MongoClientSettings.Builder mongoBuilder = MongoClientSettings.builder() + .retryWrites(c.getMongoRetryWrites()) + .applyToClusterSettings(builder -> builder.hosts( + Arrays.asList(new ServerAddress(c.getMongoHost())))); try { if (c.getMongoUser() != null) { final MongoCredential creds = MongoCredential.createCredential( c.getMongoUser(), dbName, c.getMongoPassword().toCharArray()); // unclear if and when it's safe to clear the password - return new MongoClient(new ServerAddress(c.getHost()), creds, opts); + return MongoClients.create(mongoBuilder.credential(creds).build()); } else { - return new MongoClient(new ServerAddress(c.getHost()), opts); + return MongoClients.create(mongoBuilder.build()); } } catch (MongoException e) { LoggerFactory.getLogger(InitWorkspaceServer.class).error( diff --git a/src/us/kbase/workspace/kbase/SchemaUpdaterCLI.java b/src/us/kbase/workspace/kbase/SchemaUpdaterCLI.java index e3357305..0453d85b 100644 --- a/src/us/kbase/workspace/kbase/SchemaUpdaterCLI.java +++ b/src/us/kbase/workspace/kbase/SchemaUpdaterCLI.java @@ -9,7 +9,7 @@ import org.slf4j.LoggerFactory; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoDatabase; import ch.qos.logback.classic.Level; From e1a3897e3fb1ed9fbeaf2c1bcc449365b92b0d39 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 14:49:48 -0800 Subject: [PATCH 03/19] fix test.workspace --- .../workspace/WorkspaceIntegrationWithGridFSTest.java | 9 +++++---- .../kbase/workspace/test/workspace/WorkspaceTester.java | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/us/kbase/workspace/test/workspace/WorkspaceIntegrationWithGridFSTest.java b/src/us/kbase/workspace/test/workspace/WorkspaceIntegrationWithGridFSTest.java index 14a753df..ffbb3681 100644 --- a/src/us/kbase/workspace/test/workspace/WorkspaceIntegrationWithGridFSTest.java +++ b/src/us/kbase/workspace/test/workspace/WorkspaceIntegrationWithGridFSTest.java @@ -32,7 +32,8 @@ import org.junit.Test; import org.slf4j.LoggerFactory; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import ch.qos.logback.classic.Level; @@ -114,7 +115,7 @@ public static void setUpClass() throws Exception { System.out.println("Started test mongo instance at localhost: " + MONGO.getServerPort()); @SuppressWarnings("resource") - final MongoClient mcli = new MongoClient("localhost:" + MONGO.getServerPort()); + final MongoClient mcli = MongoClients.create("mongodb://localhost:" + MONGO.getServerPort()); WSDB = mcli.getDatabase(WSDB_NAME); TYPEDB = mcli.getDatabase(TYPEDB_NAME); TestCommon.destroyDB(WSDB); @@ -145,8 +146,8 @@ public static void tearDownClass() throws Exception { @Before public void clearDB() throws Exception { - try (final MongoClient mongoClient = new MongoClient( - "localhost:" + MONGO.getServerPort())) { + try (final MongoClient mongoClient = MongoClients.create( + "mongodb://localhost:" + MONGO.getServerPort())) { TestCommon.destroyDB(mongoClient.getDatabase(WSDB_NAME)); } } diff --git a/src/us/kbase/workspace/test/workspace/WorkspaceTester.java b/src/us/kbase/workspace/test/workspace/WorkspaceTester.java index 24dc34f6..1e4e6e3f 100644 --- a/src/us/kbase/workspace/test/workspace/WorkspaceTester.java +++ b/src/us/kbase/workspace/test/workspace/WorkspaceTester.java @@ -99,7 +99,8 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; @RunWith(Parameterized.class) @@ -173,7 +174,7 @@ public static void tearDownClass() throws Exception { @Before public void clearDB() throws Exception { - try (final MongoClient cli = new MongoClient("localhost:" + mongo.getServerPort())) { + try (final MongoClient cli = MongoClients.create("mongodb://localhost:" + mongo.getServerPort())) { TestCommon.destroyDB(cli.getDatabase(DB_WS_NAME)); } ws.setConfig(DynamicConfigUpdate.getDefault()); @@ -204,7 +205,7 @@ public WorkspaceTester( } if (!CONFIGS.containsKey(config)) { @SuppressWarnings("resource") - final MongoClient mcli = new MongoClient("localhost:" + mongo.getServerPort()); + final MongoClient mcli = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); final MongoDatabase wsdb = mcli.getDatabase(DB_WS_NAME); final MongoDatabase tdb = mcli.getDatabase(DB_TYPE_NAME); TestCommon.destroyDB(wsdb); From 8a08e64719ba94b3f1aa2bfcaf3297bc0b623b4f Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 15:17:44 -0800 Subject: [PATCH 04/19] fix test.kbase --- src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java | 6 ++++-- src/us/kbase/workspace/test/kbase/LoggingTest.java | 5 +++-- .../workspace/test/kbase/SampleServiceIntegrationTest.java | 6 ++++-- src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java | 2 +- src/us/kbase/workspace/test/kbase/TypeDelegationTest.java | 6 ++++-- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java b/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java index 53bab68a..67ef7567 100644 --- a/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java +++ b/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java @@ -87,7 +87,9 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.MongoClient; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; /* @@ -538,7 +540,7 @@ public static void tearDownClass() throws Exception { @Before public void clearDB() throws Exception { - try (final MongoClient mcli = new MongoClient("localhost:" + mongo.getServerPort())) { + try (final MongoClient mcli = MongoClients.create("mongodb://localhost:" + mongo.getServerPort())) { for (final String name: list(DB_WS_NAME_1, DB_WS_NAME_2, DB_WS_NAME_AUTH2_ADMINS)) { final MongoDatabase wsdb = mcli.getDatabase(name); TestCommon.destroyDB(wsdb); diff --git a/src/us/kbase/workspace/test/kbase/LoggingTest.java b/src/us/kbase/workspace/test/kbase/LoggingTest.java index 2ef05335..bd6d45bf 100644 --- a/src/us/kbase/workspace/test/kbase/LoggingTest.java +++ b/src/us/kbase/workspace/test/kbase/LoggingTest.java @@ -52,7 +52,8 @@ import us.kbase.workspace.database.DynamicConfig; import us.kbase.workspace.test.WorkspaceServerThread; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; /** Tests application logging only - not the standard logging that comes with @@ -221,7 +222,7 @@ public static void tearDownClass() throws Exception { @Before public void clearDB() throws Exception { logout.reset(); - try (final MongoClient mcli = new MongoClient("localhost:" + mongo.getServerPort())) { + try (final MongoClient mcli = MongoClients.create("mongodb://localhost:" + mongo.getServerPort())) { final MongoDatabase wsdb = mcli.getDatabase(DB_WS_NAME); TestCommon.destroyDB(wsdb); wsdb.getCollection("dyncfg").insertOne( diff --git a/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java b/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java index c64337cc..5569cfbf 100644 --- a/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java +++ b/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java @@ -30,7 +30,9 @@ import com.arangodb.model.CollectionCreateOptions; import com.github.zafarkhaja.semver.Version; import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoClient; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.auth.AuthToken; @@ -310,7 +312,7 @@ public static void tearDownClass() throws Exception { @Before public void clearDB() throws Exception { - try (final MongoClient cli = new MongoClient("localhost:" + MONGO.getServerPort())) { + try (final MongoClient cli = MongoClients.create("mongodb://localhost:" + MONGO.getServerPort())) { final MongoDatabase db = cli.getDatabase(WS_DB); TestCommon.destroyDB(db); db.getCollection("dyncfg").insertOne( diff --git a/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java b/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java index ea402a5e..7c94f5d3 100644 --- a/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java +++ b/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java @@ -27,7 +27,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; import com.mongodb.client.MongoDatabase; import us.kbase.common.test.MapBuilder; diff --git a/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java b/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java index a1950a58..8ef1ab72 100644 --- a/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java +++ b/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java @@ -22,7 +22,9 @@ import org.junit.Test; import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoClient; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.auth.AuthToken; @@ -252,7 +254,7 @@ public static void tearDownClass() throws Exception { @Before public void clearDB() throws Exception { // since we're testing types, we nuke the type database as well as the workspace db - try (final MongoClient mcli = new MongoClient("localhost:" + MONGO.getServerPort())) { + try (final MongoClient mcli = MongoClients.create("mongodb://localhost:" + MONGO.getServerPort())) { for (final String name: list( DB_NAME_WS, DB_NAME_WS_TYPES, DB_NAME_WS_REMOTE, DB_NAME_WS_REMOTE_TYPES)) { final MongoDatabase wsdb = mcli.getDatabase(name); From d4a42384b025a3ebbafe756acb7d819c6686439b Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 15:47:41 -0800 Subject: [PATCH 05/19] fix test.database.mongo --- .../workspace/test/database/mongo/GridFSBlobStoreTest.java | 6 ++++-- .../workspace/test/database/mongo/MongoInternalsTest.java | 7 ++++--- .../workspace/test/database/mongo/MongoStartUpTest.java | 6 +++--- .../test/database/mongo/MongoWorkspaceDBTest.java | 6 ++++-- .../test/database/mongo/S3BlobStoreIntegrationTest.java | 6 +++--- .../workspace/test/database/mongo/SchemaUpdaterTest.java | 5 +++-- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/us/kbase/workspace/test/database/mongo/GridFSBlobStoreTest.java b/src/us/kbase/workspace/test/database/mongo/GridFSBlobStoreTest.java index 8b960273..4a0f308c 100644 --- a/src/us/kbase/workspace/test/database/mongo/GridFSBlobStoreTest.java +++ b/src/us/kbase/workspace/test/database/mongo/GridFSBlobStoreTest.java @@ -24,7 +24,9 @@ import com.github.zafarkhaja.semver.Version; import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoClient; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import com.mongodb.client.gridfs.GridFSBucket; import com.mongodb.client.gridfs.GridFSBuckets; @@ -58,7 +60,7 @@ public static void setUpClass() throws Exception { mongo.getTempDir()); TestCommon.stfuLoggers(); @SuppressWarnings("resource") - final MongoClient mongoClient = new MongoClient("localhost:" + mongo.getServerPort()); + final MongoClient mongoClient = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); final MongoDatabase db = mongoClient.getDatabase("GridFSBackendTest"); gfs = GridFSBuckets.create(db); gfsb = new GridFSBlobStore(db); diff --git a/src/us/kbase/workspace/test/database/mongo/MongoInternalsTest.java b/src/us/kbase/workspace/test/database/mongo/MongoInternalsTest.java index 6e92b71b..d2fe6af5 100644 --- a/src/us/kbase/workspace/test/database/mongo/MongoInternalsTest.java +++ b/src/us/kbase/workspace/test/database/mongo/MongoInternalsTest.java @@ -85,7 +85,9 @@ import us.kbase.workspace.test.workspace.WorkspaceTester; import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoClient; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; @@ -125,8 +127,7 @@ public static void setUpClass() throws Exception { System.out.println("Using mongo temp dir " + mongo.getTempDir()); TestCommon.stfuLoggers(); - String mongohost = "localhost:" + mongo.getServerPort(); - mongoClient = new MongoClient(mongohost); + mongoClient = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); db = mongoClient.getDatabase("MongoInternalsTest"); final MongoDatabase tdb = mongoClient.getDatabase("MongoInternalsTest_types"); TestCommon.destroyDB(db); diff --git a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java index 39c99ca5..f47b9e03 100644 --- a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java +++ b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java @@ -23,7 +23,8 @@ import org.junit.BeforeClass; import org.junit.Test; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; @@ -65,8 +66,7 @@ public static void setUpClass() throws Exception { TestCommon.useWiredTigerEngine()); System.out.println("Using mongo temp dir " + mongo.getTempDir()); TestCommon.stfuLoggers(); - String mongohost = "localhost:" + mongo.getServerPort(); - mongoClient = new MongoClient(mongohost); + mongoClient = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); final MongoDatabase mdb = mongoClient.getDatabase("MongoStartUpTest"); db = mongoClient.getDatabase("MongoStartUpTest"); diff --git a/src/us/kbase/workspace/test/database/mongo/MongoWorkspaceDBTest.java b/src/us/kbase/workspace/test/database/mongo/MongoWorkspaceDBTest.java index 91998822..c0242c69 100644 --- a/src/us/kbase/workspace/test/database/mongo/MongoWorkspaceDBTest.java +++ b/src/us/kbase/workspace/test/database/mongo/MongoWorkspaceDBTest.java @@ -38,7 +38,9 @@ import org.junit.Test; import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoClient; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.common.service.UObject; @@ -116,7 +118,7 @@ public static void setup() throws Exception { MONGO.getServerPort()); @SuppressWarnings("resource") - final MongoClient mc = new MongoClient("localhost:" + MONGO.getServerPort()); + final MongoClient mc = MongoClients.create("mongodb://localhost:" + MONGO.getServerPort()); MONGO_DB = mc.getDatabase("test_" + MongoWorkspaceDBTest.class.getSimpleName()); } diff --git a/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java b/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java index 61ed84ae..53dfb6b1 100644 --- a/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java +++ b/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java @@ -17,7 +17,8 @@ import org.junit.BeforeClass; import org.junit.Test; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import software.amazon.awssdk.regions.Region; @@ -67,9 +68,8 @@ public static void setUpClass() throws Exception { System.out.println("Using Mongo temp dir " + mongoCon.getTempDir()); System.out.println("Started mongo server at localhost:" + mongoCon.getServerPort()); - String mongohost = "localhost:" + mongoCon.getServerPort(); @SuppressWarnings("resource") - MongoClient mongoClient = new MongoClient(mongohost); + MongoClient mongoClient = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); mongo = mongoClient.getDatabase("MinioBackendTest"); minio = new MinioController( diff --git a/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java b/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java index e758c7ff..3d84d984 100644 --- a/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java +++ b/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java @@ -23,7 +23,8 @@ import org.junit.BeforeClass; import org.junit.Test; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; @@ -72,7 +73,7 @@ public static void setup() throws Exception { System.out.println("Started test mongo instance at localhost:" + MONGO.getServerPort()); - MC = new MongoClient("localhost:" + MONGO.getServerPort()); + MC = MongoClients.create("mongodb://localhost:" + MONGO.getServerPort()); setUpTestDB(); } From bdb88edd20e3eb582264eff94112c75561f705ff Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 16:30:59 -0800 Subject: [PATCH 06/19] fix compile issue --- src/us/kbase/workspace/kbase/InitWorkspaceServer.java | 3 ++- .../test/database/mongo/S3BlobStoreIntegrationTest.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/us/kbase/workspace/kbase/InitWorkspaceServer.java b/src/us/kbase/workspace/kbase/InitWorkspaceServer.java index dae1b392..6d4f66cd 100644 --- a/src/us/kbase/workspace/kbase/InitWorkspaceServer.java +++ b/src/us/kbase/workspace/kbase/InitWorkspaceServer.java @@ -27,6 +27,7 @@ import com.mongodb.ServerAddress; import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import us.kbase.abstracthandle.AbstractHandleClient; @@ -414,7 +415,7 @@ public static MongoClient buildMongo(final KBaseWorkspaceConfig c, final String final MongoClientSettings.Builder mongoBuilder = MongoClientSettings.builder() .retryWrites(c.getMongoRetryWrites()) .applyToClusterSettings(builder -> builder.hosts( - Arrays.asList(new ServerAddress(c.getMongoHost())))); + Arrays.asList(new ServerAddress(c.getHost())))); try { if (c.getMongoUser() != null) { final MongoCredential creds = MongoCredential.createCredential( diff --git a/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java b/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java index 53dfb6b1..8c3a3c1f 100644 --- a/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java +++ b/src/us/kbase/workspace/test/database/mongo/S3BlobStoreIntegrationTest.java @@ -69,7 +69,7 @@ public static void setUpClass() throws Exception { System.out.println("Started mongo server at localhost:" + mongoCon.getServerPort()); @SuppressWarnings("resource") - MongoClient mongoClient = MongoClients.create("mongodb://localhost:" + mongo.getServerPort()); + MongoClient mongoClient = MongoClients.create("mongodb://localhost:" + mongoCon.getServerPort()); mongo = mongoClient.getDatabase("MinioBackendTest"); minio = new MinioController( From 412bc7a9f54f7cd5681a2f68ede7c456c19120c2 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 16:47:58 -0800 Subject: [PATCH 07/19] remove HandleServiceController main function --- .../handle/HandleServiceController.java | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java b/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java index 1e348def..61ee378c 100644 --- a/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java +++ b/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java @@ -19,7 +19,6 @@ import us.kbase.auth.AuthToken; import us.kbase.common.test.controllers.mongo.MongoController; -import us.kbase.common.test.controllers.shock.ShockController; /** Q&D Utility to run the Handle Service for the purposes of testing from Java. @@ -137,36 +136,4 @@ public void destroy(final boolean deleteTempFiles, final boolean dumpLogToStdOut FileUtils.deleteDirectory(tempDir.toFile()); } } - - public static void main(String[] args) throws Exception { - MongoController monc = new MongoController( - "/kb/runtime/bin/mongod", - Paths.get("workspacetesttemp"), false); - ShockController sc = new ShockController( - "/kb/deployment/bin/shock-server", - "0.9.6", - Paths.get("workspacetesttemp"), - System.getProperty("test.user1"), - "localhost:" + monc.getServerPort(), - "shockdb", "foo", "foo", new URL("http://foo.com")); - HandleServiceController hsc = new HandleServiceController( - monc, - "http://localhost:" + sc.getServerPort(), - null, //this will break the hm, need a token - Paths.get("workspacetesttemp"), - new URL("http://foo.com"), - "KBASE_ADMIN", - "/kb/deployment/lib", - "handle_controller_test_handle_db"); - System.out.println("handlesrv: " + hsc.getHandleServerPort()); - System.out.println(hsc.getTempDir()); - Scanner reader = new Scanner(System.in); - System.out.println("any char to shut down"); - //get user input for a - reader.next(); - hsc.destroy(false); - sc.destroy(false); - monc.destroy(false); - reader.close(); - } } From c26599fbb0673f380a3e7c7ba9ca14ea6a6a6e64 Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 17:07:23 -0800 Subject: [PATCH 08/19] rm unused libraries and upgrade kbase common to 0.2.0 on clientside --- build.xml | 2 +- .../test/controllers/handle/HandleServiceController.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/build.xml b/build.xml index 463c0a75..24d71022 100644 --- a/build.xml +++ b/build.xml @@ -39,7 +39,7 @@ - + diff --git a/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java b/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java index 61ee378c..e38d1000 100644 --- a/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java +++ b/src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java @@ -9,10 +9,8 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.LinkedList; import java.util.Map; -import java.util.Scanner; import org.apache.commons.io.FileUtils; import org.ini4j.Ini; import org.ini4j.Profile.Section; From 58f2b252336b132d87920ef63cceb810244af22f Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 18:30:13 -0800 Subject: [PATCH 09/19] fix MongoStartUpTest.java --- .../test/database/mongo/MongoStartUpTest.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java index f47b9e03..d20f6dfa 100644 --- a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java +++ b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java @@ -297,9 +297,18 @@ private void createIndexes(final String dbname) throws Exception { method.invoke(null, wsdb); } - private Set getIndexes(final MongoDatabase db, final String collection) { + private Set getIndexes(final MongoDatabase db, final String collectionName) { final Set indexes = new HashSet<>(); - db.getCollection(collection).listIndexes().forEach((Consumer) indexes::add); + for (Document index: db.getCollection(collectionName).listIndexes()) { + // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes() + // no longer returns the namespace ns field in the index specification documents. + index.remove("ns"); + // some versions of Mongo return ints, some longs. Convert all to longs. + if (index.containsKey("expireAfterSeconds")) { + index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue()); + } + indexes.add(index); + } return indexes; } @@ -313,18 +322,15 @@ public void indexesConfig() throws Exception { new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("config", 1)) - .append("name", "config_1") - .append("ns", "MongoStartUpTest.config"), + .append("name", "config_1"), new Document("v", INDEX_VER) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "MongoStartUpTest.config") ); assertThat("incorrect indexes", getIndexes(db, "config"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesConfig"); createIndexes("indexesConfig"); - setNamespace(expectedIndexes, "indexesConfig.config"); assertThat("incorrect indexes", getIndexes(wsdb, "config"), is(expectedIndexes)); } @@ -334,18 +340,14 @@ public void indexesDynamicConfig() throws Exception { new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("key", 1)) - .append("name", "key_1") - .append("ns", "MongoStartUpTest.dyncfg"), + .append("name", "key_1"), new Document("v", INDEX_VER) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "MongoStartUpTest.dyncfg") ); assertThat("incorrect indexes", getIndexes(db, "dyncfg"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesDynConfig"); - createIndexes("indexesDynConfig"); - setNamespace(expectedIndexes, "indexesDynConfig.dyncfg"); assertThat("incorrect indexes", getIndexes(wsdb, "dyncfg"), is(expectedIndexes)); } From 33f1559ec9575ce240c5d582333205b687e2f63c Mon Sep 17 00:00:00 2001 From: Sijie Date: Fri, 16 Feb 2024 18:45:58 -0800 Subject: [PATCH 10/19] fix all failed tests in MongoStartUpTest.java --- .../test/database/mongo/MongoStartUpTest.java | 113 ++++++------------ 1 file changed, 39 insertions(+), 74 deletions(-) diff --git a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java index d20f6dfa..71e647e4 100644 --- a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java +++ b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java @@ -297,10 +297,10 @@ private void createIndexes(final String dbname) throws Exception { method.invoke(null, wsdb); } - private Set getIndexes(final MongoDatabase db, final String collectionName) { + private Set getAndNormalizeIndexes(final MongoDatabase db, final String collectionName) { final Set indexes = new HashSet<>(); for (Document index: db.getCollection(collectionName).listIndexes()) { - // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes() + // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getAndNormalizeIndexes() // no longer returns the namespace ns field in the index specification documents. index.remove("ns"); // some versions of Mongo return ints, some longs. Convert all to longs. @@ -312,10 +312,6 @@ private Set getIndexes(final MongoDatabase db, final String collection return indexes; } - private void setNamespace(final Set toBeModified, final String namespace) { - toBeModified.forEach(d -> d.append("ns", namespace)); - } - @Test public void indexesConfig() throws Exception { final Set expectedIndexes = set( @@ -327,11 +323,11 @@ public void indexesConfig() throws Exception { .append("key", new Document("_id", 1)) .append("name", "_id_") ); - assertThat("incorrect indexes", getIndexes(db, "config"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(db, "config"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesConfig"); createIndexes("indexesConfig"); - assertThat("incorrect indexes", getIndexes(wsdb, "config"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(wsdb, "config"), is(expectedIndexes)); } @Test @@ -345,10 +341,11 @@ public void indexesDynamicConfig() throws Exception { .append("key", new Document("_id", 1)) .append("name", "_id_") ); - assertThat("incorrect indexes", getIndexes(db, "dyncfg"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(db, "dyncfg"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesDynConfig"); - assertThat("incorrect indexes", getIndexes(wsdb, "dyncfg"), is(expectedIndexes)); + createIndexes("indexesDynConfig"); + assertThat("incorrect indexes", getAndNormalizeIndexes(wsdb, "dyncfg"), is(expectedIndexes)); } @Test @@ -357,34 +354,28 @@ public void indexesWorkspaces() throws Exception { new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("ws", 1)) - .append("name", "ws_1") - .append("ns", "MongoStartUpTest.workspaces"), + .append("name", "ws_1"), new Document("v", INDEX_VER) .append("unique", true) .append("sparse", true) .append("key", new Document("name", 1)) - .append("name", "name_1") - .append("ns", "MongoStartUpTest.workspaces"), + .append("name", "name_1"), new Document("v", INDEX_VER) .append("key", new Document("owner", 1)) - .append("name", "owner_1") - .append("ns", "MongoStartUpTest.workspaces"), + .append("name", "owner_1"), new Document("v", INDEX_VER) .append("sparse", true) .append("key", new Document("meta", 1)) - .append("name", "meta_1") - .append("ns", "MongoStartUpTest.workspaces"), + .append("name", "meta_1"), new Document("v", INDEX_VER) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "MongoStartUpTest.workspaces") ); - assertThat("incorrect indexes", getIndexes(db, "workspaces"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(db, "workspaces"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesWorkspaces"); createIndexes("indexesWorkspaces"); - setNamespace(expectedIndexes, "indexesWorkspaces.workspaces"); - assertThat("incorrect indexes", getIndexes(wsdb, "workspaces"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(wsdb, "workspaces"), is(expectedIndexes)); } @Test @@ -393,23 +384,19 @@ public void indexesWorkspaceACLs() throws Exception { new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("id", 1).append("user", 1).append("perm", 1)) - .append("name", "id_1_user_1_perm_1") - .append("ns", "MongoStartUpTest.workspaceACLs"), + .append("name", "id_1_user_1_perm_1"), new Document("v", INDEX_VER) .append("key", new Document("user", 1).append("perm", 1).append("id", 1)) - .append("name", "user_1_perm_1_id_1") - .append("ns", "MongoStartUpTest.workspaceACLs"), + .append("name", "user_1_perm_1_id_1"), new Document("v", INDEX_VER) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "MongoStartUpTest.workspaceACLs") ); - assertThat("incorrect indexes", getIndexes(db, "workspaceACLs"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(db, "workspaceACLs"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesWorkspaceACLs"); createIndexes("indexesWorkspaceACLs"); - setNamespace(expectedIndexes, "indexesWorkspaceACLs.workspaceACLs"); - assertThat("incorrect indexes", getIndexes(wsdb, "workspaceACLs"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(wsdb, "workspaceACLs"), is(expectedIndexes)); } @Test @@ -418,32 +405,26 @@ public void indexesWorkspaceObjects() throws Exception { new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("ws", 1).append("name", 1)) - .append("name", "ws_1_name_1") - .append("ns", "MongoStartUpTest.workspaceObjects"), + .append("name", "ws_1_name_1"), new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("ws", 1).append("id", 1)) - .append("name", "ws_1_id_1") - .append("ns", "MongoStartUpTest.workspaceObjects"), + .append("name", "ws_1_id_1"), new Document("v", INDEX_VER) .append("key", new Document("moddate", 1)) - .append("name", "moddate_1") - .append("ns", "MongoStartUpTest.workspaceObjects"), + .append("name", "moddate_1"), new Document("v", INDEX_VER) .append("key", new Document("del", 1).append("refcnt", 1)) - .append("name", "del_1_refcnt_1") - .append("ns", "MongoStartUpTest.workspaceObjects"), + .append("name", "del_1_refcnt_1"), new Document("v", INDEX_VER) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "MongoStartUpTest.workspaceObjects") ); - assertThat("incorrect indexes", getIndexes(db, "workspaceObjects"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(db, "workspaceObjects"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesWorkspaceObjects"); createIndexes("indexesWorkspaceObjects"); - setNamespace(expectedIndexes, "indexesWorkspaceObjects.workspaceObjects"); - assertThat("incorrect indexes", getIndexes(wsdb, "workspaceObjects"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(wsdb, "workspaceObjects"), is(expectedIndexes)); } @Test @@ -452,68 +433,55 @@ public void indexesWorkspaceObjectVersions() throws Exception { new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("ws", 1).append("id", 1).append("ver", 1)) - .append("name", "ws_1_id_1_ver_1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "ws_1_id_1_ver_1"), new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("ws", 1).append("id", 1).append("ver", -1)) - .append("name", "ws_1_id_1_ver_-1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "ws_1_id_1_ver_-1"), new Document("v", INDEX_VER) .append("key", new Document("tyname", 1) .append("tymaj", 1).append("tymin", 1) .append("ws", 1).append("id", 1).append("ver", -1)) - .append("name", "tyname_1_tymaj_1_tymin_1_ws_1_id_1_ver_-1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "tyname_1_tymaj_1_tymin_1_ws_1_id_1_ver_-1"), new Document("v", INDEX_VER) .append("key", new Document("tyname", 1).append("tymaj", 1) .append("ws", 1).append("id", 1).append("ver", -1)) - .append("name", "tyname_1_tymaj_1_ws_1_id_1_ver_-1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "tyname_1_tymaj_1_ws_1_id_1_ver_-1"), new Document("v", INDEX_VER) .append("key", new Document("tyname", 1) .append("ws", 1).append("id", 1).append("ver", -1)) - .append("name", "tyname_1_ws_1_id_1_ver_-1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "tyname_1_ws_1_id_1_ver_-1"), new Document("v", INDEX_VER) .append("key", new Document("provenance", 1)) - .append("name", "provenance_1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "provenance_1"), new Document("v", INDEX_VER) .append("key", new Document("savedby", 1)) - .append("name", "savedby_1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "savedby_1"), new Document("v", INDEX_VER) .append("sparse", true) .append("key", new Document("provrefs", 1)) - .append("name", "provrefs_1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "provrefs_1"), new Document("v", INDEX_VER) .append("sparse", true) .append("key", new Document("refs", 1)) - .append("name", "refs_1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "refs_1"), new Document("v", INDEX_VER) .append("key", new Document("savedate", 1)) - .append("name", "savedate_1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "savedate_1"), new Document("v", INDEX_VER) .append("sparse", true) .append("key", new Document("meta", 1)) - .append("name", "meta_1") - .append("ns", "MongoStartUpTest.workspaceObjVersions"), + .append("name", "meta_1"), new Document("v", INDEX_VER) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "MongoStartUpTest.workspaceObjVersions") ); - assertThat("incorrect indexes", getIndexes(db, "workspaceObjVersions"), + assertThat("incorrect indexes", getAndNormalizeIndexes(db, "workspaceObjVersions"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesWorkspaceObjectVersions"); createIndexes("indexesWorkspaceObjectVersions"); - setNamespace(expectedIndexes, "indexesWorkspaceObjectVersions.workspaceObjVersions"); - assertThat("incorrect indexes", getIndexes(wsdb, "workspaceObjVersions"), + assertThat("incorrect indexes", getAndNormalizeIndexes(wsdb, "workspaceObjVersions"), is(expectedIndexes)); } @@ -523,19 +491,16 @@ public void indexesAdmins() throws Exception { new Document("v", INDEX_VER) .append("unique", true) .append("key", new Document("user", 1)) - .append("name", "user_1") - .append("ns", "MongoStartUpTest.admins"), + .append("name", "user_1"), new Document("v", INDEX_VER) .append("key", new Document("_id", 1)) .append("name", "_id_") - .append("ns", "MongoStartUpTest.admins") ); - assertThat("incorrect indexes", getIndexes(db, "admins"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(db, "admins"), is(expectedIndexes)); final MongoDatabase wsdb = mongoClient.getDatabase("indexesAdmins"); createIndexes("indexesAdmins"); - setNamespace(expectedIndexes, "indexesAdmins.admins"); - assertThat("incorrect indexes", getIndexes(wsdb, "admins"), is(expectedIndexes)); + assertThat("incorrect indexes", getAndNormalizeIndexes(wsdb, "admins"), is(expectedIndexes)); } } From 850d0850822e6914d8476c911bbbc28a8b69d671 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 20 Feb 2024 10:06:35 -0800 Subject: [PATCH 11/19] fix SchemaUpdaterTest --- .../workspace/test/WorkspaceMongoIndex.java | 25 +++++++++++++++++++ .../test/database/mongo/MongoStartUpTest.java | 16 +----------- .../database/mongo/SchemaUpdaterTest.java | 12 +++------ 3 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 src/us/kbase/workspace/test/WorkspaceMongoIndex.java diff --git a/src/us/kbase/workspace/test/WorkspaceMongoIndex.java b/src/us/kbase/workspace/test/WorkspaceMongoIndex.java new file mode 100644 index 00000000..28b50321 --- /dev/null +++ b/src/us/kbase/workspace/test/WorkspaceMongoIndex.java @@ -0,0 +1,25 @@ +package us.kbase.workspace.test; + +import com.mongodb.client.MongoDatabase; +import org.bson.Document; + +import java.util.HashSet; +import java.util.Set; + +public class WorkspaceMongoIndex { + + public static Set getAndNormalizeIndexes(final MongoDatabase db, final String collectionName) { + final Set indexes = new HashSet<>(); + for (Document index: db.getCollection(collectionName).listIndexes()) { + // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getAndNormalizeIndexes() + // no longer returns the namespace ns field in the index specification documents. + index.remove("ns"); + // some versions of Mongo return ints, some longs. Convert all to longs. + if (index.containsKey("expireAfterSeconds")) { + index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue()); + } + indexes.add(index); + } + return indexes; + } +} diff --git a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java index 71e647e4..eb038694 100644 --- a/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java +++ b/src/us/kbase/workspace/test/database/mongo/MongoStartUpTest.java @@ -5,6 +5,7 @@ import static org.junit.Assert.fail; import static us.kbase.common.test.TestCommon.assertExceptionCorrect; import static us.kbase.common.test.TestCommon.set; +import static us.kbase.workspace.test.WorkspaceMongoIndex.getAndNormalizeIndexes; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -297,21 +298,6 @@ private void createIndexes(final String dbname) throws Exception { method.invoke(null, wsdb); } - private Set getAndNormalizeIndexes(final MongoDatabase db, final String collectionName) { - final Set indexes = new HashSet<>(); - for (Document index: db.getCollection(collectionName).listIndexes()) { - // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getAndNormalizeIndexes() - // no longer returns the namespace ns field in the index specification documents. - index.remove("ns"); - // some versions of Mongo return ints, some longs. Convert all to longs. - if (index.containsKey("expireAfterSeconds")) { - index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue()); - } - indexes.add(index); - } - return indexes; - } - @Test public void indexesConfig() throws Exception { final Set expectedIndexes = set( diff --git a/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java b/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java index 3d84d984..5fec5968 100644 --- a/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java +++ b/src/us/kbase/workspace/test/database/mongo/SchemaUpdaterTest.java @@ -5,6 +5,7 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.when; import static us.kbase.workspace.test.WorkspaceTestCommon.basicProv; +import static us.kbase.workspace.test.WorkspaceMongoIndex.getAndNormalizeIndexes; import java.nio.file.Paths; import java.time.Instant; @@ -256,23 +257,18 @@ private void assertIndexesCreated(final MongoDatabase db, final boolean oldTypeI * MongoWorkspaceDB method is already thoroughly tested elsewhere we just check a single * index. */ - final String ns = db.getName() + "." + COL_WS_VER; final Document currentIndex = new Document("v", 2) .append("key", new Document("tyname", 1) .append("tymaj", 1).append("tymin", 1) .append("ws", 1).append("id", 1).append("ver", -1)) - .append("name", "tyname_1_tymaj_1_tymin_1_ws_1_id_1_ver_-1") - .append("ns", ns); - final Set indexes = new HashSet<>(); - db.getCollection("workspaceObjVersions").listIndexes() - .forEach((Consumer) indexes::add); + .append("name", "tyname_1_tymaj_1_tymin_1_ws_1_id_1_ver_-1"); + final Set indexes = getAndNormalizeIndexes(db, "workspaceObjVersions"); assertThat("incorrect current index", indexes.contains(currentIndex), is(true)); if (oldTypeIndex) { final Document typeIndex = new Document("v", 2) .append("key", new Document("type", 1).append("chksum", 1)) - .append("name", "type_1_chksum_1") - .append("ns", ns); + .append("name", "type_1_chksum_1"); assertThat("incorrect old index", indexes.contains(typeIndex), is(true)); } } From 6a837c7c7321a7235cb1e8fdc27a734112bf2cc7 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 20 Feb 2024 11:10:42 -0800 Subject: [PATCH 12/19] fix HandleAndBytestreamIntegrationTest.java --- .../test/kbase/HandleAndBytestreamIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java b/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java index 02298bda..6b7f2be7 100644 --- a/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java +++ b/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java @@ -147,7 +147,7 @@ public static void setUpClass() throws Exception { final String dbname = HandleAndBytestreamIntegrationTest.class.getSimpleName() + "Auth"; AUTH = new AuthController( TestCommon.getJarsDir(), - "localhost:" + MONGO.getServerPort(), + "mongodb://localhost:" + MONGO.getServerPort(), dbname, Paths.get(TestCommon.getTempDir())); final URL authURL = new URL("http://localhost:" + AUTH.getServerPort() + "/testmode"); From 8ff6f83bc7d0f93482d26c07f196953366a47805 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 20 Feb 2024 11:28:27 -0800 Subject: [PATCH 13/19] revert back --- .../test/kbase/HandleAndBytestreamIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java b/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java index 6b7f2be7..02298bda 100644 --- a/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java +++ b/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java @@ -147,7 +147,7 @@ public static void setUpClass() throws Exception { final String dbname = HandleAndBytestreamIntegrationTest.class.getSimpleName() + "Auth"; AUTH = new AuthController( TestCommon.getJarsDir(), - "mongodb://localhost:" + MONGO.getServerPort(), + "localhost:" + MONGO.getServerPort(), dbname, Paths.get(TestCommon.getTempDir())); final URL authURL = new URL("http://localhost:" + AUTH.getServerPort() + "/testmode"); From 642ed48f8565fe7b99f3a1f55ca01f72e02e7877 Mon Sep 17 00:00:00 2001 From: Sijie Date: Tue, 20 Feb 2024 15:48:38 -0800 Subject: [PATCH 14/19] test kbase_auth2test_0.6.1 jar --- .classpath | 2 +- .github/workflows/test.yml | 2 +- build.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.classpath b/.classpath index 7a0894eb..1c988706 100644 --- a/.classpath +++ b/.classpath @@ -52,7 +52,7 @@ - + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28dbfa5f..8c3fcfc4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,7 +91,7 @@ jobs: cd .. # set up jars - git clone https://github.com/kbase/jars + git clone -b dev-kbase_auth2test_0.6.1 https://github.com/kbase/jars export JARSDIR=$(pwd)/jars/lib/jars/ # set up arango diff --git a/build.xml b/build.xml index 24d71022..c02f3f7e 100644 --- a/build.xml +++ b/build.xml @@ -104,7 +104,7 @@ - + From 7ccbc50ce7e364c8035d24ca8e265005805a5aa6 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 21 Feb 2024 09:57:26 -0800 Subject: [PATCH 15/19] update controller --- .classpath | 2 +- build.xml | 2 +- .../test/kbase/HandleAndBytestreamIntegrationTest.java | 1 - src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java | 1 - src/us/kbase/workspace/test/kbase/LoggingTest.java | 1 - .../workspace/test/kbase/SampleServiceIntegrationTest.java | 1 - src/us/kbase/workspace/test/kbase/TypeDelegationTest.java | 1 - 7 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.classpath b/.classpath index 1c988706..e800d861 100644 --- a/.classpath +++ b/.classpath @@ -52,7 +52,7 @@ - + diff --git a/build.xml b/build.xml index c02f3f7e..75ae7264 100644 --- a/build.xml +++ b/build.xml @@ -104,7 +104,7 @@ - + diff --git a/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java b/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java index 02298bda..3a201eb3 100644 --- a/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java +++ b/src/us/kbase/workspace/test/kbase/HandleAndBytestreamIntegrationTest.java @@ -146,7 +146,6 @@ public static void setUpClass() throws Exception { // set up auth final String dbname = HandleAndBytestreamIntegrationTest.class.getSimpleName() + "Auth"; AUTH = new AuthController( - TestCommon.getJarsDir(), "localhost:" + MONGO.getServerPort(), dbname, Paths.get(TestCommon.getTempDir())); diff --git a/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java b/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java index 67ef7567..c6c0ef84 100644 --- a/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java +++ b/src/us/kbase/workspace/test/kbase/JSONRPCLayerTester.java @@ -188,7 +188,6 @@ public static void setUpClass() throws Exception { // set up auth final String dbname = JSONRPCLayerTester.class.getSimpleName() + "Auth"; authc = new AuthController( - TestCommon.getJarsDir(), "localhost:" + mongo.getServerPort(), dbname, Paths.get(TestCommon.getTempDir())); diff --git a/src/us/kbase/workspace/test/kbase/LoggingTest.java b/src/us/kbase/workspace/test/kbase/LoggingTest.java index bd6d45bf..fe5bbaa3 100644 --- a/src/us/kbase/workspace/test/kbase/LoggingTest.java +++ b/src/us/kbase/workspace/test/kbase/LoggingTest.java @@ -101,7 +101,6 @@ public static void setUpClass() throws Exception { // set up auth final String dbname = LoggingTest.class.getSimpleName() + "Auth"; authc = new AuthController( - TestCommon.getJarsDir(), "localhost:" + mongo.getServerPort(), dbname, Paths.get(TestCommon.getTempDir())); diff --git a/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java b/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java index 5569cfbf..aa3f27ea 100644 --- a/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java +++ b/src/us/kbase/workspace/test/kbase/SampleServiceIntegrationTest.java @@ -125,7 +125,6 @@ public static void setUpClass() throws Exception { // set up auth final String dbname = SampleServiceIntegrationTest.class.getSimpleName() + "Auth"; AUTH = new AuthController( - TestCommon.getJarsDir(), "localhost:" + MONGO.getServerPort(), dbname, Paths.get(TestCommon.getTempDir())); diff --git a/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java b/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java index 8ef1ab72..4f38b53e 100644 --- a/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java +++ b/src/us/kbase/workspace/test/kbase/TypeDelegationTest.java @@ -149,7 +149,6 @@ public static void setUp() throws Exception { mongohost, MONGO.getTempDir())); AUTHC = new AuthController( - TestCommon.getJarsDir(), mongohost, CLS + "Auth", Paths.get(TestCommon.getTempDir())); From 6ccee2e7da2cd1b55b5a09e6d969459ca6f518a9 Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 21 Feb 2024 10:15:16 -0800 Subject: [PATCH 16/19] update docker file --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3c6a15f2..2b9ff3dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ COPY . /tmp/workspace_deluxe WORKDIR /tmp RUN apt-get update -y && \ apt-get install -y ant git ca-certificates python3-sphinx && \ - git clone https://github.com/kbase/jars && \ + git clone -b dev-kbase_auth2test_0.6.1 https://github.com/kbase/jars && \ cd workspace_deluxe && \ make docker_deps From 640953d37a909e1ba8c802a0b24574028afc5a3b Mon Sep 17 00:00:00 2001 From: Sijie Date: Wed, 21 Feb 2024 17:58:41 -0800 Subject: [PATCH 17/19] rename fat auth2 jar && clean up --- .classpath | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 2 +- build.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.classpath b/.classpath index e800d861..1e8fddb8 100644 --- a/.classpath +++ b/.classpath @@ -52,7 +52,7 @@ - + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c3fcfc4..28dbfa5f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,7 +91,7 @@ jobs: cd .. # set up jars - git clone -b dev-kbase_auth2test_0.6.1 https://github.com/kbase/jars + git clone https://github.com/kbase/jars export JARSDIR=$(pwd)/jars/lib/jars/ # set up arango diff --git a/Dockerfile b/Dockerfile index 2b9ff3dc..3c6a15f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ COPY . /tmp/workspace_deluxe WORKDIR /tmp RUN apt-get update -y && \ apt-get install -y ant git ca-certificates python3-sphinx && \ - git clone -b dev-kbase_auth2test_0.6.1 https://github.com/kbase/jars && \ + git clone https://github.com/kbase/jars && \ cd workspace_deluxe && \ make docker_deps diff --git a/build.xml b/build.xml index 75ae7264..c713ac80 100644 --- a/build.xml +++ b/build.xml @@ -104,7 +104,7 @@ - + From 11521dcbcb7c658cb15aa6a1ea5417b12c655f38 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 22 Feb 2024 15:24:07 -0800 Subject: [PATCH 18/19] update test.yml, build.xml, and WorkspaceMongoIndex.java --- .github/workflows/test.yml | 8 +------- build.xml | 3 ++- src/us/kbase/workspace/test/WorkspaceMongoIndex.java | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28dbfa5f..aab241b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: matrix: include: - java: '11' - mongo: 'mongodb-linux-x86_64-3.6.23' + mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' minio: '2019-05-23T00-29-34Z' wired_tiger: 'true' ant_test: 'test_quick_coverage' @@ -56,12 +56,6 @@ jobs: minio: '2019-05-23T00-29-34Z' wired_tiger: 'false' ant_test: 'test_quick_coverage' - # Mongo7 setup - - java: '11' - mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4' - minio: '2019-05-23T00-29-34Z' - wired_tiger: 'false' - ant_test: 'test_quick_coverage' steps: - uses: actions/checkout@v3 diff --git a/build.xml b/build.xml index c713ac80..5b3af13f 100644 --- a/build.xml +++ b/build.xml @@ -104,7 +104,6 @@ - @@ -112,6 +111,8 @@ + + diff --git a/src/us/kbase/workspace/test/WorkspaceMongoIndex.java b/src/us/kbase/workspace/test/WorkspaceMongoIndex.java index 28b50321..78959c14 100644 --- a/src/us/kbase/workspace/test/WorkspaceMongoIndex.java +++ b/src/us/kbase/workspace/test/WorkspaceMongoIndex.java @@ -10,8 +10,8 @@ public class WorkspaceMongoIndex { public static Set getAndNormalizeIndexes(final MongoDatabase db, final String collectionName) { final Set indexes = new HashSet<>(); - for (Document index: db.getCollection(collectionName).listIndexes()) { - // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getAndNormalizeIndexes() + for (final Document index: db.getCollection(collectionName).listIndexes()) { + // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes() // no longer returns the namespace ns field in the index specification documents. index.remove("ns"); // some versions of Mongo return ints, some longs. Convert all to longs. From 58fb9e56c4f82381edb53d10183f693d6bd57e25 Mon Sep 17 00:00:00 2001 From: Sijie Date: Thu, 22 Feb 2024 16:23:26 -0800 Subject: [PATCH 19/19] add release notes & a version bump to 0.14.3 --- docsource/conf.py | 2 +- docsource/releasenotes.rst | 8 ++++++++ scripts/workspace_container_test.py | 2 +- src/us/kbase/workspace/test/kbase/JSONRPCLayerTest.java | 2 +- .../kbase/workspace/test/kbase/SchemaUpdaterCLITest.java | 2 +- src/us/kbase/workspace/version/WorkspaceVersion.java | 2 +- 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docsource/conf.py b/docsource/conf.py index d2076633..62b8d108 100644 --- a/docsource/conf.py +++ b/docsource/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '0.14' # The full version, including alpha/beta/rc tags. -release = '0.14.2' +release = '0.14.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docsource/releasenotes.rst b/docsource/releasenotes.rst index 5dc51ea7..6d67c630 100644 --- a/docsource/releasenotes.rst +++ b/docsource/releasenotes.rst @@ -3,6 +3,14 @@ Workspace service release notes =============================== +VERSION: 0.14.3 (Released 2/22/2024) +------------------------------------ + +UPDATES: + +* The MongoDB clients have been updated to the most recent version and the service tested against Mongo 7. + + VERSION: 0.14.2 (Released 11/9/2023) ------------------------------------ diff --git a/scripts/workspace_container_test.py b/scripts/workspace_container_test.py index fb3d53be..f2b45847 100644 --- a/scripts/workspace_container_test.py +++ b/scripts/workspace_container_test.py @@ -16,7 +16,7 @@ """ -WORKSPACE_VERSION = "0.14.2" +WORKSPACE_VERSION = "0.14.3" AUTH_URL = "http://localhost:8080" WS_URL = "http://localhost:7058" diff --git a/src/us/kbase/workspace/test/kbase/JSONRPCLayerTest.java b/src/us/kbase/workspace/test/kbase/JSONRPCLayerTest.java index b2226d64..a47c90fc 100644 --- a/src/us/kbase/workspace/test/kbase/JSONRPCLayerTest.java +++ b/src/us/kbase/workspace/test/kbase/JSONRPCLayerTest.java @@ -101,7 +101,7 @@ */ public class JSONRPCLayerTest extends JSONRPCLayerTester { - private static final String VER = "0.14.2"; + private static final String VER = "0.14.3"; @Test public void ver() throws Exception { diff --git a/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java b/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java index 7c94f5d3..e84f67df 100644 --- a/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java +++ b/src/us/kbase/workspace/test/kbase/SchemaUpdaterCLITest.java @@ -41,7 +41,7 @@ public class SchemaUpdaterCLITest { - private static final String VERSION = "0.14.2"; + private static final String VERSION = "0.14.3"; private static final List HELP = list( "Usage: update_workspace_database_schema [-chosV] ", diff --git a/src/us/kbase/workspace/version/WorkspaceVersion.java b/src/us/kbase/workspace/version/WorkspaceVersion.java index f21f4096..2d7947e4 100644 --- a/src/us/kbase/workspace/version/WorkspaceVersion.java +++ b/src/us/kbase/workspace/version/WorkspaceVersion.java @@ -6,6 +6,6 @@ public class WorkspaceVersion { private WorkspaceVersion() {}; /** The version. */ - public static final String VERSION = "0.14.2"; + public static final String VERSION = "0.14.3"; }