diff --git a/.cloudbuild/graalvm/cloudbuild-test-a.yaml b/.cloudbuild/graalvm/cloudbuild-test-a.yaml index 01b99f820e..8c829b592b 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-a.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-a.yaml @@ -14,8 +14,8 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.25.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.7.1' + _SHARED_DEPENDENCIES_VERSION: '3.27.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} + _JAVA_SHARED_CONFIG_VERSION: '1.7.4' steps: # GraalVM A build diff --git a/.cloudbuild/graalvm/cloudbuild-test-b.yaml b/.cloudbuild/graalvm/cloudbuild-test-b.yaml index 97d8256ebd..b223279693 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-b.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-b.yaml @@ -14,8 +14,8 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.25.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.7.1' + _SHARED_DEPENDENCIES_VERSION: '3.27.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} + _JAVA_SHARED_CONFIG_VERSION: '1.7.4' steps: # GraalVM A build diff --git a/.cloudbuild/graalvm/cloudbuild.yaml b/.cloudbuild/graalvm/cloudbuild.yaml index d0b0589bb9..e4e3ed9048 100644 --- a/.cloudbuild/graalvm/cloudbuild.yaml +++ b/.cloudbuild/graalvm/cloudbuild.yaml @@ -14,8 +14,8 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.25.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.7.1' + _SHARED_DEPENDENCIES_VERSION: '3.27.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} + _JAVA_SHARED_CONFIG_VERSION: '1.7.4' steps: # GraalVM A build - name: gcr.io/cloud-builders/docker diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-release.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-release.yaml index 814bd001ea..c5be471583 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-release.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-release.yaml @@ -14,7 +14,7 @@ timeout: 7200s # 2 hours substitutions: - _GAPIC_GENERATOR_JAVA_VERSION: '2.35.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current} + _GAPIC_GENERATOR_JAVA_VERSION: '2.37.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current} _IMAGE_ID: "gcr.io/cloud-devrel-public-resources/java-library-generation:${_GAPIC_GENERATOR_JAVA_VERSION}" steps: # Library generation build diff --git a/.github/release-please.yml b/.github/release-please.yml index c964bc6236..c335a3f0aa 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -3,4 +3,4 @@ bumpMinorPreMajor: true handleGHRelease: true primaryBranch: main manifest: true -extraFiles: ["WORKSPACE", ".cloudbuild/cloudbuild.yaml", ".cloudbuild/cloudbuild-test-a.yaml", ".cloudbuild/cloudbuild-test-b.yaml" ] +extraFiles: ["WORKSPACE", ".cloudbuild/graalvm/cloudbuild.yaml", ".cloudbuild/graalvm/cloudbuild-test-a.yaml", ".cloudbuild/graalvm/cloudbuild-test-b.yaml", ".cloudbuild/library_generation/cloudbuild-library-generation-release.yaml" ] diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5344f99886..1de5e7d9bb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,6 +22,14 @@ jobs: run: | mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \ -Dfmt.skip -DenableTestCoverage + # The `envVarTest` profile runs tests that require an environment variable + - name: Env Var Tests + run: | + mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \ + -Dfmt.skip -DenableTestCoverage -PenvVarTest + # Set the Env Var for this step only + env: + GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com - run: bazelisk version - name: Install Maven modules run: | @@ -64,6 +72,14 @@ jobs: run: | mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \ -Dfmt.skip -DenableTestCoverage + # The `envVarTest` profile runs tests that require an environment variable + - name: Env Var Tests + run: | + mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \ + -Dfmt.skip -DenableTestCoverage -PenvVarTest + # Set the Env Var for this step only + env: + GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com - run: bazelisk version - name: Install Maven modules run: | @@ -98,6 +114,22 @@ jobs: mvn verify --batch-mode --no-transfer-progress -Dcheckstyle.skip \ -Dfmt.skip \ -Djvm="${JAVA8_HOME}/bin/java" + # The `envVarTest` profile runs tests that require an environment variable + - name: Compile with Java 17 and run tests with Java 8 (Env Var Tests) + shell: bash + run: | + set -x + export JAVA_HOME=$JAVA_HOME + export PATH=${JAVA_HOME}/bin:$PATH + # Maven surefire plugin lets us to specify the JVM when running tests via + # the "jvm" system property. + export GOOGLE_CLOUD_UNIVERSE_DOMAIN=random.com + mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \ + -Dfmt.skip -DenableTestCoverage -Dsurefire.failIfNoSpecifiedTests=false \ + -PenvVarTest + # Set the Env Var for this step only + env: + GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com build-java8-gapic-generator-java: name: "build(8) for gapic-generator-java" diff --git a/.github/workflows/sonar.yaml b/.github/workflows/sonar.yaml index 013ff88b36..585893942e 100644 --- a/.github/workflows/sonar.yaml +++ b/.github/workflows/sonar.yaml @@ -47,6 +47,10 @@ jobs: tar -xf showcase-* ./gapic-showcase run & cd - + # Intentionally do not run the Env Var Tests (no -PenvVarTests) as setting the Env Var + # may alter the results for other tests that use Env Var in the logic. Adding a Sonar + # step for a few tests (env var tests) may be overkill and should be better covered + # when we can upgrade to JUnit 5 (https://github.com/googleapis/sdk-platform-java/issues/1611#issuecomment-1970079325) - name: Build and analyze for full test coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any @@ -59,7 +63,6 @@ jobs: -Dsonar.projectKey=googleapis_gapic-generator-java \ -Dsonar.organization=googleapis \ -Dsonar.host.url=https://sonarcloud.io - - name: Build and analyze Showcase Integration Tests Coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any diff --git a/.gitignore b/.gitignore index 5a54c06a20..201526885b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ target/ **/output/ **/googleapis library_generation/test/**/golden*/ -library_generation/test/resources/test_repo_level_postprocess/ +library_generation/test/resources/test_monorepo_postprocessing/ **/*egg-info/ **/build/ +**/dist/ diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 517b0cf98a..28c811f943 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.35.0" + ".": "2.37.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index a034a206c1..7de5bc6816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Changelog +## [2.37.0](https://github.com/googleapis/sdk-platform-java/compare/v2.36.0...v2.37.0) (2024-02-29) + + +### Features + +* improve batching summary errors ([#2509](https://github.com/googleapis/sdk-platform-java/issues/2509)) ([d6964a4](https://github.com/googleapis/sdk-platform-java/commit/d6964a4cfda60ef7c5f1871b215a9cf222361e7e)) + + +### Bug Fixes + +* adjust release please config to reflect file location change. ([#2524](https://github.com/googleapis/sdk-platform-java/issues/2524)) ([1e5af1e](https://github.com/googleapis/sdk-platform-java/commit/1e5af1e651a079220b8a63952b7dbc68586d9b7d)) + +## [2.36.0](https://github.com/googleapis/sdk-platform-java/compare/v2.35.0...v2.36.0) (2024-02-29) + + +### Features + +* check library_name is unique among libraries ([#2490](https://github.com/googleapis/sdk-platform-java/issues/2490)) ([8123f0b](https://github.com/googleapis/sdk-platform-java/commit/8123f0b15970c76b08086d8988de14e1ed58a77e)) + + +### Bug Fixes + +* cleanup @BetaApi from Resource Name Builder Methods ([#2450](https://github.com/googleapis/sdk-platform-java/issues/2450)) ([6e8d098](https://github.com/googleapis/sdk-platform-java/commit/6e8d09845610401665556118cd78c7229c46d426)), closes [#2099](https://github.com/googleapis/sdk-platform-java/issues/2099) +* Fix watchdog to start with WAITING state ([#2468](https://github.com/googleapis/sdk-platform-java/issues/2468)) ([dedc40f](https://github.com/googleapis/sdk-platform-java/commit/dedc40fb451633b11be10013b74a91053e155620)) +* ignore comment in BUILD ([#2492](https://github.com/googleapis/sdk-platform-java/issues/2492)) ([6ca20e5](https://github.com/googleapis/sdk-platform-java/commit/6ca20e529a86561cdc935d0d44dd971de3b9e4b2)) +* remove @BetaApi from ApiFutures and ApiService ([#2454](https://github.com/googleapis/sdk-platform-java/issues/2454)) ([f59e717](https://github.com/googleapis/sdk-platform-java/commit/f59e717402d00bc953648e1e780dd84dda7915b3)), closes [#2098](https://github.com/googleapis/sdk-platform-java/issues/2098) + + +### Dependencies + +* grandfathering the dependencies for java-pubsublite and java-bigquery ([#2504](https://github.com/googleapis/sdk-platform-java/issues/2504)) ([9ceab23](https://github.com/googleapis/sdk-platform-java/commit/9ceab23144ba7a9ca9114d585376e09587fd1bee)) +* update dependency gradle to v7.6.4 ([#2474](https://github.com/googleapis/sdk-platform-java/issues/2474)) ([607dc59](https://github.com/googleapis/sdk-platform-java/commit/607dc59f8eb5a70f8807c4d5f7d0745438842381)) +* update dependency org.graalvm.sdk:graal-sdk to v22.3.5 ([#2475](https://github.com/googleapis/sdk-platform-java/issues/2475)) ([2de487b](https://github.com/googleapis/sdk-platform-java/commit/2de487b34c43fee3b81e4af7a2dfe4f2eb6418ac)) +* update grpc dependencies to v1.62.2 ([#2506](https://github.com/googleapis/sdk-platform-java/issues/2506)) ([f438603](https://github.com/googleapis/sdk-platform-java/commit/f4386038766c2381e141d7073959cc6bde896efb)) + + +### Documentation + +* Add contribution guidelines. ([#2045](https://github.com/googleapis/sdk-platform-java/issues/2045)) ([9939b43](https://github.com/googleapis/sdk-platform-java/commit/9939b43dce041c75405cea66a0dde171d991d3eb)) + ## [2.35.0](https://github.com/googleapis/sdk-platform-java/compare/v2.34.0...v2.35.0) (2024-02-13) diff --git a/WORKSPACE b/WORKSPACE index 9d178ad9a8..87c270e6aa 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -60,7 +60,7 @@ maven_install( repositories = ["https://repo.maven.apache.org/maven2/"], ) -_gapic_generator_java_version = "2.35.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} +_gapic_generator_java_version = "2.37.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} maven_install( artifacts = [ diff --git a/api-common-java/pom.xml b/api-common-java/pom.xml index 1cee861f26..2235d47560 100644 --- a/api-common-java/pom.xml +++ b/api-common-java/pom.xml @@ -5,14 +5,14 @@ com.google.api api-common jar - 2.26.1-SNAPSHOT + 2.28.1-SNAPSHOT API Common Common utilities for Google APIs in Java com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent diff --git a/api-common-java/src/main/java/com/google/api/core/ApiFutures.java b/api-common-java/src/main/java/com/google/api/core/ApiFutures.java index 4c263ce845..780c50a783 100644 --- a/api-common-java/src/main/java/com/google/api/core/ApiFutures.java +++ b/api-common-java/src/main/java/com/google/api/core/ApiFutures.java @@ -161,7 +161,6 @@ public static ApiFuture catching( * callback} * @see Futures#catchingAsync(ListenableFuture, Class, AsyncFunction, Executor) */ - @BetaApi public static ApiFuture catchingAsync( ApiFuture input, Class exceptionType, diff --git a/api-common-java/src/main/java/com/google/api/core/ApiService.java b/api-common-java/src/main/java/com/google/api/core/ApiService.java index ee49e78f3c..cd3545d8b9 100644 --- a/api-common-java/src/main/java/com/google/api/core/ApiService.java +++ b/api-common-java/src/main/java/com/google/api/core/ApiService.java @@ -165,7 +165,6 @@ public interface ApiService { * {@code A.compareTo(B) < 0} then there is not guaranteed to be a valid state transition * {@code A -> B}. */ - @BetaApi enum State { /** * A service in this state has encountered a problem and may not be operational. It cannot be @@ -198,7 +197,6 @@ enum State { * *

All methods are no-ops by default, implementors should override the ones they care about. */ - @BetaApi abstract class Listener { /** * Called when the service transitions to the {@linkplain State#FAILED FAILED} state. The diff --git a/coverage-report/pom.xml b/coverage-report/pom.xml index 72b712ed06..d27157fcc0 100644 --- a/coverage-report/pom.xml +++ b/coverage-report/pom.xml @@ -31,22 +31,22 @@ com.google.api gax - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT com.google.api gax-grpc - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT com.google.api gax-httpjson - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT com.google.api api-common - 2.26.1-SNAPSHOT + 2.28.1-SNAPSHOT diff --git a/gapic-generator-java-bom/pom.xml b/gapic-generator-java-bom/pom.xml index e0f21fb910..026e0b016b 100644 --- a/gapic-generator-java-bom/pom.xml +++ b/gapic-generator-java-bom/pom.xml @@ -4,7 +4,7 @@ com.google.api gapic-generator-java-bom pom - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT GAPIC Generator Java BOM BOM for the libraries in gapic-generator-java repository. Users should not @@ -15,7 +15,7 @@ com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -75,61 +75,61 @@ com.google.api api-common - 2.26.1-SNAPSHOT + 2.28.1-SNAPSHOT com.google.api gax-bom - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT pom import com.google.api gapic-generator-java - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT com.google.api.grpc grpc-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT com.google.api.grpc proto-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT com.google.api.grpc proto-google-iam-v1 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc proto-google-iam-v2 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc proto-google-iam-v2beta - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v1 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2beta - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT diff --git a/gapic-generator-java-pom-parent/pom.xml b/gapic-generator-java-pom-parent/pom.xml index 59d6c8121f..d4529348c7 100644 --- a/gapic-generator-java-pom-parent/pom.xml +++ b/gapic-generator-java-pom-parent/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT pom GAPIC Generator Java POM Parent https://github.com/googleapis/sdk-platform-java @@ -15,7 +15,7 @@ com.google.cloud google-cloud-shared-config - 1.7.1 + 1.7.4 @@ -177,6 +177,23 @@ + + envVarTest + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/*.java + + + + + + diff --git a/gapic-generator-java/pom.xml b/gapic-generator-java/pom.xml index 757862fa37..82c0b545cb 100644 --- a/gapic-generator-java/pom.xml +++ b/gapic-generator-java/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.google.api gapic-generator-java - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT GAPIC Generator Java GAPIC generator Java @@ -22,7 +22,7 @@ com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -31,7 +31,7 @@ com.google.api gapic-generator-java-bom - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT pom import diff --git a/gax-java/README.md b/gax-java/README.md index 4414ab4722..92b910cc8c 100644 --- a/gax-java/README.md +++ b/gax-java/README.md @@ -34,27 +34,27 @@ If you are using Maven, add this to your pom.xml file com.google.api gax - 2.43.0 + 2.45.0 com.google.api gax-grpc - 2.43.0 + 2.45.0 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.api:gax:2.43.0', - 'com.google.api:gax-grpc:2.43.0' +compile 'com.google.api:gax:2.45.0', + 'com.google.api:gax-grpc:2.45.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.api" % "gax" % "2.43.0" -libraryDependencies += "com.google.api" % "gax-grpc" % "2.43.0" +libraryDependencies += "com.google.api" % "gax" % "2.45.0" +libraryDependencies += "com.google.api" % "gax-grpc" % "2.45.0" ``` [//]: # ({x-version-update-end}) diff --git a/gax-java/dependencies.properties b/gax-java/dependencies.properties index 8b57ebcab7..0438c2a50d 100644 --- a/gax-java/dependencies.properties +++ b/gax-java/dependencies.properties @@ -8,16 +8,16 @@ # Versions of oneself # {x-version-update-start:gax:current} -version.gax=2.43.1-SNAPSHOT +version.gax=2.45.1-SNAPSHOT # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_grpc=2.43.1-SNAPSHOT +version.gax_grpc=2.45.1-SNAPSHOT # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_bom=2.43.1-SNAPSHOT +version.gax_bom=2.45.1-SNAPSHOT # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_httpjson=2.43.1-SNAPSHOT +version.gax_httpjson=2.45.1-SNAPSHOT # {x-version-update-end} # Versions for dependencies which actual artifacts differ between Bazel and Gradle. @@ -42,18 +42,18 @@ maven.com_google_auth_google_auth_library_credentials=com.google.auth:google-aut maven.io_opencensus_opencensus_api=io.opencensus:opencensus-api:0.31.1 maven.io_opencensus_opencensus_contrib_grpc_metrics=io.opencensus:opencensus-contrib-grpc-metrics:0.31.1 maven.io_opencensus_opencensus_contrib_http_util=io.opencensus:opencensus-contrib-http-util:0.31.1 -maven.io_netty_netty_tcnative_boringssl_static=io.netty:netty-tcnative-boringssl-static:2.0.62.Final -maven.io_netty_netty_handler=io.netty:netty-handler:4.1.105.Final -maven.io_netty_netty_common=io.netty:netty-common:4.1.105.Final -maven.io_netty_netty_codec_socks=io.netty:netty-codec-socks:4.1.105.Final -maven.io_netty_netty_codec_http2=io.netty:netty-codec-http2:4.1.105.Final -maven.io_netty_netty_codec_http=io.netty:netty-codec-http:4.1.105.Final -maven.io_netty_netty_codec=io.netty:netty-codec:4.1.105.Final -maven.io_netty_netty_buffer=io.netty:netty-buffer:4.1.105.Final -maven.io_netty_netty_resolver=io.netty:netty-resolver:4.1.105.Final -maven.io_netty_netty_transport=io.netty:netty-transport:4.1.105.Final -maven.io_netty_netty_handler_proxy=io.netty:netty-handler-proxy:4.1.105.Final -maven.io_netty_netty_transport_native_unix_common=io.netty:netty-transport-native-unix-common:4.1.105.Final +maven.io_netty_netty_tcnative_boringssl_static=io.netty:netty-tcnative-boringssl-static:2.0.63.Final +maven.io_netty_netty_handler=io.netty:netty-handler:4.1.107.Final +maven.io_netty_netty_common=io.netty:netty-common:4.1.107.Final +maven.io_netty_netty_codec_socks=io.netty:netty-codec-socks:4.1.107.Final +maven.io_netty_netty_codec_http2=io.netty:netty-codec-http2:4.1.107.Final +maven.io_netty_netty_codec_http=io.netty:netty-codec-http:4.1.107.Final +maven.io_netty_netty_codec=io.netty:netty-codec:4.1.107.Final +maven.io_netty_netty_buffer=io.netty:netty-buffer:4.1.107.Final +maven.io_netty_netty_resolver=io.netty:netty-resolver:4.1.107.Final +maven.io_netty_netty_transport=io.netty:netty-transport:4.1.107.Final +maven.io_netty_netty_handler_proxy=io.netty:netty-handler-proxy:4.1.107.Final +maven.io_netty_netty_transport_native_unix_common=io.netty:netty-transport-native-unix-common:4.1.107.Final maven.io_perfmark_perfmark_api=io.perfmark:perfmark-api:0.27.0 maven.org_apache_tomcat_annotations_api=org.apache.tomcat:annotations-api:6.0.53 maven.com_google_code_gson_gson=com.google.code.gson:gson:2.10.1 diff --git a/gax-java/gax-bom/pom.xml b/gax-java/gax-bom/pom.xml index fad06ff2cf..69a849f8fb 100644 --- a/gax-java/gax-bom/pom.xml +++ b/gax-java/gax-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.api gax-bom - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT pom GAX (Google Api eXtensions) for Java (BOM) Google Api eXtensions for Java (BOM) @@ -11,7 +11,7 @@ com.google.cloud google-cloud-shared-config - 1.7.1 + 1.7.4 @@ -43,55 +43,55 @@ com.google.api gax - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT com.google.api gax - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT test-jar testlib com.google.api gax - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT testlib com.google.api gax-grpc - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT com.google.api gax-grpc - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT test-jar testlib com.google.api gax-grpc - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT testlib com.google.api gax-httpjson - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT com.google.api gax-httpjson - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT test-jar testlib com.google.api gax-httpjson - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT testlib diff --git a/gax-java/gax-grpc/pom.xml b/gax-java/gax-grpc/pom.xml index cabf96e9bf..c66b645a86 100644 --- a/gax-java/gax-grpc/pom.xml +++ b/gax-java/gax-grpc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax-grpc - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT jar GAX (Google Api eXtensions) for Java (gRPC) Google Api eXtensions for Java (gRPC) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT diff --git a/gax-java/gax-httpjson/pom.xml b/gax-java/gax-httpjson/pom.xml index b4504dcfcb..120ab765d7 100644 --- a/gax-java/gax-httpjson/pom.xml +++ b/gax-java/gax-httpjson/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax-httpjson - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT jar GAX (Google Api eXtensions) for Java (HTTP JSON) Google Api eXtensions for Java (HTTP JSON) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT diff --git a/gax-java/gax/pom.xml b/gax-java/gax/pom.xml index 57fb6f5799..e56ebadb17 100644 --- a/gax-java/gax/pom.xml +++ b/gax-java/gax/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT jar GAX (Google Api eXtensions) for Java (Core) Google Api eXtensions for Java (Core) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT @@ -99,8 +99,27 @@ maven-surefire-plugin -Djava.util.logging.SimpleFormatter.format="%1$tY %1$tl:%1$tM:%1$tS.%1$tL %2$s %4$s: %5$s%6$s%n" + + !EndpointContextTest#endpointContextBuild_universeDomainEnvVarSet+endpointContextBuild_multipleUniverseDomainConfigurations_clientSettingsHasPriority + + + + envVarTest + + + + org.apache.maven.plugins + maven-surefire-plugin + + EndpointContextTest#endpointContextBuild_universeDomainEnvVarSet+endpointContextBuild_multipleUniverseDomainConfigurations_clientSettingsHasPriority + + + + + + \ No newline at end of file diff --git a/gax-java/gax/src/main/java/com/google/api/gax/batching/BatcherStats.java b/gax-java/gax/src/main/java/com/google/api/gax/batching/BatcherStats.java index 784af6f599..20b0f95954 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/batching/BatcherStats.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/batching/BatcherStats.java @@ -31,7 +31,9 @@ import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode.Code; +import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; +import com.google.common.collect.EvictingQueue; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -52,6 +54,21 @@ class BatcherStats { private final Map entryExceptionCounts = new HashMap<>(); private final Map entryStatusCounts = new HashMap<>(); + /** + * The maximum number of error messages that a Batcher instance will retain. By default, a Batcher + * instance will retain 50 entry error messages and 50 RPC error messages. This limit can be + * temporarily increased by setting the {@code com.google.api.gax.batching.errors.max-samples} + * system property. This should only be needed in very rare situations and should not be + * considered part of the public api. + */ + private final int MAX_ERROR_MSG_SAMPLES = + Integer.getInteger("com.google.api.gax.batching.errors.max-samples", 50); + + private final EvictingQueue sampleOfRpcErrors = + EvictingQueue.create(MAX_ERROR_MSG_SAMPLES); + private final EvictingQueue sampleOfEntryErrors = + EvictingQueue.create(MAX_ERROR_MSG_SAMPLES); + /** * Records the count of the exception and it's type when a complete batch failed to apply. * @@ -69,6 +86,8 @@ synchronized void recordBatchFailure(Throwable throwable) { requestStatusCounts.put(code, oldStatusCount + 1); } + sampleOfRpcErrors.add(throwable.toString()); + int oldExceptionCount = MoreObjects.firstNonNull(requestExceptionCounts.get(exceptionClass), 0); requestExceptionCounts.put(exceptionClass, oldExceptionCount + 1); } @@ -96,6 +115,8 @@ synchronized void recordBatchElementsCompletion( Throwable actualCause = throwable.getCause(); Class exceptionClass = actualCause.getClass(); + sampleOfEntryErrors.add(actualCause.toString()); + if (actualCause instanceof ApiException) { Code code = ((ApiException) actualCause).getStatusCode().getCode(); exceptionClass = ApiException.class; @@ -144,6 +165,17 @@ synchronized BatchingException asException() { .append(buildExceptionList(entryExceptionCounts, entryStatusCounts)) .append("."); } + + if (!sampleOfRpcErrors.isEmpty()) { + messageBuilder.append(" Sample of RPC errors: "); + messageBuilder.append(Joiner.on(", ").join(sampleOfRpcErrors)); + messageBuilder.append("."); + } + if (!sampleOfEntryErrors.isEmpty()) { + messageBuilder.append(" Sample of entry errors: "); + messageBuilder.append(Joiner.on(", ").join(sampleOfEntryErrors)); + messageBuilder.append("."); + } return new BatchingException(messageBuilder.toString()); } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java b/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java index b33a97cd0a..5b1f14fdaf 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java @@ -47,6 +47,7 @@ @InternalApi @AutoValue public abstract class EndpointContext { + private static final String GOOGLE_CLOUD_UNIVERSE_DOMAIN = "GOOGLE_CLOUD_UNIVERSE_DOMAIN"; private static final String INVALID_UNIVERSE_DOMAIN_ERROR_TEMPLATE = "The configured universe domain (%s) does not match the universe domain found in the credentials (%s). If you haven't configured the universe domain explicitly, `googleapis.com` is the default."; public static final String UNABLE_TO_RETRIEVE_CREDENTIALS_ERROR_MESSAGE = @@ -201,20 +202,27 @@ public abstract static class Builder { abstract EndpointContext autoBuild(); private String determineUniverseDomain() { + String universeDomain = universeDomain(); if (usingGDCH()) { // GDC-H has no concept of Universe Domain. User should not set a custom value - if (universeDomain() != null) { + if (universeDomain != null) { throw new IllegalArgumentException( "Universe domain configuration is incompatible with GDC-H"); } return Credentials.GOOGLE_DEFAULT_UNIVERSE; } // Check for "" (empty string) - if (universeDomain() != null && universeDomain().isEmpty()) { + if (universeDomain != null && universeDomain.isEmpty()) { throw new IllegalArgumentException("The universe domain value cannot be empty."); } - // Override with user set universe domain if provided - return universeDomain() != null ? universeDomain() : Credentials.GOOGLE_DEFAULT_UNIVERSE; + // If the universe domain wasn't configured explicitly in the settings, check the + // environment variable for the value + if (universeDomain == null) { + universeDomain = System.getenv(GOOGLE_CLOUD_UNIVERSE_DOMAIN); + } + // If the universe domain is configured by the user, the universe domain will either be + // from the settings or from the env var. The value from ClientSettings has priority. + return universeDomain != null ? universeDomain : Credentials.GOOGLE_DEFAULT_UNIVERSE; } /** Determines the fully resolved endpoint and universe domain values */ diff --git a/gax-java/gax/src/test/java/com/google/api/gax/batching/BatcherStatsTest.java b/gax-java/gax/src/test/java/com/google/api/gax/batching/BatcherStatsTest.java index 1a95e4d3cb..1d10917aeb 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/batching/BatcherStatsTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/batching/BatcherStatsTest.java @@ -55,7 +55,10 @@ public void testRequestFailuresOnly() { batcherStats.recordBatchFailure( ApiExceptionFactory.createException( - new RuntimeException(), FakeStatusCode.of(StatusCode.Code.INVALID_ARGUMENT), false)); + "fake api error", + new RuntimeException(), + FakeStatusCode.of(StatusCode.Code.INVALID_ARGUMENT), + false)); batcherStats.recordBatchFailure(new RuntimeException("Request failed")); @@ -65,6 +68,10 @@ public void testRequestFailuresOnly() { assertThat(exception).hasMessageThat().contains("1 RuntimeException"); assertThat(exception).hasMessageThat().contains("1 ApiException(1 INVALID_ARGUMENT)"); assertThat(exception).hasMessageThat().contains("and 0 partial failures."); + assertThat(exception) + .hasMessageThat() + .contains( + "com.google.api.gax.rpc.InvalidArgumentException: fake api error, java.lang.RuntimeException: Request failed."); } @Test @@ -79,7 +86,10 @@ public void testEntryFailureOnly() { SettableApiFuture batchTwoResult = SettableApiFuture.create(); batchTwoResult.setException( ApiExceptionFactory.createException( - new RuntimeException(), FakeStatusCode.of(StatusCode.Code.UNAVAILABLE), false)); + "fake entry error", + new RuntimeException(), + FakeStatusCode.of(StatusCode.Code.UNAVAILABLE), + false)); batcherStats.recordBatchElementsCompletion( ImmutableList.of(BatchEntry.create(2, batchTwoResult))); @@ -89,6 +99,10 @@ public void testEntryFailureOnly() { .contains("The 2 partial failures contained 2 entries that failed with:"); assertThat(ex).hasMessageThat().contains("1 ApiException(1 UNAVAILABLE)"); assertThat(ex).hasMessageThat().contains("1 IllegalStateException"); + assertThat(ex) + .hasMessageThat() + .contains( + "Sample of entry errors: java.lang.IllegalStateException: local element failure, com.google.api.gax.rpc.UnavailableException: fake entry error."); } @Test @@ -110,6 +124,8 @@ public void testRequestAndEntryFailures() { .contains( "Batching finished with 1 batches failed to apply due to: 1 RuntimeException and 1 " + "partial failures. The 1 partial failures contained 1 entries that failed with:" - + " 1 ApiException(1 ALREADY_EXISTS)."); + + " 1 ApiException(1 ALREADY_EXISTS)." + + " Sample of RPC errors: java.lang.RuntimeException: Batch failure." + + " Sample of entry errors: com.google.api.gax.rpc.AlreadyExistsException: java.lang.RuntimeException."); } } diff --git a/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java b/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java index f0dbae60f2..54b7d0e756 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java @@ -340,6 +340,41 @@ public void endpointContextBuild_gdchFlow_noUniverseDomain_customEndpoint() thro .isEqualTo(Credentials.GOOGLE_DEFAULT_UNIVERSE); } + // This Universe Domain should match the `GOOGLE_CLOUD_UNIVERSE_DOMAIN` Env Var + // For this test running locally or in CI, check that the Env Var is set properly. + // This test should only run when the maven profile `EnvVarTest` is enabled. + @Test + public void endpointContextBuild_universeDomainEnvVarSet() throws IOException { + String envVarUniverseDomain = "random.com"; + EndpointContext endpointContext = + defaultEndpointContextBuilder + .setUniverseDomain(null) + .setClientSettingsEndpoint(null) + .build(); + Truth.assertThat(endpointContext.resolvedEndpoint()).isEqualTo("test.random.com:443"); + Truth.assertThat(endpointContext.resolvedUniverseDomain()).isEqualTo(envVarUniverseDomain); + } + + // This Universe Domain should match the `GOOGLE_CLOUD_UNIVERSE_DOMAIN` Env Var + // For this test running locally or in CI, check that the Env Var is set properly. + // This test should only run when the maven profile `EnvVarTest` is enabled. + @Test + public void endpointContextBuild_multipleUniverseDomainConfigurations_clientSettingsHasPriority() + throws IOException { + // This test has `GOOGLE_CLOUD_UNIVERSE_DOMAIN` = `random.com` + String clientSettingsUniverseDomain = "clientSettingsUniverseDomain.com"; + EndpointContext endpointContext = + defaultEndpointContextBuilder + .setUniverseDomain(clientSettingsUniverseDomain) + .setClientSettingsEndpoint(null) + .build(); + Truth.assertThat(endpointContext.resolvedEndpoint()) + .isEqualTo("test.clientSettingsUniverseDomain.com:443"); + // Client Settings Universe Domain (if set) takes priority + Truth.assertThat(endpointContext.resolvedUniverseDomain()) + .isEqualTo(clientSettingsUniverseDomain); + } + @Test public void hasValidUniverseDomain_gdchFlow_anyCredentials() throws IOException { Credentials noCredentials = NoCredentialsProvider.create().getCredentials(); diff --git a/gax-java/pom.xml b/gax-java/pom.xml index 5e98463187..ee254e0304 100644 --- a/gax-java/pom.xml +++ b/gax-java/pom.xml @@ -4,14 +4,14 @@ com.google.api gax-parent pom - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT GAX (Google Api eXtensions) for Java (Parent) Google Api eXtensions for Java (Parent) com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -50,7 +50,7 @@ com.google.api api-common - 2.26.1-SNAPSHOT + 2.28.1-SNAPSHOT com.google.auth @@ -108,24 +108,24 @@ com.google.api gax - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT com.google.api gax - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT test-jar testlib com.google.api.grpc proto-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT com.google.api.grpc grpc-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT io.grpc diff --git a/java-common-protos/grpc-google-common-protos/pom.xml b/java-common-protos/grpc-google-common-protos/pom.xml index 2f650fa397..dc2ba76c2a 100644 --- a/java-common-protos/grpc-google-common-protos/pom.xml +++ b/java-common-protos/grpc-google-common-protos/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT grpc-google-common-protos GRPC library for grpc-google-common-protos com.google.api.grpc google-common-protos-parent - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT diff --git a/java-common-protos/pom.xml b/java-common-protos/pom.xml index 041ea3589c..4ab623205f 100644 --- a/java-common-protos/pom.xml +++ b/java-common-protos/pom.xml @@ -4,7 +4,7 @@ com.google.api.grpc google-common-protos-parent pom - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT Google Common Protos Parent Java idiomatic client for Google Cloud Platform services. @@ -13,7 +13,7 @@ com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -61,7 +61,7 @@ com.google.cloud third-party-dependencies - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT pom import @@ -75,7 +75,7 @@ com.google.api.grpc grpc-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT io.grpc @@ -87,7 +87,7 @@ com.google.api.grpc proto-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT com.google.guava diff --git a/java-common-protos/proto-google-common-protos/pom.xml b/java-common-protos/proto-google-common-protos/pom.xml index 2b9e21566f..dec29c99d5 100644 --- a/java-common-protos/proto-google-common-protos/pom.xml +++ b/java-common-protos/proto-google-common-protos/pom.xml @@ -3,13 +3,13 @@ 4.0.0 com.google.api.grpc proto-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT proto-google-common-protos PROTO library for proto-google-common-protos com.google.api.grpc google-common-protos-parent - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT diff --git a/java-core/google-cloud-core-bom/pom.xml b/java-core/google-cloud-core-bom/pom.xml index f1ae640cc7..0077cd6a09 100644 --- a/java-core/google-cloud-core-bom/pom.xml +++ b/java-core/google-cloud-core-bom/pom.xml @@ -3,13 +3,13 @@ 4.0.0 com.google.cloud google-cloud-core-bom - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT pom com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../../gapic-generator-java-pom-parent @@ -23,17 +23,17 @@ com.google.cloud google-cloud-core - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT com.google.cloud google-cloud-core-grpc - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT com.google.cloud google-cloud-core-http - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT diff --git a/java-core/google-cloud-core-grpc/pom.xml b/java-core/google-cloud-core-grpc/pom.xml index 6cc246f188..1f9251afac 100644 --- a/java-core/google-cloud-core-grpc/pom.xml +++ b/java-core/google-cloud-core-grpc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-core-grpc - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT jar Google Cloud Core gRPC @@ -12,7 +12,7 @@ com.google.cloud google-cloud-core-parent - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT google-cloud-core-grpc diff --git a/java-core/google-cloud-core-http/pom.xml b/java-core/google-cloud-core-http/pom.xml index e8f580f411..57407ebe73 100644 --- a/java-core/google-cloud-core-http/pom.xml +++ b/java-core/google-cloud-core-http/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-core-http - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT jar Google Cloud Core HTTP @@ -12,7 +12,7 @@ com.google.cloud google-cloud-core-parent - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT google-cloud-core-http diff --git a/java-core/google-cloud-core/pom.xml b/java-core/google-cloud-core/pom.xml index 06a83e5170..7dab5b72dd 100644 --- a/java-core/google-cloud-core/pom.xml +++ b/java-core/google-cloud-core/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-core - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT jar Google Cloud Core @@ -12,7 +12,7 @@ com.google.cloud google-cloud-core-parent - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT google-cloud-core diff --git a/java-core/pom.xml b/java-core/pom.xml index 95715809d0..28a58aa212 100644 --- a/java-core/pom.xml +++ b/java-core/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-core-parent pom - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT Google Cloud Core Parent Java idiomatic client for Google Cloud Platform services. @@ -13,7 +13,7 @@ com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -33,7 +33,7 @@ com.google.cloud google-cloud-shared-dependencies - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT pom import diff --git a/java-iam/grpc-google-iam-v1/pom.xml b/java-iam/grpc-google-iam-v1/pom.xml index dec5af5f8c..67380b8d99 100644 --- a/java-iam/grpc-google-iam-v1/pom.xml +++ b/java-iam/grpc-google-iam-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-iam-v1 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT grpc-google-iam-v1 GRPC library for grpc-google-iam-v1 com.google.cloud google-iam-parent - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT diff --git a/java-iam/grpc-google-iam-v2/pom.xml b/java-iam/grpc-google-iam-v2/pom.xml index a5ef2d9d33..62eb63b981 100644 --- a/java-iam/grpc-google-iam-v2/pom.xml +++ b/java-iam/grpc-google-iam-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-iam-v2 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT grpc-google-iam-v2 GRPC library for proto-google-iam-v2 com.google.cloud google-iam-parent - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT diff --git a/java-iam/grpc-google-iam-v2beta/pom.xml b/java-iam/grpc-google-iam-v2beta/pom.xml index 908fc382e7..94c8b5fe2a 100644 --- a/java-iam/grpc-google-iam-v2beta/pom.xml +++ b/java-iam/grpc-google-iam-v2beta/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-iam-v2beta - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT grpc-google-iam-v2beta GRPC library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT diff --git a/java-iam/pom.xml b/java-iam/pom.xml index 471c24acc7..29b3d14a01 100644 --- a/java-iam/pom.xml +++ b/java-iam/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-iam-parent pom - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT Google IAM Parent Java idiomatic client for Google Cloud Platform services. @@ -13,7 +13,7 @@ com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -60,7 +60,7 @@ com.google.cloud third-party-dependencies - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT pom import @@ -88,44 +88,44 @@ com.google.api gax-bom - 2.43.1-SNAPSHOT + 2.45.1-SNAPSHOT pom import com.google.api.grpc proto-google-iam-v2 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc proto-google-common-protos - 2.34.1-SNAPSHOT + 2.36.1-SNAPSHOT com.google.api.grpc proto-google-iam-v2beta - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v1 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2beta - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT com.google.api.grpc proto-google-iam-v1 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT javax.annotation diff --git a/java-iam/proto-google-iam-v1/pom.xml b/java-iam/proto-google-iam-v1/pom.xml index 9d0686668d..b9c3485deb 100644 --- a/java-iam/proto-google-iam-v1/pom.xml +++ b/java-iam/proto-google-iam-v1/pom.xml @@ -3,13 +3,13 @@ 4.0.0 com.google.api.grpc proto-google-iam-v1 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT proto-google-iam-v1 PROTO library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT diff --git a/java-iam/proto-google-iam-v2/pom.xml b/java-iam/proto-google-iam-v2/pom.xml index 5d4c8340d0..7388a05717 100644 --- a/java-iam/proto-google-iam-v2/pom.xml +++ b/java-iam/proto-google-iam-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-iam-v2 - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT proto-google-iam-v2 Proto library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT diff --git a/java-iam/proto-google-iam-v2beta/pom.xml b/java-iam/proto-google-iam-v2beta/pom.xml index ef7bda3be4..a7f9552344 100644 --- a/java-iam/proto-google-iam-v2beta/pom.xml +++ b/java-iam/proto-google-iam-v2beta/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-iam-v2beta - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT proto-google-iam-v2beta Proto library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.29.1-SNAPSHOT + 1.31.1-SNAPSHOT diff --git a/java-shared-dependencies/README.md b/java-shared-dependencies/README.md index 4c0e79e467..b5889ad650 100644 --- a/java-shared-dependencies/README.md +++ b/java-shared-dependencies/README.md @@ -14,7 +14,7 @@ If you are using Maven, add this to the `dependencyManagement` section. com.google.cloud google-cloud-shared-dependencies - 3.25.0 + 3.27.0 pom import diff --git a/java-shared-dependencies/dependency-convergence-check/pom.xml b/java-shared-dependencies/dependency-convergence-check/pom.xml index f599b9ddcc..9b9aa08e00 100644 --- a/java-shared-dependencies/dependency-convergence-check/pom.xml +++ b/java-shared-dependencies/dependency-convergence-check/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud shared-dependencies-dependency-convergence-test - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT Dependency convergence test for certain artifacts in Google Cloud Shared Dependencies An dependency convergence test case for the shared dependencies BOM. A failure of this test case means diff --git a/java-shared-dependencies/first-party-dependencies/pom.xml b/java-shared-dependencies/first-party-dependencies/pom.xml index ca857100ff..f3d73b7550 100644 --- a/java-shared-dependencies/first-party-dependencies/pom.xml +++ b/java-shared-dependencies/first-party-dependencies/pom.xml @@ -6,7 +6,7 @@ com.google.cloud first-party-dependencies pom - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT Google Cloud First-party Shared Dependencies Shared first-party dependencies for Google Cloud Java libraries. @@ -15,7 +15,7 @@ com.google.cloud google-cloud-shared-config - 1.7.1 + 1.7.4 @@ -33,7 +33,7 @@ com.google.api gapic-generator-java-bom - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT pom import @@ -45,7 +45,7 @@ com.google.cloud google-cloud-core-bom - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT pom import @@ -69,13 +69,13 @@ com.google.cloud google-cloud-core - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT test-jar com.google.cloud google-cloud-core - 2.33.1-SNAPSHOT + 2.35.1-SNAPSHOT tests diff --git a/java-shared-dependencies/pom.xml b/java-shared-dependencies/pom.xml index c29010084b..cd9ab47851 100644 --- a/java-shared-dependencies/pom.xml +++ b/java-shared-dependencies/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-shared-dependencies pom - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT first-party-dependencies third-party-dependencies @@ -17,7 +17,7 @@ com.google.api gapic-generator-java-pom-parent - 2.35.1-SNAPSHOT + 2.37.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -31,14 +31,14 @@ com.google.cloud first-party-dependencies - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT pom import com.google.cloud third-party-dependencies - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT pom import diff --git a/java-shared-dependencies/third-party-dependencies/pom.xml b/java-shared-dependencies/third-party-dependencies/pom.xml index 461329cb3d..67177a9439 100644 --- a/java-shared-dependencies/third-party-dependencies/pom.xml +++ b/java-shared-dependencies/third-party-dependencies/pom.xml @@ -6,7 +6,7 @@ com.google.cloud third-party-dependencies pom - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT Google Cloud Third-party Shared Dependencies Shared third-party dependencies for Google Cloud Java libraries. @@ -15,7 +15,7 @@ com.google.cloud google-cloud-shared-config - 1.7.1 + 1.7.4 @@ -38,7 +38,7 @@ 3.42.0 0.27.0 2.8 - 1.34.1 + 1.35.0 0.8 15.0.0 diff --git a/java-shared-dependencies/unmanaged-dependency-check/pom.xml b/java-shared-dependencies/unmanaged-dependency-check/pom.xml index 5383658501..6796ef6ba6 100644 --- a/java-shared-dependencies/unmanaged-dependency-check/pom.xml +++ b/java-shared-dependencies/unmanaged-dependency-check/pom.xml @@ -73,12 +73,12 @@ org.slf4j slf4j-api - 2.0.11 + 2.0.12 org.slf4j slf4j-simple - 2.0.11 + 2.0.12 diff --git a/java-shared-dependencies/upper-bound-check/pom.xml b/java-shared-dependencies/upper-bound-check/pom.xml index 9c9f00a1ef..4921fbfe9c 100644 --- a/java-shared-dependencies/upper-bound-check/pom.xml +++ b/java-shared-dependencies/upper-bound-check/pom.xml @@ -4,7 +4,7 @@ com.google.cloud shared-dependencies-upper-bound-test pom - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT Upper bound test for Google Cloud Shared Dependencies An upper bound test case for the shared dependencies BOM. A failure of this test case means @@ -16,7 +16,7 @@ com.google.cloud google-cloud-shared-config - 1.7.1 + 1.7.4 @@ -30,7 +30,7 @@ com.google.cloud google-cloud-shared-dependencies - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT pom import diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 5df8060cd5..e448b167f6 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -62,7 +62,6 @@ def generate_composed_library( config=config, library=library, output_folder=output_folder ) - is_monorepo = util.check_monorepo(config=config) base_arguments = __construct_tooling_arg(config=config) owlbot_cli_source_folder = util.sh_util("mktemp -d") os.makedirs(f"{library_path}", exist_ok=True) @@ -80,10 +79,7 @@ def generate_composed_library( transport=gapic_inputs.transport, library_path=library_path, ) - service_version = gapic.proto_path.split("/")[-1] - temp_destination_path = ( - f"java-{util.get_library_name(library)}-{service_version}" - ) + temp_destination_path = f"java-{gapic.proto_path.replace('/','-')}" effective_arguments = __construct_effective_arg( base_arguments=base_arguments, gapic=gapic, @@ -115,7 +111,7 @@ def generate_composed_library( owlbot_cli_source_folder, config.owlbot_cli_image, config.synthtool_commitish, - str(is_monorepo).lower(), + str(config.is_monorepo).lower(), config.path_to_yaml, ], "Library postprocessing", diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 85c10c029f..c4b2c32ea8 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -15,6 +15,7 @@ import click import library_generation.utilities as util +import os from library_generation.generate_composed_library import generate_composed_library from library_generation.model.generation_config import from_yaml @@ -78,6 +79,11 @@ def generate_from_yaml( Parses a config yaml and generates libraries via generate_composed_library.py """ + # convert paths to absolute paths so they can be correctly referenced in + # downstream scripts + generation_config_yaml = os.path.abspath(generation_config_yaml) + repository_path = os.path.abspath(repository_path) + config = from_yaml(generation_config_yaml) target_libraries = config.libraries if target_library_api_shortname is not None: @@ -102,7 +108,11 @@ def generate_from_yaml( versions_file=repo_config.versions_file, ) - util.repo_level_post_process( + # we skip monorepo_postprocessing if not in a monorepo + if not config.is_monorepo: + return + + util.monorepo_postprocessing( repository_path=repository_path, versions_file=repo_config.versions_file ) diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 3a59958a63..22823b903d 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -46,6 +46,8 @@ def __init__( self.libraries = libraries self.grpc_version = grpc_version self.protobuf_version = protobuf_version + # monorepos have more than one library defined in the config yaml + self.is_monorepo = len(libraries) > 1 def from_yaml(path_to_yaml: str) -> GenerationConfig: @@ -92,6 +94,9 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig: rpc_documentation=__optional(library, "rpc_documentation", None), cloud_api=__optional(library, "cloud_api", True), requires_billing=__optional(library, "requires_billing", True), + extra_versioned_modules=__optional( + library, "extra_versioned_modules", None + ), ) parsed_libraries.append(new_library) diff --git a/library_generation/model/library_config.py b/library_generation/model/library_config.py index 9d281b912c..b6f8ae1b48 100644 --- a/library_generation/model/library_config.py +++ b/library_generation/model/library_config.py @@ -46,6 +46,7 @@ def __init__( rpc_documentation: Optional[str] = None, cloud_api: Optional[bool] = True, requires_billing: Optional[bool] = True, + extra_versioned_modules: Optional[str] = None, ): self.api_shortname = api_shortname self.api_description = api_description @@ -69,3 +70,4 @@ def __init__( self.rpc_documentation = rpc_documentation self.cloud_api = cloud_api self.requires_billing = requires_billing + self.extra_versioned_modules = extra_versioned_modules diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index c36f7cf5ff..e7eb91c179 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -28,6 +28,7 @@ scripts_root=$1 versions_file=$2 configuration_yaml=$3 + # This script can be used to process HW libraries and monorepo # (google-cloud-java) libraries, which require a slightly different treatment # monorepo folders have an .OwlBot.yaml file in the module folder (e.g. @@ -65,7 +66,7 @@ echo "...done" # write or restore pom.xml files echo "Generating missing pom.xml..." -python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true" +python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "${monorepo}" echo "...done" # write or restore clirr-ignored-differences.xml @@ -78,16 +79,7 @@ echo "Fixing missing license headers..." python3 "${scripts_root}/owlbot/src/fix-license-headers.py" echo "...done" -# TODO: re-enable this once we resolve thrashing -# restore license headers years -# echo "Restoring copyright years..." -# /owlbot/bin/restore_license_headers.sh -# echo "...done" - # ensure formatting on all .java files in the repository echo "Reformatting source..." mvn fmt:format -V --batch-mode --no-transfer-progress echo "...done" - - - diff --git a/library_generation/owlbot/src/fix-poms.py b/library_generation/owlbot/src/fix-poms.py index b57012fcf8..4bfbe05057 100644 --- a/library_generation/owlbot/src/fix-poms.py +++ b/library_generation/owlbot/src/fix-poms.py @@ -559,7 +559,5 @@ def main(versions_file, monorepo): if __name__ == "__main__": versions_file = sys.argv[1] - monorepo = sys.argv[2] - if monorepo == "true": - monorepo = True + monorepo = True if sys.argv[2].lower() == "true" else False main(versions_file, monorepo) diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 9f9b5a3786..66c0c1a020 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -68,8 +68,8 @@ else fi # Default values for running copy-code directly from host -repo_binding="${postprocessing_target}" -repo_workspace="/repo" +repo_bindings="-v ${postprocessing_target}:/workspace" +repo_workspace="/workspace" preprocessed_libraries_binding="${owlbot_cli_source_folder}" # When running docker inside docker, we run into the issue of volume bindings @@ -86,16 +86,20 @@ preprocessed_libraries_binding="${owlbot_cli_source_folder}" if [[ -n "${RUNNING_IN_DOCKER}" ]]; then set -u # temporarily fail on unset variables - repo_binding="${REPO_BINDING_VOLUME}" + repo_bindings="${REPO_BINDING_VOLUMES}" set +u + library_name=$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev) + repo_workspace="/workspace/" if [[ "${is_monorepo}" == "true" ]]; then - repo_workspace="/repo/$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev)" + monorepo_name=$(echo "${postprocessing_target}" | rev | cut -d'/' -f2 | rev) + repo_workspace+="${monorepo_name}/" fi + repo_workspace+="${library_name}" fi docker run --rm \ --user "$(id -u)":"$(id -g)" \ - -v "${repo_binding}:/repo" \ + ${repo_bindings} \ -v "/tmp:/tmp" \ -w "${repo_workspace}" \ --env HOME=/tmp \ @@ -115,6 +119,7 @@ fi git config --global --add safe.directory /tmp/synthtool/synthtool pushd "synthtool" +git fetch --all git reset --hard "${synthtool_commitish}" python3 -m pip install -e . diff --git a/library_generation/test/container_integration_tests.sh b/library_generation/test/container_integration_tests.sh index 9a17e585a6..a60c733df9 100644 --- a/library_generation/test/container_integration_tests.sh +++ b/library_generation/test/container_integration_tests.sh @@ -8,32 +8,42 @@ if [[ -z "${TEST_IMAGE_ID}" ]]; then exit 1 fi -if [[ ! -d google-cloud-java ]]; then - git clone https://github.com/googleapis/google-cloud-java -fi -pushd google-cloud-java -git reset --hard main -popd +repo_volumes="" +for repo in google-cloud-java java-bigtable; do + if [[ ! -d "${repo}" ]]; then + git clone "https://github.com/googleapis/${repo}" + fi + pushd "${repo}" + git reset --hard main + popd -# We use a volume to hold the google-cloud-java repository used in the -# integration tests. This is because the test container creates a child -# container using the host machine's docker socket, meaning that we can only -# reference volumes created from within the host machine (i.e. the machine -# running this script) -# -# To summarize, we create a special volume that can be referenced both in the -# main container and in any child containers created by this one. -if [[ $(docker volume inspect repo) != '[]' ]]; then - docker volume rm repo -fi -docker volume create --name "repo" --opt "type=none" --opt "device=$(pwd)/google-cloud-java" --opt "o=bind" + # We use a volume to hold the repositories used in the + # integration tests. This is because the test container creates a child + # container using the host machine's docker socket, meaning that we can only + # reference volumes created from within the host machine (i.e. the machine + # running this script) + # + # To summarize, we create a special volume that can be referenced both in the + # main container and in any child containers created by this one. + volume_name="repo-${repo}" + if [[ $(docker volume inspect "${volume_name}") != '[]' ]]; then + docker volume rm "${volume_name}" + fi + docker volume create \ + --name "${volume_name}" \ + --opt "type=none" \ + --opt "device=$(pwd)/${repo}" \ + --opt "o=bind" + + repo_volumes="${repo_volumes} -v ${volume_name}:/workspace/${repo}" +done docker run --rm \ - -v repo:/workspace \ + ${repo_volumes} \ -v /tmp:/tmp \ -v /var/run/docker.sock:/var/run/docker.sock \ -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUME=repo" \ + -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ -w "/src" \ "${TEST_IMAGE_ID}" \ python -m unittest /src/test/integration_tests.py diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index c552a52e4a..354229b684 100755 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -24,7 +24,7 @@ from typing import List from typing import Dict from library_generation.generate_repo import generate_from_yaml -from library_generation.model.generation_config import from_yaml +from library_generation.model.generation_config import from_yaml, GenerationConfig from library_generation.test.compare_poms import compare_xml from library_generation.utilities import ( get_library_name, @@ -40,8 +40,12 @@ config_dir = f"{script_dir}/resources/integration" golden_dir = f"{config_dir}/golden" repo_prefix = "https://github.com/googleapis" -committish_list = ["chore/test-hermetic-build"] # google-cloud-java -output_folder = shell_call("get_output_folder") +output_dir = shell_call("get_output_folder") +# this map tells which branch of each repo should we use for our diff tests +committish_map = { + "google-cloud-java": "chore/test-hermetic-build", + "java-bigtable": "chore/test-hermetic-build", +} class IntegrationTest(unittest.TestCase): @@ -49,33 +53,45 @@ def test_generate_repo(self): shutil.rmtree(f"{golden_dir}", ignore_errors=True) os.makedirs(f"{golden_dir}", exist_ok=True) config_files = self.__get_config_files(config_dir) - i = 0 - for repo, config_file in config_files.items(): + for repo, config_file in config_files: + config = from_yaml(config_file) repo_dest = self.__pull_repo_to( - Path(f"{golden_dir}/{repo}"), repo, committish_list[i] + Path(f"{output_dir}/{repo}"), repo, committish_map[repo] ) - library_names = self.__get_library_names_from_config(config_file) + library_names = self.__get_library_names_from_config(config) # prepare golden files for library_name in library_names: - copy_tree(f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}") - copy_tree( - f"{repo_dest}/gapic-libraries-bom", f"{golden_dir}/gapic-libraries-bom" - ) - copy_file(f"{repo_dest}/pom.xml", golden_dir) + if config.is_monorepo: + copy_tree( + f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}" + ) + copy_tree( + f"{repo_dest}/gapic-libraries-bom", + f"{golden_dir}/gapic-libraries-bom", + ) + copy_file(f"{repo_dest}/pom.xml", golden_dir) + else: + copy_tree(f"{repo_dest}", f"{golden_dir}/{library_name}") generate_from_yaml( generation_config_yaml=config_file, repository_path=repo_dest ) # compare result for library_name in library_names: + actual_library = ( + f"{repo_dest}/{library_name}" if config.is_monorepo else repo_dest + ) print( f"Generation finished. Will now compare " f"the expected library in {golden_dir}/{library_name}, " - f"with the actual library in {repo_dest}/{library_name}. " + f"with the actual library in {actual_library}. " f"Compare generation result: " ) + target_repo_dest = ( + f"{repo_dest}/{library_name}" if config.is_monorepo else repo_dest + ) compare_result = dircmp( f"{golden_dir}/{library_name}", - f"{repo_dest}/{library_name}", + target_repo_dest, ignore=[".repo-metadata.json"], ) # compare source code @@ -87,11 +103,15 @@ def test_generate_repo(self): self.assertTrue( self.__compare_json_files( f"{golden_dir}/{library_name}/.repo-metadata.json", - f"{repo_dest}/{library_name}/.repo-metadata.json", + f"{target_repo_dest}/.repo-metadata.json", ), msg=f"The generated {library_name}/.repo-metadata.json is different from golden.", ) print(".repo-metadata.json comparison succeed.") + + if not config.is_monorepo: + continue + # compare gapic-libraries-bom/pom.xml and pom.xml self.assertFalse( compare_xml( @@ -109,14 +129,12 @@ def test_generate_repo(self): ) ) print("pom.xml comparison succeed.") - # remove google-cloud-java - i += 1 @classmethod def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str: if "RUNNING_IN_DOCKER" in os.environ: # the docker image expects the repo to be in /workspace - dest_in_docker = "/workspace" + dest_in_docker = f"/workspace/{repo}" run_process_and_print_output( [ "git", @@ -126,7 +144,7 @@ def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str: "safe.directory", dest_in_docker, ], - "Add /workspace to safe directories", + f"Add /workspace/{repo} to safe directories", ) dest = Path(dest_in_docker) repo = Repo(dest) @@ -140,8 +158,7 @@ def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str: return str(dest) @classmethod - def __get_library_names_from_config(cls, config_path: str) -> List[str]: - config = from_yaml(config_path) + def __get_library_names_from_config(cls, config: GenerationConfig) -> List[str]: library_names = [] for library in config.libraries: library_names.append(f"java-{get_library_name(library)}") @@ -149,15 +166,14 @@ def __get_library_names_from_config(cls, config_path: str) -> List[str]: return library_names @classmethod - def __get_config_files(cls, path: str) -> Dict[str, str]: - config_files = {} + def __get_config_files(cls, path: str) -> List[tuple[str, str]]: + config_files = [] for sub_dir in Path(path).resolve().iterdir(): repo = sub_dir.name - # skip the split repo. - if repo == "golden" or repo == "java-bigtable": + if repo == "golden": continue config = f"{sub_dir}/{config_name}" - config_files[repo] = config + config_files.append((repo, config)) return config_files diff --git a/library_generation/test/resources/goldens/.repo-metadata-golden.json b/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json similarity index 100% rename from library_generation/test/resources/goldens/.repo-metadata-golden.json rename to library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json diff --git a/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json b/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json new file mode 100644 index 0000000000..9e4a878d67 --- /dev/null +++ b/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json @@ -0,0 +1,19 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/java-bare-metal-solution", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "api_id": "baremetalsolution.googleapis.com", + "library_type": "GAPIC_COMBO", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "extra_versioned_modules": "test-module" +} \ No newline at end of file diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index a13860a8ca..5b2b4d3f0f 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -2,7 +2,7 @@ gapic_generator_version: 2.34.0 protobuf_version: 25.2 googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 +synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e template_excludes: - ".github/*" - ".kokoro/*" diff --git a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml index 997e2c14c7..55092330d6 100644 --- a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml +++ b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml @@ -1,9 +1,8 @@ -gapic_generator_version: 2.32.0 -grpc_version: 1.61.0 +gapic_generator_version: 2.35.0 protobuf_version: 25.2 -googleapis_commitish: 40203ca1880849480bbff7b8715491060bbccdf1 +googleapis_commitish: fc3043ebe12fb6bc1729c175e1526c859ce751d8 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 +synthtool_commitish: a6fb7d5f072b75698af1cbf06c5b001565753cfb template_excludes: - ".gitignore" - ".kokoro/presubmit/integration.cfg" @@ -23,6 +22,18 @@ template_excludes: libraries: - api_shortname: bigtable name_pretty: Cloud Bigtable + api_description: "Java idiomatic client for Cloud Bigtable." + product_documentation: "https://cloud.google.com/bigtable" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-bigtable/latest/history" + issue_tracker: "https://issuetracker.google.com/savedsearches/559777" + release_level: "stable" + language: "java" + repo: "googleapis/java-bigtable" + repo_short: "java-bigtable" + distribution_name: "com.google.cloud:google-cloud-bigtable" + excluded_poms: "google-cloud-bigtable-bom" + extra_versioned_modules: "google-cloud-bigtable-emulator,google-cloud-bigtable-emulator-core" + codeowner_team: "@googleapis/api-bigtable @googleapis/api-bigtable-partners" library_type: GAPIC_COMBO GAPICs: - proto_path: google/bigtable/admin/v2 diff --git a/library_generation/test/resources/test_repo_level_postprocess/gapic-libraries-bom/pom-golden.xml b/library_generation/test/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml similarity index 100% rename from library_generation/test/resources/test_repo_level_postprocess/gapic-libraries-bom/pom-golden.xml rename to library_generation/test/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml diff --git a/library_generation/test/resources/test_repo_level_postprocess/java-dns/pom.xml b/library_generation/test/resources/test_monorepo_postprocessing/java-dns/pom.xml similarity index 100% rename from library_generation/test/resources/test_repo_level_postprocess/java-dns/pom.xml rename to library_generation/test/resources/test_monorepo_postprocessing/java-dns/pom.xml diff --git a/library_generation/test/resources/test_repo_level_postprocess/java-service-control/google-cloud-service-control-bom/pom.xml b/library_generation/test/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml similarity index 100% rename from library_generation/test/resources/test_repo_level_postprocess/java-service-control/google-cloud-service-control-bom/pom.xml rename to library_generation/test/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml diff --git a/library_generation/test/resources/test_repo_level_postprocess/java-tasks/google-cloud-tasks-bom/pom.xml b/library_generation/test/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml similarity index 100% rename from library_generation/test/resources/test_repo_level_postprocess/java-tasks/google-cloud-tasks-bom/pom.xml rename to library_generation/test/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml diff --git a/library_generation/test/resources/test_repo_level_postprocess/pom-golden.xml b/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml similarity index 100% rename from library_generation/test/resources/test_repo_level_postprocess/pom-golden.xml rename to library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml diff --git a/library_generation/test/resources/test_repo_level_postprocess/versions.txt b/library_generation/test/resources/test_monorepo_postprocessing/versions.txt similarity index 100% rename from library_generation/test/resources/test_repo_level_postprocess/versions.txt rename to library_generation/test/resources/test_monorepo_postprocessing/versions.txt diff --git a/library_generation/test/unit_tests.py b/library_generation/test/unit_tests.py index 338bce6d9a..f7843f29c9 100644 --- a/library_generation/test/unit_tests.py +++ b/library_generation/test/unit_tests.py @@ -366,31 +366,35 @@ def test_get_library_returns_library_name(self): def test_get_library_returns_api_shortname(self): self.assertEqual("secretmanager", util.get_library_name(library_2)) - def test_generate_prerequisite_files_success(self): - library_path = f"{resources_dir}/goldens" - files = [ + def test_generate_prerequisite_files_non_monorepo_success(self): + library_path = self.__setup_prerequisite_files( + num_libraries=1, library_type="GAPIC_COMBO" + ) + + self.__compare_files( f"{library_path}/.repo-metadata.json", - f"{library_path}/.OwlBot.yaml", - f"{library_path}/owlbot.py", - ] - self.__cleanup(files) - config = self.__get_a_gen_config(1) - proto_path = "google/cloud/baremetalsolution/v2" - transport = "grpc" - util.generate_prerequisite_files( - config=config, - library=library_1, - proto_path=proto_path, - transport=transport, - library_path=library_path, + f"{library_path}/.repo-metadata-non-monorepo-golden.json", + ) + # since this is a single library, we treat this as HW repository, + # meaning that the owlbot yaml will be inside a .github folder + self.__compare_files( + f"{library_path}/.github/.OwlBot.yaml", + f"{library_path}/.OwlBot-golden.yaml", + ) + self.__compare_files( + f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) + def test_generate_prerequisite_files_monorepo_success(self): + library_path = self.__setup_prerequisite_files(num_libraries=2) + self.__compare_files( f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-golden.json", + f"{library_path}/.repo-metadata-monorepo-golden.json", ) self.__compare_files( - f"{library_path}/.OwlBot.yaml", f"{library_path}/.OwlBot-golden.yaml" + f"{library_path}/.OwlBot.yaml", + f"{library_path}/.OwlBot-golden.yaml", ) self.__compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" @@ -441,15 +445,15 @@ def test_prepare_repo_split_repo_success(self): library_path = sorted([Path(key).name for key in repo_config.libraries]) self.assertEqual(["misc"], library_path) - def test_repo_level_post_process_success(self): - repository_path = f"{resources_dir}/test_repo_level_postprocess" + def test_monorepo_postprocessing_valid_repository_success(self): + repository_path = f"{resources_dir}/test_monorepo_postprocessing" versions_file = f"{repository_path}/versions.txt" files = [ f"{repository_path}/pom.xml", f"{repository_path}/gapic-libraries-bom/pom.xml", ] self.__cleanup(files) - util.repo_level_post_process( + util.monorepo_postprocessing( repository_path=repository_path, versions_file=versions_file ) self.__compare_files( @@ -472,23 +476,58 @@ def __compare_files(self, expect: str, actual: str): first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff) ) + def __setup_prerequisite_files( + self, num_libraries: int, library_type: str = "GAPIC_AUTO" + ) -> str: + library_path = f"{resources_dir}/goldens" + files = [ + f"{library_path}/.repo-metadata.json", + f"{library_path}/.OwlBot.yaml", + f"{library_path}/owlbot.py", + ] + self.__cleanup(files) + config = self.__get_a_gen_config(num_libraries, library_type=library_type) + proto_path = "google/cloud/baremetalsolution/v2" + transport = "grpc" + util.generate_prerequisite_files( + config=config, + library=library_1, + proto_path=proto_path, + transport=transport, + library_path=library_path, + ) + return library_path + @staticmethod - def __get_a_gen_config(num: int): + def __get_a_gen_config( + num_libraries: int, library_type: str = "GAPIC_AUTO" + ) -> GenerationConfig: """ Returns an object of GenerationConfig with one to three of LibraryConfig objects. Other attributes are set to empty str. - :param num: the number of LibraryConfig objects associated with + :param num_libraries: the number of LibraryConfig objects associated with the GenerationConfig. Only support 1, 2 or 3. :return: an object of GenerationConfig """ - if num == 2: + if num_libraries == 2: libraries = [library_1, library_2] - elif num == 3: + elif num_libraries == 3: libraries = [library_1, library_2, library_3] else: libraries = [library_1] + # update libraries with custom configuration (for now, only + # library_type) + for library in libraries: + library.library_type = library_type + if num_libraries == 1: + # treat this as a HW library case to generate a real-life + # repo-metadata + library.extra_versioned_modules = "test-module" + else: + library.extra_versioned_modules = None + return GenerationConfig( gapic_generator_version="", googleapis_commitish="", diff --git a/library_generation/utilities.py b/library_generation/utilities.py index 8af797b792..0d1fd39c8f 100755 --- a/library_generation/utilities.py +++ b/library_generation/utilities.py @@ -224,16 +224,6 @@ def remove_version_from(proto_path: str) -> str: return proto_path -def check_monorepo(config: GenerationConfig) -> bool: - """ - Check whether to generate a monorepo according to the - generation config. - :param config: the generation configuration - :return: True if it's to generate a monorepo - """ - return len(config.libraries) > 1 - - def prepare_repo( gen_config: GenerationConfig, library_config: List[LibraryConfig], @@ -256,7 +246,6 @@ def prepare_repo( output_folder = sh_util("get_output_folder") print(f"output_folder: {output_folder}") os.makedirs(output_folder, exist_ok=True) - is_monorepo = check_monorepo(gen_config) libraries = {} for library in library_config: library_name = ( @@ -264,7 +253,9 @@ def prepare_repo( if library.library_name else f"{language}-{library.api_shortname}" ) - library_path = f"{repo_path}/{library_name}" if is_monorepo else f"{repo_path}" + library_path = ( + f"{repo_path}/{library_name}" if gen_config.is_monorepo else f"{repo_path}" + ) # use absolute path because docker requires absolute path # in volume name. absolute_library_path = str(Path(library_path).resolve()) @@ -331,7 +322,6 @@ def generate_prerequisite_files( transport: str, library_path: str, language: str = "java", - is_monorepo: bool = True, ) -> None: """ Generate prerequisite files for a library. @@ -344,7 +334,6 @@ def generate_prerequisite_files( :param transport: transport supported by the library :param library_path: the path to which the generated file goes :param language: programming language of the library - :param is_monorepo: whether the library is in a monorepo :return: None """ cloud_prefix = "cloud-" if library.cloud_api else "" @@ -356,7 +345,9 @@ def generate_prerequisite_files( ) distribution_name_short = re.split(r"[:/]", distribution_name)[-1] repo = ( - "googleapis/google-cloud-java" if is_monorepo else f"{language}-{library_name}" + "googleapis/google-cloud-java" + if config.is_monorepo + else f"googleapis/{language}-{library_name}" ) api_id = ( library.api_id if library.api_id else f"{library.api_shortname}.googleapis.com" @@ -407,6 +398,8 @@ def generate_prerequisite_files( repo_metadata["rest_documentation"] = library.rest_documentation if library.rpc_documentation: repo_metadata["rpc_documentation"] = library.rpc_documentation + if library.extra_versioned_modules: + repo_metadata["extra_versioned_modules"] = library.extra_versioned_modules # generate .repo-meta.json json_file = ".repo-metadata.json" @@ -417,11 +410,16 @@ def generate_prerequisite_files( json.dump(repo_metadata, fp, indent=2) # generate .OwlBot.yaml - yaml_file = ".OwlBot.yaml" - if not os.path.exists(f"{library_path}/{yaml_file}"): + owlbot_yaml_file = ".OwlBot.yaml" + path_to_owlbot_yaml_file = ( + f"{library_path}/{owlbot_yaml_file}" + if config.is_monorepo + else f"{library_path}/.github/{owlbot_yaml_file}" + ) + if not os.path.exists(path_to_owlbot_yaml_file): __render( template_name="owlbot.yaml.monorepo.j2", - output_name=f"{library_path}/{yaml_file}", + output_name=path_to_owlbot_yaml_file, artifact_name=distribution_name_short, proto_path=remove_version_from(proto_path), module_name=repo_metadata["repo_short"], @@ -439,7 +437,7 @@ def generate_prerequisite_files( ) -def repo_level_post_process( +def monorepo_postprocessing( repository_path: str, versions_file: str, ) -> None: diff --git a/release-please-config.json b/release-please-config.json index 658de223cb..ddf1373290 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -5,7 +5,7 @@ "group-pull-request-title-pattern": "chore(${branch}): release ${version}", "packages": { ".": { - "extra-files": ["WORKSPACE", ".cloudbuild/cloudbuild.yaml", ".cloudbuild/cloudbuild-test-a.yaml", ".cloudbuild/cloudbuild-test-b.yaml"] + "extra-files": ["WORKSPACE", ".cloudbuild/graalvm/cloudbuild.yaml", ".cloudbuild/graalvm/cloudbuild-test-a.yaml", ".cloudbuild/graalvm/cloudbuild-test-b.yaml", ".cloudbuild/library_generation/cloudbuild-library-generation-release.yaml"] } } } \ No newline at end of file diff --git a/rules_java_gapic/resources/gradle/gradle/wrapper/gradle-wrapper.properties b/rules_java_gapic/resources/gradle/gradle/wrapper/gradle-wrapper.properties index 068cdb2dc2..3994438e22 100644 --- a/rules_java_gapic/resources/gradle/gradle/wrapper/gradle-wrapper.properties +++ b/rules_java_gapic/resources/gradle/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/sdk-platform-java-config/pom.xml b/sdk-platform-java-config/pom.xml index 709c17b9f3..989e67dcfe 100644 --- a/sdk-platform-java-config/pom.xml +++ b/sdk-platform-java-config/pom.xml @@ -4,7 +4,7 @@ com.google.cloud sdk-platform-java-config pom - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT SDK Platform For Java Configurations Shared build configuration for Google Cloud Java libraries. @@ -13,10 +13,10 @@ com.google.cloud google-cloud-shared-config - 1.7.1 + 1.7.4 - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT \ No newline at end of file diff --git a/showcase/pom.xml b/showcase/pom.xml index 13fbf1e1ea..242a9b72b3 100644 --- a/showcase/pom.xml +++ b/showcase/pom.xml @@ -15,7 +15,7 @@ com.google.cloud google-cloud-shared-config - 1.7.1 + 1.7.4 @@ -34,7 +34,7 @@ com.google.cloud google-cloud-shared-dependencies - 3.25.1-SNAPSHOT + 3.27.1-SNAPSHOT pom import diff --git a/versions.txt b/versions.txt index dd4ea0a207..9faf00aea2 100644 --- a/versions.txt +++ b/versions.txt @@ -1,19 +1,19 @@ # Format: # module:released-version:current-version -gapic-generator-java:2.35.0:2.35.1-SNAPSHOT -api-common:2.26.0:2.26.1-SNAPSHOT -gax:2.43.0:2.43.1-SNAPSHOT -gax-grpc:2.43.0:2.43.1-SNAPSHOT -gax-httpjson:0.128.0:0.128.1-SNAPSHOT -proto-google-common-protos:2.34.0:2.34.1-SNAPSHOT -grpc-google-common-protos:2.34.0:2.34.1-SNAPSHOT -proto-google-iam-v1:1.29.0:1.29.1-SNAPSHOT -grpc-google-iam-v1:1.29.0:1.29.1-SNAPSHOT -proto-google-iam-v2beta:1.29.0:1.29.1-SNAPSHOT -grpc-google-iam-v2beta:1.29.0:1.29.1-SNAPSHOT -google-iam-policy:1.29.0:1.29.1-SNAPSHOT -proto-google-iam-v2:1.29.0:1.29.1-SNAPSHOT -grpc-google-iam-v2:1.29.0:1.29.1-SNAPSHOT -google-cloud-core:2.33.0:2.33.1-SNAPSHOT -google-cloud-shared-dependencies:3.25.0:3.25.1-SNAPSHOT +gapic-generator-java:2.37.0:2.37.1-SNAPSHOT +api-common:2.28.0:2.28.1-SNAPSHOT +gax:2.45.0:2.45.1-SNAPSHOT +gax-grpc:2.45.0:2.45.1-SNAPSHOT +gax-httpjson:0.130.0:0.130.1-SNAPSHOT +proto-google-common-protos:2.36.0:2.36.1-SNAPSHOT +grpc-google-common-protos:2.36.0:2.36.1-SNAPSHOT +proto-google-iam-v1:1.31.0:1.31.1-SNAPSHOT +grpc-google-iam-v1:1.31.0:1.31.1-SNAPSHOT +proto-google-iam-v2beta:1.31.0:1.31.1-SNAPSHOT +grpc-google-iam-v2beta:1.31.0:1.31.1-SNAPSHOT +google-iam-policy:1.31.0:1.31.1-SNAPSHOT +proto-google-iam-v2:1.31.0:1.31.1-SNAPSHOT +grpc-google-iam-v2:1.31.0:1.31.1-SNAPSHOT +google-cloud-core:2.35.0:2.35.1-SNAPSHOT +google-cloud-shared-dependencies:3.27.0:3.27.1-SNAPSHOT