Merge remote-tracking branch 'origin/main' into segrep_timer

.github/workflows/version.yml

@@ -30,7 +30,11 @@ jobs:
           CURRENT_VERSION_ARRAY[2]=$((CURRENT_VERSION_ARRAY[2]+1))
           NEXT_VERSION=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}")
           NEXT_VERSION_UNDERSCORE=$(IFS=_ ; echo "V_${CURRENT_VERSION_ARRAY[*]:0:3}")
-          NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99")
+          if [[ ${#CURRENT_VERSION_ARRAY[2]} -gt 1 ]]; then
+            NEXT_VERSION_ID="${CURRENT_VERSION_ARRAY[0]:0:3}0${CURRENT_VERSION_ARRAY[1]:0:3}${CURRENT_VERSION_ARRAY[2]:0:3}99"
+          else
+            NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99")
+          fi
           echo "TAG=$TAG" >> $GITHUB_ENV
           echo "BASE=$BASE" >> $GITHUB_ENV
           echo "BASE_X=$BASE_X" >> $GITHUB_ENV

CHANGELOG.md

@@ -14,7 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
 - Bump `forbiddenapis` from 3.3 to 3.4
-- Bump `avro` from 1.11.0 to 1.11.1
+- Bump `avro` from 1.11.1 to 1.11.2
 - Bump `woodstox-core` from 6.3.0 to 6.3.1
 - Bump `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354))
 - Bump `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447))
@@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
 - Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919))
 - Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909))
+- Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411))
 
 ### Deprecated
 
@@ -76,6 +77,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
 - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
 - Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541))
+- Adds log4j configuration for telemetry LogSpanExporter ([#8393](https://github.com/opensearch-project/OpenSearch/pull/8393))
 
 ### Security
 
@@ -95,6 +97,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221)
 - Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
 - Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168))
+- [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414))
+- Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545))
+- Support transport action names when registering NamedRoutes ([#7957](https://github.com/opensearch-project/OpenSearch/pull/7957))
+- Create concept of persistent ThreadContext headers that are unstashable ([#8291]()https://github.com/opensearch-project/OpenSearch/pull/8291)
+- Enable Partial Flat Object ([#7997](https://github.com/opensearch-project/OpenSearch/pull/7997))
 
 ### Dependencies
 - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814)
@@ -127,6 +134,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Upgrade] Lucene 9.7.0 release (#8272)
 - Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209))
 - Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276))
+- Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434))
+- Bump `org.apache.maven:maven-model` from 3.9.2 to 3.9.3 (#8403)
+- Bump `io.projectreactor.netty:reactor-netty` and `io.projectreactor.netty:reactor-netty-core` from 1.1.7 to 1.1.8 (#8405)
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
@@ -144,6 +154,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add self-organizing hash table to improve the performance of bucket aggregations ([#7652](https://github.com/opensearch-project/OpenSearch/pull/7652))
 - Check UTF16 string size before converting to String to avoid OOME ([#7963](https://github.com/opensearch-project/OpenSearch/pull/7963))
 - Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908))
+- Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471))
+- [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164))
 
 
 ### Deprecated

benchmarks/build.gradle

@@ -32,10 +32,16 @@ import org.opensearch.gradle.info.BuildParams
 
 apply plugin: 'opensearch.build'
 apply plugin: 'application'
-mainClassName = 'org.openjdk.jmh.Main'
 
 assemble.enabled = false
-archivesBaseName = 'opensearch-benchmarks'
+
+application {
+  mainClass = 'org.openjdk.jmh.Main'
+}
+
+base {
+  archivesBaseName = 'opensearch-benchmarks'
+}
 
 test.enabled = false
 

build.gradle

@@ -327,7 +327,7 @@ allprojects {
       javadoc.options.addStringOption('Xwerror', '-quiet')
     }
     javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"]
-    javadoc.options.addStringOption("-release", targetCompatibility.majorVersion)
+    javadoc.options.addStringOption("-release", java.targetCompatibility.majorVersion)
   }
 
   // support for reproducible builds
@@ -375,7 +375,7 @@ allprojects {
         } else {
           // Link to non-shadowed dependant projects
           project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
-          String externalLinkName = upstreamProject.archivesBaseName
+          String externalLinkName = upstreamProject.base.archivesBaseName
           String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version
           String projectRelativePath = project.relativePath(upstreamProject.buildDir)
           project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${projectRelativePath}/docs/javadoc/"
@@ -452,7 +452,7 @@ gradle.projectsEvaluated {
       testReportAggregation it
     }
     subprojects.findAll { it.pluginManager.hasPlugin('jacoco') }.forEach {
-     jacocoAggregation it
+      jacocoAggregation it
     }
   }
 }

buildSrc/build.gradle

@@ -34,7 +34,6 @@ import org.gradle.util.GradleVersion
 plugins {
   id 'java-gradle-plugin'
   id 'groovy'
-  id 'java-test-fixtures'
 }
 
 group = 'org.opensearch.gradle'
@@ -79,9 +78,17 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
 }
 
 sourceSets {
+  test {
+    java {
+      srcDirs += ['src/testFixtures/java']
+    }
+  }
   integTest {
     compileClasspath += sourceSets["main"].output + configurations["testRuntimeClasspath"]
     runtimeClasspath += output + compileClasspath
+    java {
+      srcDirs += ['src/testFixtures/java']
+    }
   }
 }
 
@@ -111,23 +118,23 @@ dependencies {
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.13.0'
   api "net.java.dev.jna:jna:5.13.0"
-  api 'gradle.plugin.com.github.johnrengelman:shadow:8.0.0'
+  api 'com.github.johnrengelman:shadow:8.1.1'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.5.1'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
-  api 'org.apache.maven:maven-model:3.9.2'
+  api 'org.apache.maven:maven-model:3.9.3'
   api 'com.networknt:json-schema-validator:1.0.85'
   api 'org.jruby.jcodings:jcodings:1.0.58'
   api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
   api "org.ajoberstar.grgit:grgit-core:5.2.0"
 
-  testFixturesApi "junit:junit:${props.getProperty('junit')}"
-  testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
-  testFixturesApi gradleApi()
-  testFixturesApi gradleTestKit()
+  testImplementation "junit:junit:${props.getProperty('junit')}"
+  testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+  testRuntimeOnly gradleApi()
+  testRuntimeOnly gradleTestKit()
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.35.0'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
   integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') {
@@ -164,8 +171,10 @@ if (project != rootProject) {
   apply plugin: 'opensearch.publish'
 
   allprojects {
-    targetCompatibility = JavaVersion.VERSION_11
-    sourceCompatibility = JavaVersion.VERSION_11
+    java {
+        targetCompatibility = JavaVersion.VERSION_11
+        sourceCompatibility = JavaVersion.VERSION_11
+    }
   }
 
   // groovydoc succeeds, but has some weird internal exception...
@@ -174,7 +183,7 @@ if (project != rootProject) {
   // build-tools is not ready for primetime with these...
   tasks.named("dependencyLicenses").configure { it.enabled = false }
   dependenciesInfo.enabled = false
-  disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures')
+  disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest')
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
   if (org.opensearch.gradle.info.BuildParams.inFipsJvm) {
@@ -241,12 +250,6 @@ if (project != rootProject) {
     }
   }
 
-  // disable fail-on-warnings for this specific task which trips Java 11 bug
-  // https://bugs.openjdk.java.net/browse/JDK-8209058
-  tasks.named("compileTestFixturesJava").configure {
-    options.compilerArgs -= '-Werror'
-  }
-
   tasks.register("integTest", Test) {
     inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE)
     systemProperty 'test.version_under_test', version

buildSrc/reaper/build.gradle

@@ -11,8 +11,10 @@
 
 apply plugin: 'java'
 
-targetCompatibility = JavaVersion.VERSION_11
-sourceCompatibility = JavaVersion.VERSION_11
+java {
+  targetCompatibility = JavaVersion.VERSION_11
+  sourceCompatibility = JavaVersion.VERSION_11
+}
 
 jar {
   archiveFileName = "${project.name}.jar"

buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -44,6 +44,8 @@ import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.plugins.BasePlugin
+import org.gradle.api.plugins.BasePluginExtension
+import org.gradle.api.plugins.JavaPluginExtension
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.publish.maven.tasks.GenerateMavenPom
@@ -85,7 +87,9 @@ class PluginBuildPlugin implements Plugin<Project> {
             PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class)
             configurePublishing(project, extension1)
             String name = extension1.name
-            project.archivesBaseName = name
+
+            BasePluginExtension base = project.getExtensions().findByType(BasePluginExtension.class)
+            base.archivesBaseName = name
             project.description = extension1.description
 
             if (extension1.name == null) {
@@ -98,12 +102,13 @@ class PluginBuildPlugin implements Plugin<Project> {
                 throw new InvalidUserDataException('classname is a required setting for opensearchplugin')
             }
 
+            JavaPluginExtension java = project.getExtensions().findByType(JavaPluginExtension.class)
             Map<String, String> properties = [
                     'name'                : extension1.name,
                     'description'         : extension1.description,
                     'version'             : extension1.version,
                     'opensearchVersion'   : Version.fromString(VersionProperties.getOpenSearch()).toString(),
-                    'javaVersion'         : project.targetCompatibility as String,
+                    'javaVersion'         : java.targetCompatibility as String,
                     'classname'           : extension1.classname,
                     'customFolderName'    : extension1.customFolderName,
                     'extendedPlugins'     : extension1.extendedPlugins.join(','),
@@ -156,8 +161,9 @@ class PluginBuildPlugin implements Plugin<Project> {
             }
             // always configure publishing for client jars
             project.publishing.publications.nebula(MavenPublication).artifactId(extension.name + "-client")
+            final BasePluginExtension base = project.getExtensions().findByType(BasePluginExtension.class)
             project.tasks.withType(GenerateMavenPom.class).configureEach { GenerateMavenPom generatePOMTask ->
-                generatePOMTask.destination = "${project.buildDir}/distributions/${project.archivesBaseName}-client-${project.versions.opensearch}.pom"
+                generatePOMTask.destination = "${project.buildDir}/distributions/${base.archivesBaseName}-client-${project.versions.opensearch}.pom"
             }
         } else {
             if (project.plugins.hasPlugin(MavenPublishPlugin)) {

buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java

@@ -46,7 +46,7 @@
 import org.gradle.api.XmlProvider;
 import org.gradle.api.artifacts.ProjectDependency;
 import org.gradle.api.plugins.BasePlugin;
-import org.gradle.api.plugins.BasePluginConvention;
+import org.gradle.api.plugins.BasePluginExtension;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.publish.PublishingExtension;
 import org.gradle.api.publish.maven.MavenPublication;
@@ -77,7 +77,7 @@ public void apply(Project project) {
     }
 
     private static String getArchivesBaseName(Project project) {
-        return project.getConvention().getPlugin(BasePluginConvention.class).getArchivesBaseName();
+        return project.getExtensions().getByType(BasePluginExtension.class).getArchivesBaseName();
     }
 
     /**Configuration generation of maven poms. */

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java

@@ -36,7 +36,7 @@
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
@@ -106,8 +106,8 @@ public ForbiddenPatternsTask() {
     @IgnoreEmptyDirectories
     @PathSensitive(PathSensitivity.RELATIVE)
     public FileCollection getFiles() {
-        return getProject().getConvention()
-            .getPlugin(JavaPluginConvention.class)
+        return getProject().getExtensions()
+            .getByType(JavaPluginExtension.class)
             .getSourceSets()
             .stream()
             .map(sourceSet -> sourceSet.getAllSource().matching(filesFilter))

buildSrc/src/main/java/org/opensearch/gradle/precommit/LoggerUsageTask.java

@@ -34,7 +34,7 @@
 
 import org.opensearch.gradle.LoggedExec;
 import org.gradle.api.file.FileCollection;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Classpath;
 import org.gradle.api.tasks.IgnoreEmptyDirectories;
@@ -82,8 +82,8 @@ public void setClasspath(FileCollection classpath) {
     @SkipWhenEmpty
     @IgnoreEmptyDirectories
     public FileCollection getClassDirectories() {
-        return getProject().getConvention()
-            .getPlugin(JavaPluginConvention.class)
+        return getProject().getExtensions()
+            .getByType(JavaPluginExtension.class)
             .getSourceSets()
             .stream()
             // Don't pick up all source sets like the java9 ones as logger-check doesn't support the class format

buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestApiTask.java

@@ -41,7 +41,7 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
@@ -240,7 +240,7 @@ private File getTestOutputResourceDir() {
 
     private Optional<SourceSet> getSourceSet() {
         Project project = getProject();
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(getSourceSetName()));
     }

buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestTestsTask.java

@@ -41,7 +41,7 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
@@ -178,7 +178,7 @@ void copy() {
 
     private Optional<SourceSet> getSourceSet() {
         Project project = getProject();
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(getSourceSetName()));
     }

buildSrc/src/main/java/org/opensearch/gradle/util/GradleUtils.java

@@ -40,7 +40,7 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.plugins.JavaBasePlugin;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.services.BuildService;
 import org.gradle.api.services.BuildServiceRegistration;
@@ -68,7 +68,7 @@ public static <T> Action<T> noop() {
     }
 
     public static SourceSetContainer getJavaSourceSets(Project project) {
-        return project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets();
+        return project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets();
     }
 
     public static <T extends Task> TaskProvider<T> maybeRegister(TaskContainer tasks, String name, Class<T> clazz, Action<T> action) {