From fcc5d37dcfc9310e34d50cde110e4aa8aed7cfe3 Mon Sep 17 00:00:00 2001
From: Rene Groeschke <rene@elastic.co>
Date: Wed, 7 Aug 2024 19:33:29 +0200
Subject: [PATCH] [Gradle] Add buildkite ci validation script for configuration
 cache (#110269)

We see gradle cache incompatibilities sneaking into our code base. Until
we can enable this feature by default we should regular run checks
existing configuration cache achievements are not degrading.

This change includes: - updating spotless gradle plugin to configuration
cache compatible version (its still BETA but imo good enough for our
needs)
---
 ...on.sh => gradle-build-cache-validation.sh} |  5 ++--
 .../gradle-configuration-cache-validation.sh  | 27 +++++++++++++++++++
 .../ElasticsearchBuildCompletePlugin.java     | 12 ++++++++-
 x-pack/plugin/esql/build.gradle               |  6 +++--
 x-pack/plugin/esql/compute/build.gradle       |  4 ++-
 5 files changed, 48 insertions(+), 6 deletions(-)
 rename .buildkite/scripts/{gradle-cache-validation.sh => gradle-build-cache-validation.sh} (97%)
 create mode 100755 .buildkite/scripts/gradle-configuration-cache-validation.sh

diff --git a/.buildkite/scripts/gradle-cache-validation.sh b/.buildkite/scripts/gradle-build-cache-validation.sh
similarity index 97%
rename from .buildkite/scripts/gradle-cache-validation.sh
rename to .buildkite/scripts/gradle-build-cache-validation.sh
index fbb957bc3b26b..75dc9b264b8bc 100755
--- a/.buildkite/scripts/gradle-cache-validation.sh
+++ b/.buildkite/scripts/gradle-build-cache-validation.sh
@@ -12,17 +12,18 @@ curl -s -L -O https://github.com/gradle/gradle-enterprise-build-validation-scrip
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
+set +e
 gradle-enterprise-gradle-build-validation/03-validate-local-build-caching-different-locations.sh -r https://github.com/elastic/elasticsearch.git -b $BUILDKITE_BRANCH --gradle-enterprise-server https://gradle-enterprise.elastic.co -t precommit --fail-if-not-fully-cacheable | tee $tmpOutputFile
-
 # Capture the return value
 retval=$?
+set -e
 
 # Now read the content from the temporary file into a variable
 perfOutput=$(cat $tmpOutputFile | sed -n '/Performance Characteristics/,/See https:\/\/gradle.com\/bvs\/main\/Gradle.md#performance-characteristics for details./p' | sed '$d' | sed 's/\x1b\[[0-9;]*m//g')
 investigationOutput=$(cat $tmpOutputFile | sed -n '/Investigation Quick Links/,$p' | sed 's/\x1b\[[0-9;]*m//g')
 
 # Initialize HTML output variable
-summaryHtml="<h4>Performance Characteristics</h4>"
+summaryHtml="<h4>Build Cache Performance Characteristics</h4>"
 summaryHtml+="<ul>"
 
 # Process each line of the string
diff --git a/.buildkite/scripts/gradle-configuration-cache-validation.sh b/.buildkite/scripts/gradle-configuration-cache-validation.sh
new file mode 100755
index 0000000000000..8249155c5ffc5
--- /dev/null
+++ b/.buildkite/scripts/gradle-configuration-cache-validation.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# TODO/ FIXIT without a full resolved gradle home, we see issues configuration cache reuse
+./gradlew --max-workers=8 --parallel --scan --no-daemon precommit
+
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+
+# Create a temporary file
+tmpOutputFile=$(mktemp)
+trap "rm $tmpOutputFile" EXIT
+
+echo "2nd run"
+# TODO run-gradle.sh script causes issues because of init script handling
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+
+# Check if the command was successful
+if grep -q "Configuration cache entry reused." $tmpOutputFile; then
+    echo "Gradle configuration cache reused"
+    exit 0
+else
+    echo "Failed to reuse Gradle configuration cache."
+    exit 1
+fi
+
+
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
index b513fd7b93631..b27f480df4e63 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -61,7 +61,7 @@ public void apply(Project target) {
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
         if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
-            File targetFile = target.file("build/" + buildNumber + ".tar.bz2");
+            File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
             DevelocityConfiguration extension = target.getExtensions().getByType(DevelocityConfiguration.class);
@@ -86,9 +86,19 @@ public void apply(Project target) {
         }
     }
 
+    private File calculateTargetFile(Project target, String buildNumber) {
+        File uploadFile = target.file("build/" + buildNumber + ".tar.bz2");
+        int artifactIndex = 1;
+        while (uploadFile.exists()) {
+            uploadFile = target.file("build/" + buildNumber + "-" + artifactIndex++ + ".tar.bz2");
+        }
+        return uploadFile;
+    }
+
     private List<File> resolveProjectLogs(File projectDir) {
         var projectDirFiles = getFileOperations().fileTree(projectDir);
         projectDirFiles.include("**/*.hprof");
+        projectDirFiles.include("**/build/reports/configuration-cache/**");
         projectDirFiles.include("**/build/test-results/**/*.xml");
         projectDirFiles.include("**/build/testclusters/**");
         projectDirFiles.include("**/build/testrun/*/temp/**");
diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index adecd78c72c2c..26cf53b334b1e 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -50,10 +50,12 @@ dependencies {
   internalClusterTestImplementation project(":modules:mapper-extras")
 }
 
+def projectDirectory = project.layout.projectDirectory
+def generatedSourceDir = projectDirectory.dir("src/main/generated")
 tasks.named("compileJava").configure {
-  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(project.layout.projectDirectory.dir("src/main/generated")))
+  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(generatedSourceDir))
   // IntelliJ sticks generated files here and we can't stop it....
-  exclude { it.file.toString().contains("$projectDir/src/main/generated-src/generated") }
+  exclude { it.file.toString().contains("src/main/generated-src/generated") }
 }
 
 interface Injected {
diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle
index 3a42f88b2c7ba..ccf93a277a50d 100644
--- a/x-pack/plugin/esql/compute/build.gradle
+++ b/x-pack/plugin/esql/compute/build.gradle
@@ -18,8 +18,10 @@ dependencies {
   testImplementation(project(xpackModule('esql-core')))
 }
 
+def projectDirectory = project.layout.projectDirectory
+def generatedSourceDir = projectDirectory.dir("src/main/generated")
 tasks.named("compileJava").configure {
-  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(project.layout.projectDirectory.dir("src/main/generated")))
+  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(generatedSourceDir))
 }
 
 tasks.named('checkstyleMain').configure {