Merge branch 'elastic:main' into batch-search-requests-per-node-perli…

…minary-work
piergm · Nov 18, 2024 · 1885d50 · 1885d50
2 parents d87d2c0 + 27eb013
commit 1885d50
Show file tree

Hide file tree

Showing 300 changed files with 7,036 additions and 1,266 deletions.
diff --git a/.buildkite/scripts/gradle-configuration-cache-validation.sh b/.buildkite/scripts/gradle-configuration-cache-validation.sh
@@ -2,18 +2,17 @@
 
 set -euo pipefail
 
-# TODO/ FIXIT without a full resolved gradle home, we see issues configuration cache reuse
-./gradlew --max-workers=8 --parallel --scan --no-daemon precommit
+# This is a workaround for https://github.com/gradle/gradle/issues/28159
+.ci/scripts/run-gradle.sh --no-daemon precommit
 
-./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
 
 # Create a temporary file
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
 echo "2nd run"
-# TODO run-gradle.sh script causes issues because of init script handling
-./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
 
 # Check if the command was successful
 if grep -q "Configuration cache entry reused." $tmpOutputFile; then

diff --git a/TESTING.asciidoc b/TESTING.asciidoc
@@ -472,7 +472,7 @@ You can run a group of YAML test by using wildcards:
   --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=index/*/*}"
 ---------------------------------------------------------------------------
 
-or 
+or
 
 ---------------------------------------------------------------------------
 ./gradlew :rest-api-spec:yamlRestTest \
@@ -564,8 +564,8 @@ Sometimes a backward compatibility change spans two versions.
 A common case is a new functionality that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
 Another use case, since the introduction of serverless, is to test BWC against main in addition to the other released branches.
 To do so, specify the `bwc.refspec` remote and branch to use for the BWC build as `origin/main`.
-To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java], 
-increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp 
+To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java],
+increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp
 in link:./x-pack/plugin/ml/build.gradle[ml/build.gradle].
 
 In general, to test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of pulling the release branch from GitHub.
@@ -625,7 +625,7 @@ For specific YAML rest tests one can use
 For disabling entire types of tests for subprojects, one can use for example:
 
 ------------------------------------------------
-if (BuildParams.inFipsJvm){
+if (buildParams.inFipsJvm) {
   // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
   tasks.named("javaRestTest").configure{enabled = false }
 }

diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
@@ -1,4 +1,3 @@
-import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.TestUtil
 
 /*
@@ -78,7 +77,7 @@ tasks.register("copyPainless", Copy) {
 }
 
 tasks.named("run").configure {
-  executable = "${BuildParams.runtimeJavaHome}/bin/java"
+  executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
   dependsOn "copyExpression", "copyPainless", configurations.nativeLib
   systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())

diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle
@@ -12,9 +12,6 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
 buildscript {
   repositories {
-    maven {
-      url 'https://jitpack.io'
-    }
     mavenCentral()
   }
 }
@@ -70,10 +67,6 @@ gradlePlugin {
 }
 
 repositories {
-    maven {
-      url 'https://jitpack.io'
-    }
-
     mavenCentral()
     gradlePluginPortal()
 }

diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java
@@ -16,4 +16,12 @@ public abstract class GUtils {
     public static String capitalize(String s) {
         return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
     }
+
+    public static <T> T elvis(T given, T fallback) {
+        if (given == null) {
+            return fallback;
+        } else {
+            return given;
+        }
+    }
 }
diff --git a/...onventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java b/...onventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java
@@ -9,15 +9,19 @@
 
 package org.elasticsearch.gradle.internal.conventions;
 
-import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin;
+import groovy.util.Node;
+
 import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
-import groovy.util.Node;
-import org.elasticsearch.gradle.internal.conventions.util.Util;
+
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
+import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin;
+import org.elasticsearch.gradle.internal.conventions.util.Util;
+import org.gradle.api.Action;
 import org.gradle.api.NamedDomainObjectSet;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.Task;
 import org.gradle.api.XmlProvider;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.plugins.BasePlugin;
@@ -35,11 +39,12 @@
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.initialization.layout.BuildLayout;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
+import org.w3c.dom.Element;
 
-import javax.inject.Inject;
 import java.io.File;
 import java.util.Map;
 import java.util.concurrent.Callable;
+import javax.inject.Inject;
 
 public class PublishPlugin implements Plugin<Project> {
 
@@ -113,19 +118,22 @@ private void configurePomGeneration(Project project) {
         var archivesBaseName = providerFactory.provider(() -> getArchivesBaseName(extensions));
         var projectVersion = providerFactory.provider(() -> project.getVersion());
         var generateMavenPoms = project.getTasks().withType(GenerateMavenPom.class);
-        generateMavenPoms.configureEach(
-            pomTask -> pomTask.setDestination(
+        generateMavenPoms.configureEach(pomTask -> {
+            pomTask.setDestination(
                 (Callable<String>) () -> String.format(
                     "%s/distributions/%s-%s.pom",
                     projectLayout.getBuildDirectory().get().getAsFile().getPath(),
                     archivesBaseName.get(),
                     projectVersion.get()
                 )
-            )
-        );
+            );
+            pomTask.doFirst(t -> pomTask.getPom().withXml(xml -> formatDependencies(xml)));
+        });
+
         var publishing = extensions.getByType(PublishingExtension.class);
         final var mavenPublications = publishing.getPublications().withType(MavenPublication.class);
-        addNameAndDescriptiontoPom(project, mavenPublications);
+
+        addNameAndDescriptionToPom(project, mavenPublications);
         mavenPublications.configureEach(publication -> {
             // Add git origin info to generated POM files for internal builds
             publication.getPom().withXml(xml -> addScmInfo(xml, gitInfo.get()));
@@ -135,11 +143,26 @@ private void configurePomGeneration(Project project) {
         });
     }
 
-    private void addNameAndDescriptiontoPom(Project project, NamedDomainObjectSet<MavenPublication> mavenPublications) {
+    /**
+     * just ensure we put dependencies to the end. more a cosmetic thing than anything else
+     * */
+    private void formatDependencies(XmlProvider xml) {
+        Element rootElement = xml.asElement();
+        var dependencies = rootElement.getElementsByTagName("dependencies");
+        if (dependencies.getLength() == 1 && dependencies.item(0) != null) {
+            org.w3c.dom.Node item = dependencies.item(0);
+            rootElement.removeChild(item);
+            rootElement.appendChild(item);
+        }
+    }
+
+    private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet<MavenPublication> mavenPublications) {
         var name = project.getName();
         var description = providerFactory.provider(() -> project.getDescription() != null ? project.getDescription() : "");
         mavenPublications.configureEach(p -> p.getPom().withXml(xml -> {
             var root = xml.asNode();
+            // Node versionNode = root.get("version");
+            // versionNode.plus(1, "name", name);
             root.appendNode("name", name);
             root.appendNode("description", description.get());
         }));

diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
@@ -258,9 +258,6 @@ tasks.named('licenseHeaders').configure {
  *****************************************************************************/
 
 repositories {
-  maven {
-    url 'https://jitpack.io'
-  }
   mavenCentral()
   gradlePluginPortal()
 }
@@ -386,10 +383,13 @@ tasks.named("jar") {
 
 spotless {
   java {
-    // IDEs can sometimes run annotation processors that leave files in
-    // here, causing Spotless to complain. Even though this path ought not
-    // to exist, exclude it anyway in order to avoid spurious failures.
-    toggleOffOn()
+
+    // workaround for https://github.com/diffplug/spotless/issues/2317
+    //toggleOffOn()
+    target project.fileTree("src/main/java") {
+      include '**/*.java'
+      exclude '**/DockerBase.java'
+    }
   }
 }
 

diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle
@@ -1,8 +1,5 @@
 pluginManagement {
   repositories {
-    maven {
-      url 'https://jitpack.io'
-    }
     mavenCentral()
     gradlePluginPortal()
   }

diff --git a/...egTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy b/...egTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy
@@ -73,7 +73,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
             buildFile << """
                 plugins {
                     id 'elasticsearch.java-doc'
-                    id 'com.github.johnrengelman.shadow' version '7.1.2'
+                    id 'com.gradleup.shadow'
                     id 'java'
                 }
                 group = 'org.acme.depending'

diff --git a/...roovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/...roovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy
@@ -9,16 +9,10 @@
 
 package org.elasticsearch.gradle.internal
 
-import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
-import spock.lang.IgnoreIf
 import spock.lang.Unroll
 
-/*
- * Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM
- */
-@IgnoreIf({ Architecture.current() == Architecture.AARCH64 })
 class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
     def setup() {

diff --git a/...ernal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/...ernal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy
@@ -96,7 +96,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
             plugins {
                 id 'elasticsearch.java'
                 id 'elasticsearch.publish'
-                id 'com.github.johnrengelman.shadow'
+                id 'com.gradleup.shadow'
             }
 
             repositories {
@@ -117,7 +117,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
             }
             version = "1.0"
             group = 'org.acme'
-            description = 'some description'
+            description = 'shadowed project'
         """
 
         when:
@@ -137,7 +137,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
               <artifactId>hello-world</artifactId>
               <version>1.0</version>
               <name>hello-world</name>
-              <description>some description</description>
+              <description>shadowed project</description>
               <url>unknown</url>
               <scm>
                 <url>unknown</url>
@@ -186,7 +186,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
             plugins {
                 id 'elasticsearch.java'
                 id 'elasticsearch.publish'
-                id 'com.github.johnrengelman.shadow'
+                id 'com.gradleup.shadow'
             }
 
             dependencies {
@@ -206,7 +206,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
                 group = 'org.acme'
             }
 
-            description = 'some description'
+            description = 'with shadowed dependencies'
         """
 
         when:
@@ -226,7 +226,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
               <artifactId>hello-world</artifactId>
               <version>1.0</version>
               <name>hello-world</name>
-              <description>some description</description>
+              <description>with shadowed dependencies</description>
               <url>unknown</url>
               <scm>
                 <url>unknown</url>
@@ -277,13 +277,13 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
             plugins {
                 id 'elasticsearch.internal-es-plugin'
                 id 'elasticsearch.publish'
-                id 'com.github.johnrengelman.shadow'
+                id 'com.gradleup.shadow'
             }
 
             esplugin {
                 name = 'hello-world-plugin'
                 classname 'org.acme.HelloWorldPlugin'
-                description = "custom project description"
+                description = "shadowed es plugin"
             }
 
             publishing {
@@ -324,7 +324,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
               <artifactId>hello-world-plugin</artifactId>
               <version>1.0</version>
               <name>hello-world</name>
-              <description>custom project description</description>
+              <description>shadowed es plugin</description>
               <url>unknown</url>
               <scm>
                 <url>unknown</url>
@@ -353,7 +353,6 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
                   <url>https://www.elastic.co</url>
                 </developer>
               </developers>
-              <dependencies/>
             </project>"""
         )
     }
@@ -440,8 +439,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         // scm info only added for internal builds
         internalBuild()
         buildFile << """
-            BuildParams.init { it.setGitOrigin("https://some-repo.com/repo.git") }
-
+            buildParams.getGitOriginProperty().set("https://some-repo.com/repo.git")
             apply plugin:'elasticsearch.java'
             apply plugin:'elasticsearch.publish'
 

diff --git a/...rg/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/...rg/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy
@@ -161,7 +161,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna
             },
             "target": {
                 "remoteUrl": "http://acme.org",
-                "branch": "unknown"
+                "branch": "$version"
             },
             "targetReference": "$version",
             "projectAttributes": {

diff --git a/...roovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy b/...roovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
@@ -10,6 +10,7 @@
 package org.elasticsearch.gradle.internal.test.rest
 
 import spock.lang.IgnoreIf
+import spock.lang.IgnoreRest
 
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
@@ -20,16 +21,16 @@ import org.gradle.testkit.runner.TaskOutcome
 class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def setup() {
+        configurationCacheCompatible = true
         buildApiRestrictionsDisabled = true
     }
 
 
     def "yamlRestTest does nothing when there are no tests"() {
         given:
+        internalBuild()
         buildFile << """
-        plugins {
-          id 'elasticsearch.legacy-yaml-rest-test'
-        }
+            apply plugin: 'elasticsearch.legacy-yaml-rest-test'
         """
 
         when:
@@ -136,7 +137,7 @@ class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         """
 
         when:
-        def result = gradleRunner("yamlRestTest", "--console", 'plain', '--stacktrace').buildAndFail()
+        def result = gradleRunner("yamlRestTest", "--console", 'plain').buildAndFail()
 
         then:
         result.task(":distribution:archives:integ-test-zip:buildExpanded").outcome == TaskOutcome.SUCCESS