Merge remote-tracking branch 'upstream/main' into lucene_snapshot

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.20", "8.13.1", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -529,8 +529,8 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
-      - label: "{{matrix.image}} / 8.13.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.1
+      - label: "{{matrix.image}} / 8.13.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -543,7 +543,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.1
+          BWC_VERSION: 8.13.2
 
       - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0

.buildkite/pipelines/periodic.template.yml

@@ -85,7 +85,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -108,7 +107,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             BWC_VERSION: $BWC_LIST
         agents:
           provider: gcp

.buildkite/pipelines/periodic.yml

@@ -312,16 +312,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.2
-      - label: 8.13.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.1#bwcTest
+      - label: 8.13.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.1
+          BWC_VERSION: 8.13.2
       - label: 8.14.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.0#bwcTest
         timeout_in_minutes: 300
@@ -396,7 +396,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.20", "8.13.1", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -416,7 +416,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -439,8 +438,7 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
-            BWC_VERSION: ["7.17.20", "8.13.1", "8.14.0"]
+            BWC_VERSION: ["7.17.20", "8.13.2", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -30,5 +30,5 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.4"
   - "8.12.2"
-  - "8.13.1"
+  - "8.13.2"
   - "8.14.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.20"
-  - "8.13.1"
+  - "8.13.2"
   - "8.14.0"

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.server.cli;
 
-import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
@@ -18,11 +17,13 @@
 import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.IntStream;
 
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
@@ -41,7 +42,6 @@
 
 @WithoutSecurityManager
 @SuppressFileSystems("*")
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106554")
 public class JvmErgonomicsTests extends ESTestCase {
 
     public void testExtractValidHeapSizeUsingXmx() throws Exception {
@@ -193,18 +193,21 @@ public void testConcGCThreadsNotSetBasedOnProcessors() throws Exception {
         Settings.Builder nodeSettingsBuilder = Settings.builder()
             .put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.SEARCH_ROLE.roleName());
         if (randomBoolean()) {
-            nodeSettingsBuilder.put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), randomBoolean() ? between(1, 3) : between(6, 100));
+            int maxProcessors = Runtime.getRuntime().availableProcessors();
+            List<Integer> possibleProcessors = new ArrayList<>();
+            IntStream.range(1, maxProcessors + 1).filter(i -> i < 4 || i > 5).forEach(possibleProcessors::add);
+            nodeSettingsBuilder.put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), randomFrom(possibleProcessors));
         }
         assertThat(JvmErgonomics.choose(List.of(), nodeSettingsBuilder.build()), everyItem(not(startsWith("-XX:ConcGCThreads="))));
     }
 
     public void testConcGCThreadsNotSetBasedOnRoles() throws Exception {
         Settings.Builder nodeSettingsBuilder = Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), between(4, 5));
         if (randomBoolean()) {
-            nodeSettingsBuilder.put(
-                NodeRoleSettings.NODE_ROLES_SETTING.getKey(),
-                randomValueOtherThan(DiscoveryNodeRole.SEARCH_ROLE, () -> randomFrom(DiscoveryNodeRole.roles())).roleName()
-            );
+            List<DiscoveryNodeRole> possibleRoles = new ArrayList<>(DiscoveryNodeRole.roles());
+            possibleRoles.remove(DiscoveryNodeRole.SEARCH_ROLE);
+            possibleRoles.remove(DiscoveryNodeRole.VOTING_ONLY_NODE_ROLE);
+            nodeSettingsBuilder.put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), randomFrom(possibleRoles).roleName());
         }
         assertThat(JvmErgonomics.choose(List.of(), nodeSettingsBuilder.build()), everyItem(not(startsWith("-XX:ConcGCThreads="))));
 
@@ -228,14 +231,17 @@ public void testMinimumNewSizeNotSetBasedOnHeap() throws Exception {
     }
 
     public void testMinimumNewSizeNotSetBasedOnRoles() throws Exception {
-        Settings nodeSettings = randomBoolean()
-            ? Settings.EMPTY
-            : Settings.builder()
-                .put(
-                    NodeRoleSettings.NODE_ROLES_SETTING.getKey(),
-                    randomValueOtherThan(DiscoveryNodeRole.SEARCH_ROLE, () -> randomFrom(DiscoveryNodeRole.roles())).roleName()
-                )
+        Settings nodeSettings;
+        if (randomBoolean()) {
+            nodeSettings = Settings.EMPTY;
+        } else {
+            List<DiscoveryNodeRole> possibleRoles = new ArrayList<>(DiscoveryNodeRole.roles());
+            possibleRoles.remove(DiscoveryNodeRole.SEARCH_ROLE);
+            possibleRoles.remove(DiscoveryNodeRole.VOTING_ONLY_NODE_ROLE);
+            nodeSettings = Settings.builder()
+                .put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), randomFrom(possibleRoles).roleName())
                 .build();
+        }
         List<String> chosen = JvmErgonomics.choose(List.of("-Xmx" + between(1, 4) + "g"), nodeSettings);
         assertThat(chosen, everyItem(not(is("-XX:+UnlockExperimentalVMOptions"))));
         assertThat(chosen, everyItem(not(startsWith("-XX:G1NewSizePercent="))));

docs/changelog/105168.yaml

@@ -0,0 +1,5 @@
+pr: 105168
+summary: Add ?master_timeout query parameter to ccr apis
+area: CCR
+type: bug
+issues: []

docs/changelog/106378.yaml

@@ -0,0 +1,5 @@
+pr: 106378
+summary: Add Cohere rerank to `_inference` service
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/106503.yaml

@@ -0,0 +1,5 @@
+pr: 106503
+summary: "Support ST_CONTAINS and ST_WITHIN"
+area: "ES|QL"
+type: enhancement
+issues: []

docs/changelog/106516.yaml

@@ -0,0 +1,5 @@
+pr: 106516
+summary: "ESQL: perform a reduction on the data node"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106579.yaml

@@ -0,0 +1,5 @@
+pr: 106579
+summary: "ESQL: Allow grouping key inside stats expressions"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106673.yaml

@@ -0,0 +1,6 @@
+pr: 106673
+summary: "ESQL: Fix fully pruned aggregates"
+area: ES|QL
+type: bug
+issues:
+ - 106427

docs/changelog/106714.yaml

@@ -0,0 +1,5 @@
+pr: 106714
+summary: Add non-indexed fields to ecs templates
+area: Data streams
+type: bug
+issues: []

docs/changelog/106720.yaml

@@ -0,0 +1,5 @@
+pr: 106720
+summary: "ESQL: Fix treating all fields as MV in COUNT pushdown"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/106824.yaml

@@ -0,0 +1,5 @@
+pr: 106824
+summary: "ESQL: Introduce language versioning to REST API"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106836.yaml

@@ -0,0 +1,5 @@
+pr: 106836
+summary: Make int8_hnsw our default index for new dense-vector fields
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/106852.yaml

@@ -0,0 +1,6 @@
+pr: 106852
+summary: Introduce ordinal bytesref block
+area: ES|QL
+type: enhancement
+issues:
+ - 106387

docs/changelog/106860.yaml

@@ -0,0 +1,5 @@
+pr: 106860
+summary: "[Profiling] Add TopN Functions API"
+area: Application
+type: enhancement
+issues: []

docs/changelog/106866.yaml

@@ -0,0 +1,5 @@
+pr: 106866
+summary: Add ES|QL signum function
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106934.yaml

@@ -0,0 +1,5 @@
+pr: 106934
+summary: Adjust array resizing in block builder
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106952.yaml

@@ -0,0 +1,5 @@
+pr: 106952
+summary: Add Lucene spanish plural stemmer
+area: Search
+type: enhancement
+issues: []

docs/changelog/106990.yaml

@@ -0,0 +1,5 @@
+pr: 106990
+summary: Address concurrency issue in top hits aggregation
+area: Aggregations
+type: bug
+issues: []

docs/changelog/107016.yaml

@@ -0,0 +1,5 @@
+pr: 107016
+summary: "ESQL: Enable VALUES agg for datetime"
+area: Aggregations
+type: bug
+issues: []

docs/changelog/107038.yaml

@@ -0,0 +1,5 @@
+pr: 107038
+summary: Replace `UnsupportedOperationException` with `IllegalArgumentException` for non-existing columns
+area: Search
+type: bug
+issues: []

docs/changelog/107046.yaml

@@ -0,0 +1,6 @@
+pr: 107046
+summary: "[Security Solution] Add `read` permission for third party agent indices\
+  \ for `kibana_system`"
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/107054.yaml

@@ -0,0 +1,6 @@
+pr: 107054
+summary: Query API Keys support for both `aggs` and `aggregations` keywords
+area: Security
+type: enhancement
+issues:
+ - 106839

docs/changelog/107059.yaml

@@ -0,0 +1,5 @@
+pr: 107059
+summary: "[Connector API] Support numeric for configuration select option value type"
+area: Application
+type: bug
+issues: []