From fba1cd509bc712f7f0be262ef329420675439664 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 9 Oct 2023 22:46:06 +0200
Subject: [PATCH 01/52] buildkite test on Ubuntu

---
 .buildkite/pipeline.yml | 70 ++++++++++++++++++++++++-----------------
 1 file changed, 42 insertions(+), 28 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8ff198708fd..a01c8ed35c8 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,7 +4,7 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
-  - label: "Unit tests"
+  - label: "Unit tests - Ubuntu 22.04"
     key: "unit-tests"
     command: ".buildkite/scripts/steps/unit-tests.sh"
     artifact_paths:
@@ -12,40 +12,54 @@ steps:
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
+      image: "ubuntu-2204-lts"
     retry:
       manual:
         allowed: true
 
-  - label: ":sonarqube: Continuous Code Inspection"
-    env:
-      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+  - label: "Unit tests - Ubuntu 22.04 ARM64"
+    key: "unit-tests"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
     agents:
-      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
-    command:
-      - "buildkite-agent artifact download build/TEST-go-unit.cov ."
-      - "buildkite-agent artifact download build/TEST-go-unit.out.json ."
-      - "/scan-source-code.sh"
-    depends_on:
-      - "unit-tests"
+      provider: "aws"
+      image: "ubuntu-2204-aarch64"
     retry:
       manual:
         allowed: true
 
-  - label: "Integration tests"
-    key: "integration-tests"
-    command: ".buildkite/scripts/steps/integration_tests.sh"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
+  # - label: ":sonarqube: Continuous Code Inspection"
+  #   env:
+  #     VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+  #   agents:
+  #     image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+  #   command:
+  #     - "buildkite-agent artifact download build/TEST-go-unit.cov ."
+  #     - "buildkite-agent artifact download build/TEST-go-unit.out.json ."
+  #     - "/scan-source-code.sh"
+  #   depends_on:
+  #     - "unit-tests"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
-  - wait: ~
-    continue_on_failure: true
-  - label: "Processing test results"
-    agents:
-      provider: "gcp"
-    plugins:
-      - junit-annotate#v2.4.1:
-          artifacts: build/TEST-go-*.xml
+  # - label: "Integration tests"
+  #   key: "integration-tests"
+  #   command: ".buildkite/scripts/steps/integration_tests.sh"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #   agents:
+  #     provider: "gcp"
+  #     machineType: "n1-standard-8"
+
+  # - wait: ~
+  #   continue_on_failure: true
+  # - label: "Processing test results"
+  #   agents:
+  #     provider: "gcp"
+  #   plugins:
+  #     - junit-annotate#v2.4.1:
+  #         artifacts: build/TEST-go-*.xml

From 0b6c3dc32e54c28ad3ba646b837ef2a9235f53d2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 10 Oct 2023 02:12:16 +0200
Subject: [PATCH 02/52] Fixed step ids

---
 .buildkite/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index a01c8ed35c8..3bae56e329b 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -5,7 +5,7 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
   - label: "Unit tests - Ubuntu 22.04"
-    key: "unit-tests"
+    key: "unit-tests-2204"
     command: ".buildkite/scripts/steps/unit-tests.sh"
     artifact_paths:
       - "build/TEST-**"
@@ -18,7 +18,7 @@ steps:
         allowed: true
 
   - label: "Unit tests - Ubuntu 22.04 ARM64"
-    key: "unit-tests"
+    key: "unit-tests-2204-arm64"
     command: ".buildkite/scripts/steps/unit-tests.sh"
     artifact_paths:
       - "build/TEST-**"

From 6eb573ad846daeca55843c0722879b376242abdc Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 10 Oct 2023 14:58:11 +0200
Subject: [PATCH 03/52] fix image family

---
 .buildkite/pipeline.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 3bae56e329b..d65427108a0 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -12,7 +12,7 @@ steps:
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
-      image: "ubuntu-2204-lts"
+      image: "family/core-ubuntu-2204"
     retry:
       manual:
         allowed: true
@@ -24,8 +24,8 @@ steps:
       - "build/TEST-**"
       - "build/diagnostics/*"
     agents:
-      provider: "aws"
-      image: "ubuntu-2204-aarch64"
+      provider: "gcp"
+      image: "family/core-ubuntu-2204-aarch64"
     retry:
       manual:
         allowed: true

From 31ece98be92d30058349796ab3a298b06392a885 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 10 Oct 2023 15:06:43 +0200
Subject: [PATCH 04/52] Use aws from arm64

---
 .buildkite/pipeline.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index d65427108a0..7a3755b0b24 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -24,8 +24,10 @@ steps:
       - "build/TEST-**"
       - "build/diagnostics/*"
     agents:
-      provider: "gcp"
-      image: "family/core-ubuntu-2204-aarch64"
+      provider: "aws"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 200
+      instanceType: "m6g.4xlarge"
     retry:
       manual:
         allowed: true

From 75711d344a14787a27d639dd1df9df1a4f4bcf3e Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 01:22:02 +0200
Subject: [PATCH 05/52] try windows post-checkout hook

---
 .buildkite/hooks/post-checkout.ps1 | 57 ++++++++++++++++++++++++++++++
 .buildkite/hooks/pre-command.ps1   |  0
 .buildkite/pipeline.yml            | 47 +++++++++++++++---------
 3 files changed, 87 insertions(+), 17 deletions(-)
 create mode 100644 .buildkite/hooks/post-checkout.ps1
 create mode 100644 .buildkite/hooks/pre-command.ps1

diff --git a/.buildkite/hooks/post-checkout.ps1 b/.buildkite/hooks/post-checkout.ps1
new file mode 100644
index 00000000000..14ca754bc7a
--- /dev/null
+++ b/.buildkite/hooks/post-checkout.ps1
@@ -0,0 +1,57 @@
+# Set error handling
+$ErrorActionPreference = "Stop"
+
+# Define a function to checkout and merge
+function Checkout-Merge {
+    param (
+        [string]$targetBranch,
+        [string]$prCommit,
+        [string]$mergeBranch
+    )
+
+    if (-not $targetBranch) {
+        Write-Host "No pull request target branch"
+        exit 1
+    }
+
+    git fetch -v origin $targetBranch
+    git checkout FETCH_HEAD
+    Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Create a temporary branch to merge the PR with the target branch
+    git checkout -b $mergeBranch
+    Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Set author identity so it can be used for git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit $prCommit
+
+    if ($LASTEXITCODE -ne 0) {
+        $mergeResult = $LASTEXITCODE
+        Write-Host "Merge failed: $mergeResult"
+        git merge --abort
+        exit $mergeResult
+    }
+}
+
+$pullRequest = $env:BUILDKITE_PULL_REQUEST
+
+if ($pullRequest -eq "false") {
+    Write-Host "Not a pull request, skipping"
+    exit 0
+}
+
+$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
+$prCommit = $env:BUILDKITE_COMMIT
+$prId = $env:BUILDKITE_PULL_REQUEST
+$mergeBranch = "pr_merge_$prId"
+
+Checkout-Merge $targetBranch $prCommit $mergeBranch
+
+Write-Host "Commit information"
+git --no-pager log --format=%B -n 1
+
+# Ensure Buildkite groups are rendered
+Write-Host ""
diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7a3755b0b24..eacbe387996 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,30 +4,43 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
-  - label: "Unit tests - Ubuntu 22.04"
-    key: "unit-tests-2204"
-    command: ".buildkite/scripts/steps/unit-tests.sh"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      image: "family/core-ubuntu-2204"
-    retry:
-      manual:
-        allowed: true
+  # - label: "Unit tests - Ubuntu 22.04"
+  #   key: "unit-tests-2204"
+  #   command: ".buildkite/scripts/steps/unit-tests.sh"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "family/core-ubuntu-2204"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
-  - label: "Unit tests - Ubuntu 22.04 ARM64"
+  # - label: "Unit tests - Ubuntu 22.04 ARM64"
+  #   key: "unit-tests-2204-arm64"
+  #   command: ".buildkite/scripts/steps/unit-tests.sh"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #   agents:
+  #     provider: "aws"
+  #     imagePrefix: "core-ubuntu-2204-aarch64"
+  #     diskSizeGb: 200
+  #     instanceType: "m6g.4xlarge"
+  #   retry:
+  #     manual:
+  #       allowed: true
+
+  - label: "Unit tests - Windows 2022"
     key: "unit-tests-2204-arm64"
     command: ".buildkite/scripts/steps/unit-tests.sh"
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
     agents:
-      provider: "aws"
-      imagePrefix: "core-ubuntu-2204-aarch64"
-      diskSizeGb: 200
-      instanceType: "m6g.4xlarge"
+      provider: "gcp"
+      image: "family/core-windows-2022"
     retry:
       manual:
         allowed: true

From c33a46de88bb1df0dd39cbc75e7ea0bda6829fc9 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 01:52:13 +0200
Subject: [PATCH 06/52] Windows pre-command hook and unit tests ps

---
 .buildkite/hooks/pre-command.ps1        | 14 ++++++++++++++
 .buildkite/pipeline.yml                 |  5 ++++-
 .buildkite/scripts/steps/unit-tests.ps1 |  5 +++++
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 .buildkite/scripts/steps/unit-tests.ps1

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index e69de29bb2d..327241f9d37 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -0,0 +1,14 @@
+# Install gvm and go
+$env:GvmVersion = "0.5.2"
+[Net.ServicePointManager]::SecurityProtocol = "tls12"
+$env:GoVersion = Get-Content -Path .go-version
+Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
+gvm --format=powershell $env:GoVersion | Invoke-Expression
+go version
+
+# Install tools
+go install github.com/magefile/mage
+go install github.com/elastic/go-licenser
+go install golang.org/x/tools/cmd/goimports
+go install github.com/jstemmer/go-junit-report
+go install gotest.tools/gotestsum
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index eacbe387996..87ac0f787c5 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -33,7 +33,7 @@ steps:
   #       allowed: true
 
   - label: "Unit tests - Windows 2022"
-    key: "unit-tests-2204-arm64"
+    key: "unit-tests-win2022"
     command: ".buildkite/scripts/steps/unit-tests.sh"
     artifact_paths:
       - "build/TEST-**"
@@ -41,6 +41,9 @@ steps:
     agents:
       provider: "gcp"
       image: "family/core-windows-2022"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
     retry:
       manual:
         allowed: true
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
new file mode 100644
index 00000000000..06d8ac08a49
--- /dev/null
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -0,0 +1,5 @@
+$ErrorActionPreference = "Stop"
+
+Write-Host "--- Unit tests"
+$Env:TEST_COVERAGE = $true
+mage unitTest

From d45138b2be064bac65c9aaca1cb9ec4b9c81cac8 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 02:19:46 +0200
Subject: [PATCH 07/52] Windows pre-command hook and unit tests ps

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 87ac0f787c5..324cca03914 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -34,7 +34,7 @@ steps:
 
   - label: "Unit tests - Windows 2022"
     key: "unit-tests-win2022"
-    command: ".buildkite/scripts/steps/unit-tests.sh"
+    command: ".buildkite/scripts/steps/unit-tests.ps1"
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"

From 5d98e25c90dfa5e06f79d3bf0c7e6d769efbf195 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 02:31:20 +0200
Subject: [PATCH 08/52] Windows pre-command hook and unit tests ps

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 324cca03914..e90c537d862 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -34,7 +34,7 @@ steps:
 
   - label: "Unit tests - Windows 2022"
     key: "unit-tests-win2022"
-    command: ".buildkite/scripts/steps/unit-tests.ps1"
+    command: ".\\buildkite\\scripts\\steps\\unit-tests.ps1"
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"

From 0d9b49836532bd536f6ebeeb8009ffbac229c263 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 02:38:52 +0200
Subject: [PATCH 09/52] Windows pre-command hook and unit tests ps

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index e90c537d862..e8cf019dea2 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -34,7 +34,7 @@ steps:
 
   - label: "Unit tests - Windows 2022"
     key: "unit-tests-win2022"
-    command: ".\\buildkite\\scripts\\steps\\unit-tests.ps1"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"

From 59b65bfddbce91ca645855853a49381142703faf Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 02:53:19 +0200
Subject: [PATCH 10/52] Windows pre-command hook and unit tests ps

---
 .buildkite/scripts/steps/unit-tests.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 06d8ac08a49..d5e12e2a01d 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -2,4 +2,4 @@ $ErrorActionPreference = "Stop"
 
 Write-Host "--- Unit tests"
 $Env:TEST_COVERAGE = $true
-mage unitTest
+mage.exe unitTest

From 108a01a01009c92d5c0f87b1331050cffe7cf28d Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 03:14:55 +0200
Subject: [PATCH 11/52] Windows pre-command hook and unit tests ps

---
 .buildkite/hooks/pre-command.ps1        | 1 +
 .buildkite/scripts/steps/unit-tests.ps1 | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 327241f9d37..9bdc89d82fa 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,5 +1,6 @@
 # Install gvm and go
 $env:GvmVersion = "0.5.2"
+$Env:GOTMPDIR = $Env:BUILDKITE_BUILD_CHECKOUT_PATH https://github.com/golang/go/issues/42224#issuecomment-1021149948
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
 $env:GoVersion = Get-Content -Path .go-version
 Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index d5e12e2a01d..5ddd29f6bb6 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -1,5 +1,10 @@
 $ErrorActionPreference = "Stop"
-
 Write-Host "--- Unit tests"
+#debug
+Write-Host "GOPATH: $env:GOPATH"
+Get-ChildItem -Path $env:GOPATH\bin
+
+# run tests
 $Env:TEST_COVERAGE = $true
-mage.exe unitTest
+& "$env:GOPATH\bin\mage.exe" unitTest
+

From d2ac3485cfab11d396722df33becb9eaa191c96f Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 03:22:35 +0200
Subject: [PATCH 12/52] Windows pre-command hook and unit tests ps

---
 .buildkite/hooks/pre-command.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 9bdc89d82fa..40af84b3851 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,6 +1,6 @@
 # Install gvm and go
 $env:GvmVersion = "0.5.2"
-$Env:GOTMPDIR = $Env:BUILDKITE_BUILD_CHECKOUT_PATH https://github.com/golang/go/issues/42224#issuecomment-1021149948
+$Env:GOTMPDIR = $Env:BUILDKITE_BUILD_CHECKOUT_PATH #https://github.com/golang/go/issues/42224#issuecomment-1021149948
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
 $env:GoVersion = Get-Content -Path .go-version
 Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe

From 8c5936cb36428c34f191d7c5be9643d960062a5c Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 03:35:09 +0200
Subject: [PATCH 13/52] Windows pre-command hook and unit tests ps

---
 .buildkite/hooks/pre-command.ps1 | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 40af84b3851..1454033eadc 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -13,3 +13,7 @@ go install github.com/elastic/go-licenser
 go install golang.org/x/tools/cmd/goimports
 go install github.com/jstemmer/go-junit-report
 go install gotest.tools/gotestsum
+
+# debug
+Write-Host "GOPATH: $env:GOPATH"
+Get-ChildItem -Path $env:GOPATH\bin

From 4a1c285aa743bd379526e3d3a00ec206d4a037e5 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 14:33:34 +0200
Subject: [PATCH 14/52] set GOPATH

---
 .buildkite/hooks/pre-command.ps1        | 2 ++
 .buildkite/scripts/steps/unit-tests.ps1 | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 1454033eadc..6db45a7b2db 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,6 +1,8 @@
 # Install gvm and go
 $env:GvmVersion = "0.5.2"
 $Env:GOTMPDIR = $Env:BUILDKITE_BUILD_CHECKOUT_PATH #https://github.com/golang/go/issues/42224#issuecomment-1021149948
+$Env:GOPATH = $Env:BUILDKITE_BUILD_CHECKOUT_PATH
+
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
 $env:GoVersion = Get-Content -Path .go-version
 Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 5ddd29f6bb6..5f0b58d4998 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -6,5 +6,5 @@ Get-ChildItem -Path $env:GOPATH\bin
 
 # run tests
 $Env:TEST_COVERAGE = $true
-& "$env:GOPATH\bin\mage.exe" unitTest
+mage unitTest
 

From 43ebbedffac210f94bb7f0570145098c37364458 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 15:25:12 +0200
Subject: [PATCH 15/52] set GOPATH

---
 .buildkite/scripts/steps/unit-tests.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 5f0b58d4998..c0bcc99799b 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -2,7 +2,7 @@ $ErrorActionPreference = "Stop"
 Write-Host "--- Unit tests"
 #debug
 Write-Host "GOPATH: $env:GOPATH"
-Get-ChildItem -Path $env:GOPATH\bin
+# Get-ChildItem -Path $env:GOPATH\bin
 
 # run tests
 $Env:TEST_COVERAGE = $true

From 603220996076885b84b842e082a163a7d17b2b25 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 15:51:21 +0200
Subject: [PATCH 16/52] set GOPATH

---
 .buildkite/hooks/pre-command.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 6db45a7b2db..5e805319ac2 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -18,4 +18,4 @@ go install gotest.tools/gotestsum
 
 # debug
 Write-Host "GOPATH: $env:GOPATH"
-Get-ChildItem -Path $env:GOPATH\bin
+#Get-ChildItem -Path $env:GOPATH\bin

From ff83b145a03671fa76ed5249221e9c944ab4608f Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 18:49:03 +0200
Subject: [PATCH 17/52] set GOPATH

---
 .buildkite/hooks/pre-command.ps1        | 2 +-
 .buildkite/scripts/steps/unit-tests.ps1 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 5e805319ac2..04b8bdf4566 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,7 +1,7 @@
 # Install gvm and go
 $env:GvmVersion = "0.5.2"
 $Env:GOTMPDIR = $Env:BUILDKITE_BUILD_CHECKOUT_PATH #https://github.com/golang/go/issues/42224#issuecomment-1021149948
-$Env:GOPATH = $Env:BUILDKITE_BUILD_CHECKOUT_PATH
+$Env:GOPATH = $Env:BUILDKITE_BUILD_CHECKOUT_PATH\gopath
 
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
 $env:GoVersion = Get-Content -Path .go-version
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index c0bcc99799b..e6149ebadf6 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -6,5 +6,5 @@ Write-Host "GOPATH: $env:GOPATH"
 
 # run tests
 $Env:TEST_COVERAGE = $true
-mage unitTest
+.\gopath\mage unitTest
 

From 49fe289e99acbe5f157148dbc84a172a8b8611ba Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 13 Oct 2023 18:57:00 +0200
Subject: [PATCH 18/52] set GOPATH

---
 .buildkite/hooks/pre-command.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 04b8bdf4566..8b9698d2578 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,7 +1,7 @@
 # Install gvm and go
 $env:GvmVersion = "0.5.2"
 $Env:GOTMPDIR = $Env:BUILDKITE_BUILD_CHECKOUT_PATH #https://github.com/golang/go/issues/42224#issuecomment-1021149948
-$Env:GOPATH = $Env:BUILDKITE_BUILD_CHECKOUT_PATH\gopath
+$Env:GOPATH = "$Env:BUILDKITE_BUILD_CHECKOUT_PATH\gopath"
 
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
 $env:GoVersion = Get-Content -Path .go-version

From 6e5c49d4e3b54045d80c46cc312cd06f487131b2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 17 Oct 2023 01:34:03 +0200
Subject: [PATCH 19/52] Installed GCC

---
 .buildkite/hooks/pre-command.ps1        | 16 +++++++++-------
 .buildkite/scripts/steps/unit-tests.ps1 | 12 +++++-------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index 8b9698d2578..ed1d4adfd19 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,21 +1,23 @@
+# Install gcc TODO: Move to the VM image
+choco install mingw
+
 # Install gvm and go
+# TODO: Move GVM download to the base VM image
 $env:GvmVersion = "0.5.2"
-$Env:GOTMPDIR = $Env:BUILDKITE_BUILD_CHECKOUT_PATH #https://github.com/golang/go/issues/42224#issuecomment-1021149948
-$Env:GOPATH = "$Env:BUILDKITE_BUILD_CHECKOUT_PATH\gopath"
-
 [Net.ServicePointManager]::SecurityProtocol = "tls12"
 $env:GoVersion = Get-Content -Path .go-version
 Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
 gvm --format=powershell $env:GoVersion | Invoke-Expression
 go version
 
+$GOPATH = $(go env GOPATH)
+$env:Path = "$GOPATH\bin;" + $env:Path
+[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
+[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
+
 # Install tools
 go install github.com/magefile/mage
 go install github.com/elastic/go-licenser
 go install golang.org/x/tools/cmd/goimports
 go install github.com/jstemmer/go-junit-report
 go install gotest.tools/gotestsum
-
-# debug
-Write-Host "GOPATH: $env:GOPATH"
-#Get-ChildItem -Path $env:GOPATH\bin
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index e6149ebadf6..2589e7ac323 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -1,10 +1,8 @@
 $ErrorActionPreference = "Stop"
-Write-Host "--- Unit tests"
-#debug
-Write-Host "GOPATH: $env:GOPATH"
-# Get-ChildItem -Path $env:GOPATH\bin
 
-# run tests
-$Env:TEST_COVERAGE = $true
-.\gopath\mage unitTest
+Write-Host "--- Build"
+mage build
 
+Write-Host "--- Unit tests"
+$Env:TEST_COVERAGE = $true
+mage unitTest

From 59e011299ff5b43c2dcc61da514585fa3363d1e7 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 17 Oct 2023 01:45:40 +0200
Subject: [PATCH 20/52] Installed GCC

---
 .buildkite/hooks/pre-command.ps1        | 1 +
 .buildkite/scripts/steps/unit-tests.ps1 | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index ed1d4adfd19..ace56141ed9 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,5 +1,6 @@
 # Install gcc TODO: Move to the VM image
 choco install mingw
+refreshenv
 
 # Install gvm and go
 # TODO: Move GVM download to the base VM image
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 2589e7ac323..b2e48c156d5 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -4,5 +4,5 @@ Write-Host "--- Build"
 mage build
 
 Write-Host "--- Unit tests"
-$Env:TEST_COVERAGE = $true
+$env:TEST_COVERAGE = $true
 mage unitTest

From 4f8a97e0f6df6187eca30eb7e42668db2763d455 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 17 Oct 2023 02:03:47 +0200
Subject: [PATCH 21/52] Installed GCC

---
 .buildkite/hooks/pre-command.ps1 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
index ace56141ed9..ca5b34bc920 100644
--- a/.buildkite/hooks/pre-command.ps1
+++ b/.buildkite/hooks/pre-command.ps1
@@ -1,5 +1,6 @@
 # Install gcc TODO: Move to the VM image
 choco install mingw
+Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
 refreshenv
 
 # Install gvm and go

From aa64a97e9a55269ae21ed413f59d0e636e332ad1 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 17 Oct 2023 02:31:03 +0200
Subject: [PATCH 22/52] Windows 2016

---
 .buildkite/pipeline.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index e8cf019dea2..e39ff071790 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -48,6 +48,22 @@ steps:
       manual:
         allowed: true
 
+  - label: "Unit tests - Windows 2016"
+    key: "unit-tests-win2016"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2016"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
   # - label: ":sonarqube: Continuous Code Inspection"
   #   env:
   #     VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"

From dcaddb2b2041c9d127e5319cc5f638a5c174e040 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 17 Oct 2023 15:46:06 +0200
Subject: [PATCH 23/52] Propagate exit code on windows

---
 .buildkite/pipeline.yml                 | 80 ++++++++++++-------------
 .buildkite/scripts/steps/unit-tests.ps1 |  8 +++
 2 files changed, 48 insertions(+), 40 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index e39ff071790..5345c3db34d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,33 +4,33 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
-  # - label: "Unit tests - Ubuntu 22.04"
-  #   key: "unit-tests-2204"
-  #   command: ".buildkite/scripts/steps/unit-tests.sh"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/core-ubuntu-2204"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Ubuntu 22.04"
+    key: "unit-tests-2204"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
 
-  # - label: "Unit tests - Ubuntu 22.04 ARM64"
-  #   key: "unit-tests-2204-arm64"
-  #   command: ".buildkite/scripts/steps/unit-tests.sh"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "aws"
-  #     imagePrefix: "core-ubuntu-2204-aarch64"
-  #     diskSizeGb: 200
-  #     instanceType: "m6g.4xlarge"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Ubuntu 22.04 ARM64"
+    key: "unit-tests-2204-arm64"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "aws"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 200
+      instanceType: "m6g.4xlarge"
+    retry:
+      manual:
+        allowed: true
 
   - label: "Unit tests - Windows 2022"
     key: "unit-tests-win2022"
@@ -64,20 +64,20 @@ steps:
       manual:
         allowed: true
 
-  # - label: ":sonarqube: Continuous Code Inspection"
-  #   env:
-  #     VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
-  #   agents:
-  #     image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
-  #   command:
-  #     - "buildkite-agent artifact download build/TEST-go-unit.cov ."
-  #     - "buildkite-agent artifact download build/TEST-go-unit.out.json ."
-  #     - "/scan-source-code.sh"
-  #   depends_on:
-  #     - "unit-tests"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command:
+      - "buildkite-agent artifact download build/TEST-go-unit.cov ."
+      - "buildkite-agent artifact download build/TEST-go-unit.out.json ."
+      - "/scan-source-code.sh"
+    depends_on:
+      - "unit-tests"
+    retry:
+      manual:
+        allowed: true
 
   # - label: "Integration tests"
   #   key: "integration-tests"
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index b2e48c156d5..bb2d05f87a3 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -3,6 +3,14 @@ $ErrorActionPreference = "Stop"
 Write-Host "--- Build"
 mage build
 
+if ($LASTEXITCODE -ne 0) {  
+  exit 1 
+}
+
 Write-Host "--- Unit tests"
 $env:TEST_COVERAGE = $true
 mage unitTest
+
+if ($LASTEXITCODE -ne 0) {  
+  exit 1 
+}

From 477422c31a20dc35c0dade6cae6fc7c74f68d31c Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 18 Oct 2023 00:47:46 +0200
Subject: [PATCH 24/52] Added GOTMPDIR

---
 .buildkite/pipeline.yml                 | 82 ++++++++++++-------------
 .buildkite/scripts/steps/unit-tests.ps1 |  2 +
 2 files changed, 43 insertions(+), 41 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 5345c3db34d..a8d31b4c700 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,33 +4,33 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
-  - label: "Unit tests - Ubuntu 22.04"
-    key: "unit-tests-2204"
-    command: ".buildkite/scripts/steps/unit-tests.sh"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      image: "family/core-ubuntu-2204"
-    retry:
-      manual:
-        allowed: true
+  # - label: "Unit tests - Ubuntu 22.04"
+  #   key: "unit-tests-2204"
+  #   command: ".buildkite/scripts/steps/unit-tests.sh"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "family/core-ubuntu-2204"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
-  - label: "Unit tests - Ubuntu 22.04 ARM64"
-    key: "unit-tests-2204-arm64"
-    command: ".buildkite/scripts/steps/unit-tests.sh"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "aws"
-      imagePrefix: "core-ubuntu-2204-aarch64"
-      diskSizeGb: 200
-      instanceType: "m6g.4xlarge"
-    retry:
-      manual:
-        allowed: true
+  # - label: "Unit tests - Ubuntu 22.04 ARM64"
+  #   key: "unit-tests-2204-arm64"
+  #   command: ".buildkite/scripts/steps/unit-tests.sh"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #   agents:
+  #     provider: "aws"
+  #     imagePrefix: "core-ubuntu-2204-aarch64"
+  #     diskSizeGb: 200
+  #     instanceType: "m6g.4xlarge"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
   - label: "Unit tests - Windows 2022"
     key: "unit-tests-win2022"
@@ -48,21 +48,21 @@ steps:
       manual:
         allowed: true
 
-  - label: "Unit tests - Windows 2016"
-    key: "unit-tests-win2016"
-    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      image: "family/core-windows-2016"
-      machine_type: "n2-standard-8"
-      disk_size: 200
-      disk_type: "pd-ssd"
-    retry:
-      manual:
-        allowed: true
+  # - label: "Unit tests - Windows 2016"
+  #   key: "unit-tests-win2016"
+  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "family/core-windows-2016"
+  #     machine_type: "n2-standard-8"
+  #     disk_size: 200
+  #     disk_type: "pd-ssd"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
   - label: ":sonarqube: Continuous Code Inspection"
     env:
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index bb2d05f87a3..d87abebb90f 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -1,5 +1,7 @@
 $ErrorActionPreference = "Stop"
 
+$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH\temp"
+
 Write-Host "--- Build"
 mage build
 

From 7fa1a5e626c5dd0d5917fe36272fe2fcde6719eb Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 18 Oct 2023 01:04:16 +0200
Subject: [PATCH 25/52] Added GOTMPDIR

---
 .buildkite/scripts/steps/unit-tests.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index d87abebb90f..02ede82c469 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -1,6 +1,6 @@
 $ErrorActionPreference = "Stop"
 
-$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH\temp"
+$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"
 
 Write-Host "--- Build"
 mage build

From 86d18a8b614a738ddaaecdc2446d334320076e9e Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 18 Oct 2023 16:22:00 +0200
Subject: [PATCH 26/52] Merge coverage reports step

---
 .buildkite/pipeline.yml                 | 111 ++++++++++++++----------
 .buildkite/scripts/steps/unit-tests.ps1 |   1 +
 2 files changed, 66 insertions(+), 46 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index a8d31b4c700..627d34d3075 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,49 +4,68 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
-  # - label: "Unit tests - Ubuntu 22.04"
-  #   key: "unit-tests-2204"
-  #   command: ".buildkite/scripts/steps/unit-tests.sh"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/core-ubuntu-2204"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Ubuntu 22.04"
+    key: "unit-tests-2204"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
 
-  # - label: "Unit tests - Ubuntu 22.04 ARM64"
-  #   key: "unit-tests-2204-arm64"
-  #   command: ".buildkite/scripts/steps/unit-tests.sh"
+  - label: "Unit tests - Ubuntu 22.04 ARM64"
+    key: "unit-tests-2204-arm64"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "aws"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 200
+      instanceType: "m6g.4xlarge"
+    retry:
+      manual:
+        allowed: true
+
+  # - label: "Unit tests - Windows 2022"
+  #   key: "unit-tests-win2022"
+  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
   #   artifact_paths:
   #     - "build/TEST-**"
   #     - "build/diagnostics/*"
   #   agents:
-  #     provider: "aws"
-  #     imagePrefix: "core-ubuntu-2204-aarch64"
-  #     diskSizeGb: 200
-  #     instanceType: "m6g.4xlarge"
+  #     provider: "gcp"
+  #     image: "family/core-windows-2022"
+  #     machine_type: "n2-standard-8"
+  #     disk_size: 200
+  #     disk_type: "pd-ssd"
   #   retry:
   #     manual:
   #       allowed: true
-
-  - label: "Unit tests - Windows 2022"
-    key: "unit-tests-win2022"
-    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+  - label: "Merge coverage reports"
+    key: "merge-coverage"
+    env:
+      BUILDKITE_REPO: "" # skip checkout
+    command:
+      - "go install go.shabbyrobe.org/gocovmerge@latest"
+      - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov ./unit-tests-2204"
+      - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov ./unit-tests-2204-arm64"
+      - "ls ./unit-tests-2204"
+      - "gocovmerge ./unit-tests-2204/TEST-go-unit.cov unit-tests-2204-arm64/TEST-go-unit.cov > build/TEST-go-unit.cov"
+      - "cat build/TEST-go-unit.cov"
     artifact_paths:
       - "build/TEST-**"
-      - "build/diagnostics/*"
     agents:
-      provider: "gcp"
-      image: "family/core-windows-2022"
-      machine_type: "n2-standard-8"
-      disk_size: 200
-      disk_type: "pd-ssd"
-    retry:
-      manual:
-        allowed: true
+      image: "golang:1.20.10"
+    depends_on:
+      - "unit-tests"
+    # plugins:
+    #   - cultureamp/skip-checkout#v1.0.0: ~
 
   # - label: "Unit tests - Windows 2016"
   #   key: "unit-tests-win2016"
@@ -64,20 +83,20 @@ steps:
   #     manual:
   #       allowed: true
 
-  - label: ":sonarqube: Continuous Code Inspection"
-    env:
-      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
-    agents:
-      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
-    command:
-      - "buildkite-agent artifact download build/TEST-go-unit.cov ."
-      - "buildkite-agent artifact download build/TEST-go-unit.out.json ."
-      - "/scan-source-code.sh"
-    depends_on:
-      - "unit-tests"
-    retry:
-      manual:
-        allowed: true
+  # - label: ":sonarqube: Continuous Code Inspection"
+  #   env:
+  #     VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+  #   agents:
+  #     image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+  #   command:
+  #     - "buildkite-agent artifact download build/TEST-go-unit.cov ."
+  #     - "buildkite-agent artifact download build/TEST-go-unit.out.json ."
+  #     - "/scan-source-code.sh"
+  #   depends_on:
+  #     - "unit-tests"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
   # - label: "Integration tests"
   #   key: "integration-tests"
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 02ede82c469..9a6b220ad53 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -3,6 +3,7 @@ $ErrorActionPreference = "Stop"
 $env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"
 
 Write-Host "--- Build"
+go env
 mage build
 
 if ($LASTEXITCODE -ne 0) {  

From 210bf64d1cd3d45f0d2cd3fa6a91c2e900379894 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 18 Oct 2023 16:33:16 +0200
Subject: [PATCH 27/52] Merge coverage reports step

---
 .buildkite/pipeline.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 627d34d3075..6007c4beb66 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -63,7 +63,8 @@ steps:
     agents:
       image: "golang:1.20.10"
     depends_on:
-      - "unit-tests"
+      - "unit-tests-2204"
+      - "unit-tests-2204-arm64"
     # plugins:
     #   - cultureamp/skip-checkout#v1.0.0: ~
 

From 2b633ead36b36cc1c01829310db8f47f14524bfd Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 18 Oct 2023 18:00:43 +0200
Subject: [PATCH 28/52] Merge coverage reports step

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 6007c4beb66..30284680779 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -52,7 +52,7 @@ steps:
     env:
       BUILDKITE_REPO: "" # skip checkout
     command:
-      - "go install go.shabbyrobe.org/gocovmerge@latest"
+      - "go install go.shabbyrobe.org/gocovmerge"
       - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov ./unit-tests-2204"
       - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov ./unit-tests-2204-arm64"
       - "ls ./unit-tests-2204"

From 61adfbd544aa4a8137e56ec274a1d9d421f57d24 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 18 Oct 2023 18:28:53 +0200
Subject: [PATCH 29/52] changed installation path

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 30284680779..881951f4b13 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -52,7 +52,7 @@ steps:
     env:
       BUILDKITE_REPO: "" # skip checkout
     command:
-      - "go install go.shabbyrobe.org/gocovmerge"
+      - "go install github.com/wadey/gocovmerge@latest"
       - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov ./unit-tests-2204"
       - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov ./unit-tests-2204-arm64"
       - "ls ./unit-tests-2204"

From fc5a39ed58820133c3bb8e42640936c4623d485b Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 18 Oct 2023 19:15:32 +0200
Subject: [PATCH 30/52] changed installation path

---
 .buildkite/pipeline.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 881951f4b13..16d15728d61 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -53,10 +53,10 @@ steps:
       BUILDKITE_REPO: "" # skip checkout
     command:
       - "go install github.com/wadey/gocovmerge@latest"
-      - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov ./unit-tests-2204"
-      - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov ./unit-tests-2204-arm64"
+      - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov coverage-unit-tests-2204.cov"
+      - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov coverage-unit-tests-2204-arm64.cov ./"
       - "ls ./unit-tests-2204"
-      - "gocovmerge ./unit-tests-2204/TEST-go-unit.cov unit-tests-2204-arm64/TEST-go-unit.cov > build/TEST-go-unit.cov"
+      - "gocovmerge coverage-unit-tests-2204.cov coverage-unit-tests-2204-arm64.cov > build/TEST-go-unit.cov"
       - "cat build/TEST-go-unit.cov"
     artifact_paths:
       - "build/TEST-**"

From 447c4e873402c28703b76e6851e239771f6595c4 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 19 Oct 2023 13:57:01 +0200
Subject: [PATCH 31/52] changed installation path

---
 .buildkite/pipeline.yml           | 15 ++++++++-------
 .buildkite/scripts/steps/merge.sh |  9 +++++++++
 2 files changed, 17 insertions(+), 7 deletions(-)
 create mode 100644 .buildkite/scripts/steps/merge.sh

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 16d15728d61..7299c8d126c 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -51,13 +51,7 @@ steps:
     key: "merge-coverage"
     env:
       BUILDKITE_REPO: "" # skip checkout
-    command:
-      - "go install github.com/wadey/gocovmerge@latest"
-      - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov coverage-unit-tests-2204.cov"
-      - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov coverage-unit-tests-2204-arm64.cov ./"
-      - "ls ./unit-tests-2204"
-      - "gocovmerge coverage-unit-tests-2204.cov coverage-unit-tests-2204-arm64.cov > build/TEST-go-unit.cov"
-      - "cat build/TEST-go-unit.cov"
+    command: ".buildkite/scripts/steps/merge.sh"
     artifact_paths:
       - "build/TEST-**"
     agents:
@@ -68,6 +62,13 @@ steps:
     # plugins:
     #   - cultureamp/skip-checkout#v1.0.0: ~
 
+    # - "go install github.com/wadey/gocovmerge@latest"
+    # - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov coverage-unit-tests-2204.cov"
+    # - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov coverage-unit-tests-2204-arm64.cov ./"
+    # - "ls ./unit-tests-2204"
+    # - "gocovmerge coverage-unit-tests-2204.cov coverage-unit-tests-2204-arm64.cov > build/TEST-go-unit.cov"
+    # - "cat build/TEST-go-unit.cov"
+
   # - label: "Unit tests - Windows 2016"
   #   key: "unit-tests-win2016"
   #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
new file mode 100644
index 00000000000..6f50eb863ca
--- /dev/null
+++ b/.buildkite/scripts/steps/merge.sh
@@ -0,0 +1,9 @@
+
+set -exuo pipefail
+
+go install github.com/wadey/gocovmerge@latest
+buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov coverage-unit-tests-2204.cov
+buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov coverage-unit-tests-2204-arm64.cov
+ls ./unit-tests-2204
+gocovmerge coverage-unit-tests-2204.cov coverage-unit-tests-2204-arm64.cov > build/TEST-go-unit.cov
+cat build/TEST-go-unit.cov
\ No newline at end of file

From 0ec623ca606e346fb2a0b6a54a524f042ec72ebf Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 19 Oct 2023 14:32:28 +0200
Subject: [PATCH 32/52] changed installation path

---
 .buildkite/scripts/steps/merge.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
index 6f50eb863ca..4e6b857dfad 100644
--- a/.buildkite/scripts/steps/merge.sh
+++ b/.buildkite/scripts/steps/merge.sh
@@ -1,9 +1,12 @@
 
 set -exuo pipefail
 
+$COV_FILE="build/TEST-go-unit.cov"
+
 go install github.com/wadey/gocovmerge@latest
-buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov coverage-unit-tests-2204.cov
-buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov coverage-unit-tests-2204-arm64.cov
-ls ./unit-tests-2204
-gocovmerge coverage-unit-tests-2204.cov coverage-unit-tests-2204-arm64.cov > build/TEST-go-unit.cov
+mkdir unit-tests-2204 && coverage-unit-tests-2204buildkite-agent artifact download --step unit-tests-2204 $COV_FILE unit-tests-2204
+mkdir unit-tests-2204-arm64 && buildkite-agent artifact download --step unit-tests-2204-arm64 $COV_FILE unit-tests-2204-arm64
+ls unit-tests-2204
+ls unit-tests-2204-arm64
+gocovmerge unit-tests-2204/$COV_FILE unit-tests-2204-arm64/$COV_FILE > build/TEST-go-unit.cov
 cat build/TEST-go-unit.cov
\ No newline at end of file

From 22583c9bf1507aaf85dcdaadb2dd4c3fc76e8ffd Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 19 Oct 2023 14:39:17 +0200
Subject: [PATCH 33/52] changed installation path

---
 .buildkite/scripts/steps/merge.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
index 4e6b857dfad..b0d347f7bc2 100644
--- a/.buildkite/scripts/steps/merge.sh
+++ b/.buildkite/scripts/steps/merge.sh
@@ -4,7 +4,7 @@ set -exuo pipefail
 $COV_FILE="build/TEST-go-unit.cov"
 
 go install github.com/wadey/gocovmerge@latest
-mkdir unit-tests-2204 && coverage-unit-tests-2204buildkite-agent artifact download --step unit-tests-2204 $COV_FILE unit-tests-2204
+mkdir unit-tests-2204 && buildkite-agent artifact download --step unit-tests-2204 $COV_FILE unit-tests-2204
 mkdir unit-tests-2204-arm64 && buildkite-agent artifact download --step unit-tests-2204-arm64 $COV_FILE unit-tests-2204-arm64
 ls unit-tests-2204
 ls unit-tests-2204-arm64

From 8bcf6b2bf8a200d34fd78b36a99b605f86c14d44 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 19 Oct 2023 14:51:17 +0200
Subject: [PATCH 34/52] changed installation path

---
 .buildkite/scripts/steps/merge.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
index b0d347f7bc2..ac2fc1ef043 100644
--- a/.buildkite/scripts/steps/merge.sh
+++ b/.buildkite/scripts/steps/merge.sh
@@ -1,7 +1,6 @@
-
 set -exuo pipefail
 
-$COV_FILE="build/TEST-go-unit.cov"
+COV_FILE="build/TEST-go-unit.cov"
 
 go install github.com/wadey/gocovmerge@latest
 mkdir unit-tests-2204 && buildkite-agent artifact download --step unit-tests-2204 $COV_FILE unit-tests-2204

From 542556eb004e84de8bb2f8415b5e090fa3ad5699 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 19 Oct 2023 15:38:28 +0200
Subject: [PATCH 35/52] test

---
 .buildkite/scripts/steps/merge.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
index ac2fc1ef043..8f6bb4f545b 100644
--- a/.buildkite/scripts/steps/merge.sh
+++ b/.buildkite/scripts/steps/merge.sh
@@ -3,9 +3,9 @@ set -exuo pipefail
 COV_FILE="build/TEST-go-unit.cov"
 
 go install github.com/wadey/gocovmerge@latest
-mkdir unit-tests-2204 && buildkite-agent artifact download --step unit-tests-2204 $COV_FILE unit-tests-2204
-mkdir unit-tests-2204-arm64 && buildkite-agent artifact download --step unit-tests-2204-arm64 $COV_FILE unit-tests-2204-arm64
+mkdir -p unit-tests-2204 && buildkite-agent artifact download --step unit-tests-2204 $COV_FILE unit-tests-2204
+mkdir -p unit-tests-2204-arm64 && buildkite-agent artifact download --step unit-tests-2204-arm64 $COV_FILE unit-tests-2204-arm64
 ls unit-tests-2204
 ls unit-tests-2204-arm64
-gocovmerge unit-tests-2204/$COV_FILE unit-tests-2204-arm64/$COV_FILE > build/TEST-go-unit.cov
+mkdir -p build && gocovmerge unit-tests-2204/$COV_FILE unit-tests-2204-arm64/$COV_FILE > $COV_FILE
 cat build/TEST-go-unit.cov
\ No newline at end of file

From a345b148ae9263ac75c1992cfad080e34d310086 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 19 Oct 2023 20:11:45 +0200
Subject: [PATCH 36/52] test

---
 .buildkite/pipeline.yml           | 114 +++++++++++++++---------------
 .buildkite/scripts/steps/merge.sh |  22 ++++--
 2 files changed, 73 insertions(+), 63 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7299c8d126c..bcdda2a58e6 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -32,73 +32,73 @@ steps:
       manual:
         allowed: true
 
-  # - label: "Unit tests - Windows 2022"
-  #   key: "unit-tests-win2022"
-  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/core-windows-2022"
-  #     machine_type: "n2-standard-8"
-  #     disk_size: 200
-  #     disk_type: "pd-ssd"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Windows 2022"
+    key: "unit-tests-win2022"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2022"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Windows 2016"
+    key: "unit-tests-win2016"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2016"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
   - label: "Merge coverage reports"
     key: "merge-coverage"
     env:
       BUILDKITE_REPO: "" # skip checkout
-    command: ".buildkite/scripts/steps/merge.sh"
+    command: "
+      .buildkite/scripts/steps/merge.sh
+      unit-tests-2204
+      unit-tests-2204-arm64
+      unit-tests-win2016
+      unit-tests-win2022
+      "
     artifact_paths:
       - "build/TEST-**"
     agents:
       image: "golang:1.20.10"
     depends_on:
-      - "unit-tests-2204"
-      - "unit-tests-2204-arm64"
-    # plugins:
-    #   - cultureamp/skip-checkout#v1.0.0: ~
-
-    # - "go install github.com/wadey/gocovmerge@latest"
-    # - "buildkite-agent artifact download --step unit-tests-2204 build/TEST-go-unit.cov coverage-unit-tests-2204.cov"
-    # - "buildkite-agent artifact download --step unit-tests-2204-arm64 build/TEST-go-unit.cov coverage-unit-tests-2204-arm64.cov ./"
-    # - "ls ./unit-tests-2204"
-    # - "gocovmerge coverage-unit-tests-2204.cov coverage-unit-tests-2204-arm64.cov > build/TEST-go-unit.cov"
-    # - "cat build/TEST-go-unit.cov"
-
-  # - label: "Unit tests - Windows 2016"
-  #   key: "unit-tests-win2016"
-  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/core-windows-2016"
-  #     machine_type: "n2-standard-8"
-  #     disk_size: 200
-  #     disk_type: "pd-ssd"
-  #   retry:
-  #     manual:
-  #       allowed: true
+      - unit-tests-2204
+      - unit-tests-2204-arm64
+      - unit-tests-win2022
+      - unit-tests-win2016
+    allow_dependency_failure: true
 
-  # - label: ":sonarqube: Continuous Code Inspection"
-  #   env:
-  #     VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
-  #   agents:
-  #     image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
-  #   command:
-  #     - "buildkite-agent artifact download build/TEST-go-unit.cov ."
-  #     - "buildkite-agent artifact download build/TEST-go-unit.out.json ."
-  #     - "/scan-source-code.sh"
-  #   depends_on:
-  #     - "unit-tests"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command:
+      - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
+      - "/scan-source-code.sh"
+    depends_on:
+      - "merge-coverage"
+    retry:
+      manual:
+        allowed: true
 
   # - label: "Integration tests"
   #   key: "integration-tests"
diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
index 8f6bb4f545b..7cac4a0b1f6 100644
--- a/.buildkite/scripts/steps/merge.sh
+++ b/.buildkite/scripts/steps/merge.sh
@@ -1,11 +1,21 @@
+#!/bin/bash
+# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov).
+# Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#  
+
 set -exuo pipefail
 
 COV_FILE="build/TEST-go-unit.cov"
+# Space 
+COV_PATHS=""
 
 go install github.com/wadey/gocovmerge@latest
-mkdir -p unit-tests-2204 && buildkite-agent artifact download --step unit-tests-2204 $COV_FILE unit-tests-2204
-mkdir -p unit-tests-2204-arm64 && buildkite-agent artifact download --step unit-tests-2204-arm64 $COV_FILE unit-tests-2204-arm64
-ls unit-tests-2204
-ls unit-tests-2204-arm64
-mkdir -p build && gocovmerge unit-tests-2204/$COV_FILE unit-tests-2204-arm64/$COV_FILE > $COV_FILE
-cat build/TEST-go-unit.cov
\ No newline at end of file
+mkdir -p build
+
+for STEP_ID in "$@"; do
+  mkdir -p $STEP_ID 
+  buildkite-agent artifact download --step $STEP_ID $COV_FILE $STEP_ID
+  COV_PATHS="${COV_PATHS} $STEP_ID/$COV_FILE"
+done
+
+gocovmerge $COV_PATHS > $COV_FILE
+echo "Merged coverage file: $COV_FILE. See artifacts"
\ No newline at end of file

From 5f9c79c09285a0a8623a14306d65d79425f074ad Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 19 Oct 2023 20:39:16 +0200
Subject: [PATCH 37/52] merge.sh

---
 .buildkite/scripts/steps/merge.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 .buildkite/scripts/steps/merge.sh

diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
old mode 100644
new mode 100755
index 7cac4a0b1f6..6853b0664df
--- a/.buildkite/scripts/steps/merge.sh
+++ b/.buildkite/scripts/steps/merge.sh
@@ -1,11 +1,12 @@
 #!/bin/bash
+
 # Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov).
 # Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#  
 
 set -exuo pipefail
 
 COV_FILE="build/TEST-go-unit.cov"
-# Space 
+# Space separated list of paths to coverage files
 COV_PATHS=""
 
 go install github.com/wadey/gocovmerge@latest

From 750f0fc991b65e3d80ab296c1c81f3d5b338ac24 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 20 Oct 2023 02:52:27 +0200
Subject: [PATCH 38/52] Added coverage.out to simplify artifacts download

---
 .buildkite/pipeline.yml                 |   4 +
 .buildkite/scripts/steps/merge.sh       |  11 +--
 .buildkite/scripts/steps/unit-tests.ps1 |   7 +-
 .buildkite/scripts/steps/unit-tests.sh  |   4 +-
 .ci/Jenkinsfile                         | 102 ++++++++++++------------
 5 files changed, 68 insertions(+), 60 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index bcdda2a58e6..8ab27bca782 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -10,6 +10,7 @@ steps:
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
+      - "coverage.out"
     agents:
       provider: "gcp"
       image: "family/core-ubuntu-2204"
@@ -23,6 +24,7 @@ steps:
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
+      - "coverage.out"
     agents:
       provider: "aws"
       imagePrefix: "core-ubuntu-2204-aarch64"
@@ -38,6 +40,7 @@ steps:
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
+      - "coverage.out"
     agents:
       provider: "gcp"
       image: "family/core-windows-2022"
@@ -54,6 +57,7 @@ steps:
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
+      - "coverage.out"
     agents:
       provider: "gcp"
       image: "family/core-windows-2016"
diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
index 6853b0664df..4eda3e2fd3b 100755
--- a/.buildkite/scripts/steps/merge.sh
+++ b/.buildkite/scripts/steps/merge.sh
@@ -5,7 +5,8 @@
 
 set -exuo pipefail
 
-COV_FILE="build/TEST-go-unit.cov"
+COV_ARTIFACT="coverage.out"
+MERGED_COV_FILE="build/TEST-go-unit.cov"
 # Space separated list of paths to coverage files
 COV_PATHS=""
 
@@ -14,9 +15,9 @@ mkdir -p build
 
 for STEP_ID in "$@"; do
   mkdir -p $STEP_ID 
-  buildkite-agent artifact download --step $STEP_ID $COV_FILE $STEP_ID
-  COV_PATHS="${COV_PATHS} $STEP_ID/$COV_FILE"
+  buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID
+  COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT"
 done
 
-gocovmerge $COV_PATHS > $COV_FILE
-echo "Merged coverage file: $COV_FILE. See artifacts"
\ No newline at end of file
+gocovmerge $COV_PATHS > $MERGED_COV_FILE
+echo "Merged coverage file: $MERGED_COV_FILE. See artifacts"
\ No newline at end of file
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 9a6b220ad53..5af11764f95 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -3,10 +3,9 @@ $ErrorActionPreference = "Stop"
 $env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"
 
 Write-Host "--- Build"
-go env
 mage build
 
-if ($LASTEXITCODE -ne 0) {  
+if ($LASTEXITCODE -ne 0) {
   exit 1 
 }
 
@@ -14,6 +13,8 @@ Write-Host "--- Unit tests"
 $env:TEST_COVERAGE = $true
 mage unitTest
 
-if ($LASTEXITCODE -ne 0) {  
+if ($LASTEXITCODE -ne 0) {
   exit 1 
 }
+cp .\build\TEST-go-unit.cov coverage.out
+
diff --git a/.buildkite/scripts/steps/unit-tests.sh b/.buildkite/scripts/steps/unit-tests.sh
index b4d399801bf..6549b4514d3 100644
--- a/.buildkite/scripts/steps/unit-tests.sh
+++ b/.buildkite/scripts/steps/unit-tests.sh
@@ -4,4 +4,6 @@ set -euxo pipefail
 source .buildkite/scripts/common.sh
 
 echo "--- Unit tests"
-TEST_COVERAGE=true mage unitTest
\ No newline at end of file
+TEST_COVERAGE=true mage unitTest
+# Copy coverage file to build directory so it can be downloaded as an artifact
+cp build/TEST-go-unit.cov coverage.out
\ No newline at end of file
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 237e62ab2fc..fbaad654c87 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -95,57 +95,57 @@ pipeline {
             values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64'
           }
         }
-        stages {
-          stage('build'){
-            steps {
-              withGithubNotify(context: "Build-${PLATFORM}") {
-                deleteDir()
-                unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-                withMageEnv(){
-                  dir("${BASE_DIR}"){
-                    cmd(label: 'Go build', script: 'mage build')
-                  }
-                }
-              }
-            }
-          }
-          stage('Test') {
-            steps {
-              withGithubNotify(context: "Test-${PLATFORM}") {
-                withMageEnv(){
-                  dir("${BASE_DIR}"){
-                    withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
-                      cmd(label: 'Go unitTest', script: 'mage unitTest')
-                    }
-                  }
-                }
-              }
-            }
-            post {
-              always {
-                junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
-                whenTrue(isCodeCoverageEnabled()) {
-                  coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
-                }
-              }
-            }
-          }
-          stage('Integration tests') {
-            when {
-              // Always when running builds on branches/tags
-              // Enable if integration test related changes.
-              beforeAgent true
-              anyOf {
-                not { changeRequest() }
-                expression { return params.integration_tests_ci }
-                expression { return env.GITHUB_COMMENT?.contains('integration tests') }
-                expression { matchesPrLabel(label: 'ci:integration') }
-              }
-            }
-            steps {
-              echo 'TBD'
-            }
-          }
+        // stages {
+        //   stage('build'){
+        //     steps {
+        //       withGithubNotify(context: "Build-${PLATFORM}") {
+        //         deleteDir()
+        //         unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+        //         withMageEnv(){
+        //           dir("${BASE_DIR}"){
+        //             cmd(label: 'Go build', script: 'mage build')
+        //           }
+        //         }
+        //       }
+        //     }
+        //   }
+        //   stage('Test') {
+        //     steps {
+        //       withGithubNotify(context: "Test-${PLATFORM}") {
+        //         withMageEnv(){
+        //           dir("${BASE_DIR}"){
+        //             withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
+        //               cmd(label: 'Go unitTest', script: 'mage unitTest')
+        //             }
+        //           }
+        //         }
+        //       }
+        //     }
+        //     post {
+        //       always {
+        //         junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
+        //         whenTrue(isCodeCoverageEnabled()) {
+        //           coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
+        //         }
+        //       }
+        //     }
+        //   }
+          // stage('Integration tests') {
+          //   when {
+          //     // Always when running builds on branches/tags
+          //     // Enable if integration test related changes.
+          //     beforeAgent true
+          //     anyOf {
+          //       not { changeRequest() }
+          //       expression { return params.integration_tests_ci }
+          //       expression { return env.GITHUB_COMMENT?.contains('integration tests') }
+          //       expression { matchesPrLabel(label: 'ci:integration') }
+          //     }
+          //   }
+          //   steps {
+          //     echo 'TBD'
+          //   }
+          // }
           stage('K8s') {
             when {
               beforeAgent true

From a6878040c7bfd5d7eeabaaf93e4440901d2d0c44 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 20 Oct 2023 03:22:43 +0200
Subject: [PATCH 39/52] Fixed artifacts copy

---
 .buildkite/scripts/steps/unit-tests.ps1 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 5af11764f95..7f74ac74529 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -12,9 +12,11 @@ if ($LASTEXITCODE -ne 0) {
 Write-Host "--- Unit tests"
 $env:TEST_COVERAGE = $true
 mage unitTest
+# Copy coverage file to build directory so it can be downloaded as an artifact
+cp .\build\TEST-go-unit.cov coverage.out
 
 if ($LASTEXITCODE -ne 0) {
   exit 1 
 }
-cp .\build\TEST-go-unit.cov coverage.out
+
 

From 53923a9e0918f3887421ef41a5641ec929e88740 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Oct 2023 13:07:10 +0200
Subject: [PATCH 40/52] buildite orka integration

---
 .buildkite/pipeline.yml | 186 +++++++++++++++++++++-------------------
 1 file changed, 100 insertions(+), 86 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8ab27bca782..6bcd63c274d 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,105 +4,119 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
-  - label: "Unit tests - Ubuntu 22.04"
-    key: "unit-tests-2204"
-    command: ".buildkite/scripts/steps/unit-tests.sh"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-      - "coverage.out"
-    agents:
-      provider: "gcp"
-      image: "family/core-ubuntu-2204"
-    retry:
-      manual:
-        allowed: true
+  # - label: "Unit tests - Ubuntu 22.04"
+  #   key: "unit-tests-2204"
+  #   command: ".buildkite/scripts/steps/unit-tests.sh"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #     - "coverage.out"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "family/core-ubuntu-2204"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
-  - label: "Unit tests - Ubuntu 22.04 ARM64"
-    key: "unit-tests-2204-arm64"
-    command: ".buildkite/scripts/steps/unit-tests.sh"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-      - "coverage.out"
-    agents:
-      provider: "aws"
-      imagePrefix: "core-ubuntu-2204-aarch64"
-      diskSizeGb: 200
-      instanceType: "m6g.4xlarge"
-    retry:
-      manual:
-        allowed: true
+  # - label: "Unit tests - Ubuntu 22.04 ARM64"
+  #   key: "unit-tests-2204-arm64"
+  #   command: ".buildkite/scripts/steps/unit-tests.sh"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #     - "coverage.out"
+  #   agents:
+  #     provider: "aws"
+  #     imagePrefix: "core-ubuntu-2204-aarch64"
+  #     diskSizeGb: 200
+  #     instanceType: "m6g.4xlarge"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
-  - label: "Unit tests - Windows 2022"
-    key: "unit-tests-win2022"
-    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-      - "coverage.out"
-    agents:
-      provider: "gcp"
-      image: "family/core-windows-2022"
-      machine_type: "n2-standard-8"
-      disk_size: 200
-      disk_type: "pd-ssd"
-    retry:
-      manual:
-        allowed: true
+  # - label: "Unit tests - Windows 2022"
+  #   key: "unit-tests-win2022"
+  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #     - "coverage.out"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "family/core-windows-2022"
+  #     machine_type: "n2-standard-8"
+  #     disk_size: 200
+  #     disk_type: "pd-ssd"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
-  - label: "Unit tests - Windows 2016"
-    key: "unit-tests-win2016"
-    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+  # - label: "Unit tests - Windows 2016"
+  #   key: "unit-tests-win2016"
+  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #     - "build/diagnostics/*"
+  #     - "coverage.out"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "family/core-windows-2016"
+  #     machine_type: "n2-standard-8"
+  #     disk_size: 200
+  #     disk_type: "pd-ssd"
+  #   retry:
+  #     manual:
+  #       allowed: true
+
+  - label: "Unit tests - MacOS 13 ARM"
+    key: "unit-tests-macos-13-arm"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
       - "coverage.out"
     agents:
-      provider: "gcp"
-      image: "family/core-windows-2016"
-      machine_type: "n2-standard-8"
-      disk_size: 200
-      disk_type: "pd-ssd"
+      provider: "orka"
+      imagePrefix: generic-13-ventura-arm
     retry:
       manual:
         allowed: true
 
-  - label: "Merge coverage reports"
-    key: "merge-coverage"
-    env:
-      BUILDKITE_REPO: "" # skip checkout
-    command: "
-      .buildkite/scripts/steps/merge.sh
-      unit-tests-2204
-      unit-tests-2204-arm64
-      unit-tests-win2016
-      unit-tests-win2022
-      "
-    artifact_paths:
-      - "build/TEST-**"
-    agents:
-      image: "golang:1.20.10"
-    depends_on:
-      - unit-tests-2204
-      - unit-tests-2204-arm64
-      - unit-tests-win2022
-      - unit-tests-win2016
-    allow_dependency_failure: true
+  # - label: "Merge coverage reports"
+  #   key: "merge-coverage"
+  #   env:
+  #     BUILDKITE_REPO: "" # skip checkout
+  #   command: "
+  #     .buildkite/scripts/steps/merge.sh
+  #     unit-tests-2204
+  #     unit-tests-2204-arm64
+  #     unit-tests-win2016
+  #     unit-tests-win2022
+  #     "
+  #   artifact_paths:
+  #     - "build/TEST-**"
+  #   agents:
+  #     image: "golang:1.20.10"
+  #   depends_on:
+  #     - unit-tests-2204
+  #     - unit-tests-2204-arm64
+  #     - unit-tests-win2022
+  #     - unit-tests-win2016
+  #   allow_dependency_failure: true
 
-  - label: ":sonarqube: Continuous Code Inspection"
-    env:
-      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
-    agents:
-      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
-    command:
-      - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
-      - "/scan-source-code.sh"
-    depends_on:
-      - "merge-coverage"
-    retry:
-      manual:
-        allowed: true
+  # - label: ":sonarqube: Continuous Code Inspection"
+  #   env:
+  #     VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+  #   agents:
+  #     image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+  #   command:
+  #     - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
+  #     - "/scan-source-code.sh"
+  #   depends_on:
+  #     - "merge-coverage"
+  #   retry:
+  #     manual:
+  #       allowed: true
 
   # - label: "Integration tests"
   #   key: "integration-tests"

From 0fd119413902a316cbfbc50c41664efcc4aee963 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Oct 2023 13:19:13 +0200
Subject: [PATCH 41/52] fix os name

---
 .buildkite/scripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index b1ec9608143..16d32fb77ab 100644
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -22,7 +22,7 @@ getOSOptions() {
       export AGENT_OS_NAME=linux
       ;;
     darwin*)
-      export AGENT_OS_NAME=osx
+      export AGENT_OS_NAME=darwin
       ;;
     msys*)
       export AGENT_OS_NAME=windows

From 81663ed3c1dabe12a5c26166e3bce8ab72d4a055 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Oct 2023 13:48:32 +0200
Subject: [PATCH 42/52] Add err message to upgrade test

---
 internal/pkg/agent/application/upgrade/upgrade_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go
index edbf1dd9b6a..d8de6923cfe 100644
--- a/internal/pkg/agent/application/upgrade/upgrade_test.go
+++ b/internal/pkg/agent/application/upgrade/upgrade_test.go
@@ -99,7 +99,7 @@ func Test_CopyFile(t *testing.T) {
 			}
 
 			err := copyDir(l, tc.From, tc.To, tc.IgnoreErr)
-			require.Equal(t, tc.ExpectedErr, err != nil)
+			require.Equal(t, tc.ExpectedErr, err != nil, err)
 		})
 	}
 }

From 1ed495c4e873ea30afa98c21477abf81483661e6 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Oct 2023 18:31:11 +0200
Subject: [PATCH 43/52] Fix autocrlf for windows

---
 .buildkite/hooks/post-checkout.ps1      |  3 ++
 .buildkite/pipeline.yml                 | 64 ++++++++++++-------------
 .buildkite/scripts/steps/unit-tests.ps1 |  5 ++
 3 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/.buildkite/hooks/post-checkout.ps1 b/.buildkite/hooks/post-checkout.ps1
index 14ca754bc7a..f2c590378fa 100644
--- a/.buildkite/hooks/post-checkout.ps1
+++ b/.buildkite/hooks/post-checkout.ps1
@@ -53,5 +53,8 @@ Checkout-Merge $targetBranch $prCommit $mergeBranch
 Write-Host "Commit information"
 git --no-pager log --format=%B -n 1
 
+Write-Host "Fixing CRLF in git checkout --"
+git config core.autocrlf true
+
 # Ensure Buildkite groups are rendered
 Write-Host ""
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index fd9eecc8926..1cf48c53eaa 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -34,39 +34,39 @@ steps:
   #     manual:
   #       allowed: true
 
-  # - label: "Unit tests - Windows 2022"
-  #   key: "unit-tests-win2022"
-  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #     - "coverage.out"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/core-windows-2022"
-  #     machine_type: "n2-standard-8"
-  #     disk_size: 200
-  #     disk_type: "pd-ssd"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Windows 2022"
+    key: "unit-tests-win2022"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2022"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
 
-  # - label: "Unit tests - Windows 2016"
-  #   key: "unit-tests-win2016"
-  #   command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #     - "coverage.out"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/core-windows-2016"
-  #     machine_type: "n2-standard-8"
-  #     disk_size: 200
-  #     disk_type: "pd-ssd"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Windows 2016"
+    key: "unit-tests-win2016"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2016"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
 
   - label: "Unit tests - MacOS 13 ARM"
     key: "unit-tests-macos-13-arm"
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 7f74ac74529..18b17c6a631 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -1,5 +1,10 @@
 $ErrorActionPreference = "Stop"
 
+Write-Host "-- Fixing CRLF in git checkout --"
+git config core.autocrlf input
+git rm --quiet --cached -r .
+git reset --quiet --hard
+
 $env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"
 
 Write-Host "--- Build"

From 802597bb58fb2a92cd6364771d3954713262ae86 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Oct 2023 21:01:31 +0200
Subject: [PATCH 44/52] Craig's patch

---
 internal/pkg/agent/application/upgrade/upgrade.go | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go
index 36276f239b6..5bf94cbe07f 100644
--- a/internal/pkg/agent/application/upgrade/upgrade.go
+++ b/internal/pkg/agent/application/upgrade/upgrade.go
@@ -386,14 +386,16 @@ func copyDir(l *logger.Logger, from, to string, ignoreErrs bool) error {
 		}
 	}
 
+	// Try to detect if we are running with SSDs. If we are increase the copy concurrency,
+	// otherwise fall back to the default.
+	copyConcurrency := 1
 	block, err := ghw.Block()
 	if err != nil {
-		return fmt.Errorf("ghw.Block() returned error: %w", err)
-	}
-
-	copyConcurrency := 1
-	if install.HasAllSSDs(*block) {
-		copyConcurrency = runtime.NumCPU() * 4
+		l.Warnw("Error detecting block storage type", "error.message", err)
+	} else {
+		if install.HasAllSSDs(*block) {
+			copyConcurrency = runtime.NumCPU() * 4
+		}
 	}
 
 	return copy.Copy(from, to, copy.Options{

From ee09aba8df280a330e9404ba81e3324bb3481915 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 24 Oct 2023 21:27:04 +0200
Subject: [PATCH 45/52] Complete test pipeline

---
 .buildkite/pipeline.yml | 178 +++++++++++++++++++++-------------------
 .ci/Jenkinsfile         |  52 +-----------
 2 files changed, 94 insertions(+), 136 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 1cf48c53eaa..859440c1dad 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,35 +4,35 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
-  # - label: "Unit tests - Ubuntu 22.04"
-  #   key: "unit-tests-2204"
-  #   command: ".buildkite/scripts/steps/unit-tests.sh"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #     - "coverage.out"
-  #   agents:
-  #     provider: "gcp"
-  #     image: "family/core-ubuntu-2204"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Ubuntu 22.04"
+    key: "unit-tests-2204"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
 
-  # - label: "Unit tests - Ubuntu 22.04 ARM64"
-  #   key: "unit-tests-2204-arm64"
-  #   command: ".buildkite/scripts/steps/unit-tests.sh"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #     - "coverage.out"
-  #   agents:
-  #     provider: "aws"
-  #     imagePrefix: "core-ubuntu-2204-aarch64"
-  #     diskSizeGb: 200
-  #     instanceType: "m6g.4xlarge"
-  #   retry:
-  #     manual:
-  #       allowed: true
+  - label: "Unit tests - Ubuntu 22.04 ARM64"
+    key: "unit-tests-2204-arm64"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "aws"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 200
+      instanceType: "m6g.4xlarge"
+    retry:
+      manual:
+        allowed: true
 
   - label: "Unit tests - Windows 2022"
     key: "unit-tests-win2022"
@@ -76,69 +76,77 @@ steps:
       - "build/diagnostics/*"
       - "coverage.out"
     agents:
-      provider: "orka"
+      provider: orka
       imagePrefix: generic-13-ventura-arm
     retry:
       manual:
         allowed: true
 
-  # - label: "Merge coverage reports"
-  #   key: "merge-coverage"
-  #   env:
-  #     BUILDKITE_REPO: "" # skip checkout
-  #   command: "
-  #     .buildkite/scripts/steps/merge.sh
-  #     unit-tests-2204
-  #     unit-tests-2204-arm64
-  #     unit-tests-win2016
-  #     unit-tests-win2022
-  #     "
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #   agents:
-  #     image: "golang:1.20.10"
-  #   depends_on:
-  #     - unit-tests-2204
-  #     - unit-tests-2204-arm64
-  #     - unit-tests-win2022
-  #     - unit-tests-win2016
-  #   allow_dependency_failure: true
+  - label: "Unit tests - MacOS 13"
+    key: "unit-tests-macos-13"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: orka
+      imagePrefix: generic-13-ventura-x64
+    retry:
+      manual:
+        allowed: true
 
-  # - label: "Serverless integration test"
-  #   key: "serverless-integration-tests"
-  #   command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "gcp"
-  #     machineType: "n1-standard-8"
+  - label: "Merge coverage reports"
+    key: "merge-coverage"
+    env:
+      BUILDKITE_REPO: "" # skip checkout
+    command: "
+      .buildkite/scripts/steps/merge.sh
+      unit-tests-2204
+      unit-tests-2204-arm64
+      unit-tests-win2016
+      unit-tests-win2022
+      unit-tests-macos-13
+      unit-tests-macos-13-arm
+      "
+    artifact_paths:
+      - "build/TEST-**"
+    agents:
+      image: "golang:1.20.10"
+    depends_on:
+      - unit-tests-2204
+      - unit-tests-2204-arm64
+      - unit-tests-win2022
+      - unit-tests-win2016
+      - unit-tests-macos-13
+      - unit-tests-macos-13-arm
+    allow_dependency_failure: true
 
-  # - label: "Integration tests"
-  #   key: "integration-tests"
-  #   command: ".buildkite/scripts/steps/integration_tests.sh stateful"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "gcp"
-  #     machineType: "n1-standard-8"
+  - label: "Serverless integration test"
+    key: "serverless-integration-tests"
+    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
 
-  # - label: "Integration tests"
-  #   key: "integration-tests"
-  #   command: ".buildkite/scripts/steps/integration_tests.sh"
-  #   artifact_paths:
-  #     - "build/TEST-**"
-  #     - "build/diagnostics/*"
-  #   agents:
-  #     provider: "gcp"
-  #     machineType: "n1-standard-8"
+  - label: "Integration tests"
+    key: "integration-tests"
+    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
 
-  # - wait: ~
-  #   continue_on_failure: true
-  # - label: "Processing test results"
-  #   agents:
-  #     provider: "gcp"
-  #   plugins:
-  #     - junit-annotate#v2.4.1:
-  #         artifacts: build/TEST-go-*.xml
+  - wait: ~
+    continue_on_failure: true
+  - label: "Processing test results"
+    agents:
+      provider: "gcp"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: build/TEST-go-*.xml
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index fbaad654c87..b18ad801180 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -95,57 +95,7 @@ pipeline {
             values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64'
           }
         }
-        // stages {
-        //   stage('build'){
-        //     steps {
-        //       withGithubNotify(context: "Build-${PLATFORM}") {
-        //         deleteDir()
-        //         unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-        //         withMageEnv(){
-        //           dir("${BASE_DIR}"){
-        //             cmd(label: 'Go build', script: 'mage build')
-        //           }
-        //         }
-        //       }
-        //     }
-        //   }
-        //   stage('Test') {
-        //     steps {
-        //       withGithubNotify(context: "Test-${PLATFORM}") {
-        //         withMageEnv(){
-        //           dir("${BASE_DIR}"){
-        //             withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
-        //               cmd(label: 'Go unitTest', script: 'mage unitTest')
-        //             }
-        //           }
-        //         }
-        //       }
-        //     }
-        //     post {
-        //       always {
-        //         junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
-        //         whenTrue(isCodeCoverageEnabled()) {
-        //           coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
-        //         }
-        //       }
-        //     }
-        //   }
-          // stage('Integration tests') {
-          //   when {
-          //     // Always when running builds on branches/tags
-          //     // Enable if integration test related changes.
-          //     beforeAgent true
-          //     anyOf {
-          //       not { changeRequest() }
-          //       expression { return params.integration_tests_ci }
-          //       expression { return env.GITHUB_COMMENT?.contains('integration tests') }
-          //       expression { matchesPrLabel(label: 'ci:integration') }
-          //     }
-          //   }
-          //   steps {
-          //     echo 'TBD'
-          //   }
-          // }
+        stages {        
           stage('K8s') {
             when {
               beforeAgent true

From 2f264ef1efcb3143c2640278878e7a9f4b113ba1 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 25 Oct 2023 14:32:28 +0200
Subject: [PATCH 46/52] Complete test pipeline

---
 .buildkite/pipeline.yml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 859440c1dad..f16076ab7e0 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -99,7 +99,7 @@ steps:
   - label: "Merge coverage reports"
     key: "merge-coverage"
     env:
-      BUILDKITE_REPO: "" # skip checkout
+      BUILDKITE_REPO: ""
     command: "
       .buildkite/scripts/steps/merge.sh
       unit-tests-2204
@@ -122,6 +122,20 @@ steps:
       - unit-tests-macos-13-arm
     allow_dependency_failure: true
 
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command:
+      - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
+      - "/scan-source-code.sh"
+    depends_on:
+      - "merge-coverage"
+    retry:
+      manual:
+        allowed: true
+
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
     command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite

From 33f21f22d18ffa0671885e1cc9f9dabe6aed99cb Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 25 Oct 2023 15:10:31 +0200
Subject: [PATCH 47/52] Remove macos GHA workflow

---
 .github/workflows/macos.yml | 23 -----------------------
 1 file changed, 23 deletions(-)
 delete mode 100644 .github/workflows/macos.yml

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
deleted file mode 100644
index 23a7c6a4da8..00000000000
--- a/.github/workflows/macos.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: macos
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-      - 8.*
-
-jobs:
-  macos:
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-go@v3
-      with:
-        go-version-file: .go-version
-    - name: Install dependencies
-      run:  go install github.com/magefile/mage
-    - name: Run build
-      run: mage build
-    - name: Run test
-      run: mage unitTest

From b5a72cd4c52dce8748c6c71e0cb56638777360a2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 26 Oct 2023 00:26:39 +0200
Subject: [PATCH 48/52] Fix Processing test results step

---
 .buildkite/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f16076ab7e0..2ae836321ce 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -163,4 +163,4 @@ steps:
       provider: "gcp"
     plugins:
       - junit-annotate#v2.4.1:
-          artifacts: build/TEST-go-*.xml
+          artifacts: build/TEST-go-integration*.xml

From 609f062b720b23d2fc3b0343e4f93adc6e467b70 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 26 Oct 2023 15:34:30 +0200
Subject: [PATCH 49/52] fix k8s jenkins step

---
 .ci/Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index b18ad801180..e615c8c1bab 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -110,6 +110,8 @@ pipeline {
               }
             }
             steps {
+              deleteDir()
+              unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
               runK8s(k8sVersion: 'v1.27.3', kindVersion: 'v0.20.0', context: "K8s-${PLATFORM}")
             }
           }

From f8b31276f346a085cd15246cec196dcc73ca6c3b Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 26 Oct 2023 16:50:21 +0200
Subject: [PATCH 50/52] Removed m1 stage

---
 .ci/Jenkinsfile | 72 +------------------------------------------------
 1 file changed, 1 insertion(+), 71 deletions(-)

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index e615c8c1bab..5cde7f706ed 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -300,77 +300,7 @@ pipeline {
           }
         }
       }
-    }
-    stage('m1') {
-      agent { label 'orka && darwin && aarch64' }
-      options { skipDefaultCheckout() }
-      when {
-        // Always when running builds on branches/tags
-        // Enable if extended M1 support related changes.
-        beforeAgent true
-        anyOf {
-          not { changeRequest() }
-          expression { return isExtendedM1Enabled() && env.ONLY_DOCS == "false"}
-        }
-      }
-      stages {
-        stage('build'){
-          steps {
-            withGithubNotify(context: "Build-darwin-aarch64") {
-              deleteDir()
-              unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-              withMageEnv(){
-                dir("${BASE_DIR}"){
-                  cmd(label: 'Go build', script: 'mage build')
-                }
-              }
-            }
-          }
-        }
-        stage('Test') {
-          steps {
-            withGithubNotify(context: "Test-darwin-aarch64") {
-              withMageEnv(){
-                dir("${BASE_DIR}"){
-                  withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
-                    cmd(label: 'Go unitTest', script: 'mage unitTest')
-                  }
-                }
-              }
-            }
-          }
-          post {
-            always {
-              junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
-              whenTrue(isCodeCoverageEnabled()) {
-                coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
-              }
-            }
-          }
-        }
-      }
-    }
-    stage('e2e tests') {
-      when {
-        // Disable until we fix the e2e test.
-        // This should be running when there is a comment or the e2e label
-        beforeAgent true
-        anyOf {
-          // package artifacts are not generated if ONLY_DOCS, therefore e2e should not run if ONLY_DOCS
-          expression { return isE2eEnabled() && env.ONLY_DOCS == "false"}
-        }
-      }
-      steps {
-        runE2E(testMatrixFile: '.ci/.e2e-tests-for-elastic-agent.yaml',
-               beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
-               elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT",
-               gitHubCheckName: "e2e-tests",
-               gitHubCheckRepo: env.REPO,
-               gitHubCheckSha1: env.GIT_BASE_COMMIT,
-               propagate: true,
-               wait: true)
-      }
-    }
+    }        
   }
   post {
     cleanup {

From ffc3991e3896ec1fc0a920609c4f96f153fb9ca5 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 30 Oct 2023 14:47:48 +0100
Subject: [PATCH 51/52] Enabled race detector for unit tests

---
 .buildkite/scripts/steps/unit-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/unit-tests.sh b/.buildkite/scripts/steps/unit-tests.sh
index 6549b4514d3..6df75a26a08 100644
--- a/.buildkite/scripts/steps/unit-tests.sh
+++ b/.buildkite/scripts/steps/unit-tests.sh
@@ -4,6 +4,6 @@ set -euxo pipefail
 source .buildkite/scripts/common.sh
 
 echo "--- Unit tests"
-TEST_COVERAGE=true mage unitTest
+RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
 # Copy coverage file to build directory so it can be downloaded as an artifact
 cp build/TEST-go-unit.cov coverage.out
\ No newline at end of file

From 12a13043045fa7372338f5255abb368105b8942e Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 30 Oct 2023 14:50:52 +0100
Subject: [PATCH 52/52] Enables race detector for windows

---
 .buildkite/scripts/steps/unit-tests.ps1 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
index 18b17c6a631..e1645554532 100644
--- a/.buildkite/scripts/steps/unit-tests.ps1
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -16,6 +16,7 @@ if ($LASTEXITCODE -ne 0) {
 
 Write-Host "--- Unit tests"
 $env:TEST_COVERAGE = $true
+$env:RACE_DETECTOR = $true
 mage unitTest
 # Copy coverage file to build directory so it can be downloaded as an artifact
 cp .\build\TEST-go-unit.cov coverage.out