Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.11](backport #3573) [CI] Pull requests migration to buildkite #3720

Merged
merged 1 commit into from
Nov 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions .buildkite/hooks/post-checkout.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Set error handling
$ErrorActionPreference = "Stop"

# Define a function to checkout and merge
function Checkout-Merge {
param (
[string]$targetBranch,
[string]$prCommit,
[string]$mergeBranch
)

if (-not $targetBranch) {
Write-Host "No pull request target branch"
exit 1
}

git fetch -v origin $targetBranch
git checkout FETCH_HEAD
Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)"

# Create a temporary branch to merge the PR with the target branch
git checkout -b $mergeBranch
Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)"

# Set author identity so it can be used for git merge
git config user.name "github-merged-pr-post-checkout"
git config user.email "auto-merge@buildkite"

git merge --no-edit $prCommit

if ($LASTEXITCODE -ne 0) {
$mergeResult = $LASTEXITCODE
Write-Host "Merge failed: $mergeResult"
git merge --abort
exit $mergeResult
}
}

$pullRequest = $env:BUILDKITE_PULL_REQUEST

if ($pullRequest -eq "false") {
Write-Host "Not a pull request, skipping"
exit 0
}

$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
$prCommit = $env:BUILDKITE_COMMIT
$prId = $env:BUILDKITE_PULL_REQUEST
$mergeBranch = "pr_merge_$prId"

Checkout-Merge $targetBranch $prCommit $mergeBranch

Write-Host "Commit information"
git --no-pager log --format=%B -n 1

Write-Host "Fixing CRLF in git checkout --"
git config core.autocrlf true

# Ensure Buildkite groups are rendered
Write-Host ""
25 changes: 25 additions & 0 deletions .buildkite/hooks/pre-command.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Install gcc TODO: Move to the VM image
choco install mingw
Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
refreshenv

# Install gvm and go
# TODO: Move GVM download to the base VM image
$env:GvmVersion = "0.5.2"
[Net.ServicePointManager]::SecurityProtocol = "tls12"
$env:GoVersion = Get-Content -Path .go-version
Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
gvm --format=powershell $env:GoVersion | Invoke-Expression
go version

$GOPATH = $(go env GOPATH)
$env:Path = "$GOPATH\bin;" + $env:Path
[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)

# Install tools
go install github.com/magefile/mage
go install github.com/elastic/go-licenser
go install golang.org/x/tools/cmd/goimports
go install github.com/jstemmer/go-junit-report
go install gotest.tools/gotestsum
117 changes: 111 additions & 6 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,134 @@ env:
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
DOCKER_REGISTRY: "docker.elastic.co"
steps:
- label: "Unit tests"
key: "unit-tests"
- label: "Unit tests - Ubuntu 22.04"
key: "unit-tests-2204"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-ubuntu-2204"
retry:
manual:
allowed: true

- label: "Unit tests - Ubuntu 22.04 ARM64"
key: "unit-tests-2204-arm64"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "aws"
imagePrefix: "core-ubuntu-2204-aarch64"
diskSizeGb: 200
instanceType: "m6g.4xlarge"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2022"
key: "unit-tests-win2022"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2022"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2016"
key: "unit-tests-win2016"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2016"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13 ARM"
key: "unit-tests-macos-13-arm"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-arm
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13"
key: "unit-tests-macos-13"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-x64
retry:
manual:
allowed: true

- label: "Merge coverage reports"
key: "merge-coverage"
env:
BUILDKITE_REPO: ""
command: "
.buildkite/scripts/steps/merge.sh
unit-tests-2204
unit-tests-2204-arm64
unit-tests-win2016
unit-tests-win2022
unit-tests-macos-13
unit-tests-macos-13-arm
"
artifact_paths:
- "build/TEST-**"
agents:
image: "golang:1.20.10"
depends_on:
- unit-tests-2204
- unit-tests-2204-arm64
- unit-tests-win2022
- unit-tests-win2016
- unit-tests-macos-13
- unit-tests-macos-13-arm
allow_dependency_failure: true

- label: ":sonarqube: Continuous Code Inspection"
env:
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
agents:
image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
command:
- "buildkite-agent artifact download build/TEST-go-unit.cov ."
- "buildkite-agent artifact download build/TEST-go-unit.out.json ."
- "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
- "/scan-source-code.sh"
depends_on:
- "unit-tests"
- "merge-coverage"
retry:
manual:
allowed: true
Expand All @@ -48,4 +153,4 @@ steps:
provider: "gcp"
plugins:
- junit-annotate#v2.4.1:
artifacts: build/TEST-go-*.xml
artifacts: build/TEST-go-integration*.xml
2 changes: 1 addition & 1 deletion .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ getOSOptions() {
export AGENT_OS_NAME=linux
;;
darwin*)
export AGENT_OS_NAME=osx
export AGENT_OS_NAME=darwin
;;
msys*)
export AGENT_OS_NAME=windows
Expand Down
23 changes: 23 additions & 0 deletions .buildkite/scripts/steps/merge.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash

# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov).
# Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#

set -exuo pipefail

COV_ARTIFACT="coverage.out"
MERGED_COV_FILE="build/TEST-go-unit.cov"
# Space separated list of paths to coverage files
COV_PATHS=""

go install github.com/wadey/gocovmerge@latest
mkdir -p build

for STEP_ID in "$@"; do
mkdir -p $STEP_ID
buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID
COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT"
done

gocovmerge $COV_PATHS > $MERGED_COV_FILE
echo "Merged coverage file: $MERGED_COV_FILE. See artifacts"
28 changes: 28 additions & 0 deletions .buildkite/scripts/steps/unit-tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
$ErrorActionPreference = "Stop"

Write-Host "-- Fixing CRLF in git checkout --"
git config core.autocrlf input
git rm --quiet --cached -r .
git reset --quiet --hard

$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"

Write-Host "--- Build"
mage build

if ($LASTEXITCODE -ne 0) {
exit 1
}

Write-Host "--- Unit tests"
$env:TEST_COVERAGE = $true
$env:RACE_DETECTOR = $true
mage unitTest
# Copy coverage file to build directory so it can be downloaded as an artifact
cp .\build\TEST-go-unit.cov coverage.out

if ($LASTEXITCODE -ne 0) {
exit 1
}


4 changes: 3 additions & 1 deletion .buildkite/scripts/steps/unit-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,6 @@ set -euxo pipefail
source .buildkite/scripts/common.sh

echo "--- Unit tests"
TEST_COVERAGE=true mage unitTest
RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
# Copy coverage file to build directory so it can be downloaded as an artifact
cp build/TEST-go-unit.cov coverage.out
Loading
Loading