Skip to content

Commit

Permalink
Separate builder and runner image
Browse files Browse the repository at this point in the history
  • Loading branch information
erikzaadi committed Oct 21, 2024
1 parent ac745b3 commit 13c390a
Show file tree
Hide file tree
Showing 9 changed files with 202 additions and 118 deletions.
82 changes: 82 additions & 0 deletions .github/workflows/actions/build-docker-image/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
name: Build docker images
description: Build Docker Images
inputs:
dockerfile:
description: Dockerfile to build
required: true
tags:
description: Docker tags to publish
required: true
platforms:
description: Platforms to build (csv)
required: false
default: 'linux/arm64,linux/amd64'
test:
description: Test command to run on the created image (Optional)
required: false
default: ''
build-args:
description: Explicit docker build-args
required: false
default: ''
skip-init:
description: Skip docker init (if ran after another invocation of this action)
required: false
default: ''
docker-user:
required: true
description: Docker Hub User
docker-password:
required: true
description: Docker Hub User
skip-push:
required: false
description: Optionally skip push
default: ''
load-created-image:
required: false
description: Optionally load created docker image
default: ''

runs:
using: 'composite'
steps:
# - name: Setup docker (missing on MacOS)
# if: matrix.platform == 'linux/arm64'
# uses: douglascamata/setup-docker-macos-action@v1-alpha

- name: Set up QEMU
uses: docker/setup-qemu-action@v3
if: ${{ inputs.skip-init == '' }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
if: ${{ inputs.skip-init == '' }}

- name: Login to Docker Hub
uses: docker/login-action@v3
if: ${{ inputs.skip-init == '' }}
with:
registry: ghcr.io
username: ${{ inputs.docker-user }}
password: ${{ inputs.docker-password }}

- name: Build Runner Image
uses: docker/build-push-action@v6
with:
context: .
file: ${{ inputs.dockerfile }}
platforms: ${{ inputs.platforms }}
push: ${{ inputs.skip-push == '' }}
load: ${{ inputs.test != '' || inputs.load-created-image != '' }}
tags: ${{ inputs.tags }}
build-args: |
${{ inputs.build-args }}
- name: Verify Built Image
shell: bash
if: ${{ inputs.test != '' }}
run: |
SINGLE_TAG=$(echo "${{ inputs.tags }}" | awk -F ',' '{print $1};' )
SINGLE_PLATFORM=$(echo "${{ inputs.platforms }}" | awk -F ',' '{print $1};' )
docker run --platform "${SINGLE_PLATFORM}" --rm --entrypoint bash "${SINGLE_TAG}" -c '${{ inputs.test }}'
50 changes: 6 additions & 44 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
name: Build integration images
on:
pull_request:
workflow_dispatch:

jobs:
prepare-matrix:
Expand Down Expand Up @@ -39,7 +38,6 @@ jobs:
build-integration:
# runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }}
runs-on: 'ubuntu-latest'
if: needs.prepare-matrix.outputs.matrix != '[]'
outputs:
Expand All @@ -58,20 +56,6 @@ jobs:
- name: Check out code
uses: actions/checkout@v4

# - name: Setup docker (missing on MacOS)
# if: matrix.platform == 'linux/arm64'
# uses: douglascamata/setup-docker-macos-action@v1-alpha

- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: ${{ matrix.platform }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
platforms: ${{ matrix.platform }}

- name: Prepare Docker images tags
id: prepare_tags
run: |
Expand Down Expand Up @@ -106,37 +90,15 @@ jobs:
echo "is_dev_version=false" >> $GITHUB_OUTPUT
fi
- name: Get used docker base image
id: get-docker-image
run: |
echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT
- name: Login to Docker Hub
uses: docker/login-action@v3
- name: Build Docker Image
uses: ./.github/workflows/actions/build-docker-image
with:
registry: ghcr.io
username: ${{ secrets.DOCKER_MACHINE_USER }}
password: ${{ secrets.DOCKER_MACHINE_TOKEN }}

# - name: Cache Docker images
# uses: ScribeMD/[email protected]
# with:
# key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }}

- name: Build
uses: docker/build-push-action@v6
with:
context: .
file: ${{ steps.prepare_tags.outputs.dockerfile_path }}
dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }}
platforms: ${{ matrix.platform }}
push: false
load: true
tags: ${{ steps.prepare_tags.outputs.tags }}
build-args: |
BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
- name: Verify Built Image
run: |
SINGLE_TAG=$(echo "${{ steps.prepare_tags.outputs.tags }}" | awk -F ',' '{print $1};' )
docker run --platform ${{ matrix.platform }} --rm --entrypoint bash "${SINGLE_TAG}" -c 'ocean version'
docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
skip-push: 'yupp'
9 changes: 9 additions & 0 deletions .github/workflows/detect-changes-matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ on:
core:
value: ${{ jobs.detect-changes.outputs.core }}
description: "Determine if any core changes per git commit changes"
infra:
value: ${{ jobs.detect-changes.outputs.infra }}
description: "Determine if any changes to docker infra"

jobs:
detect-changes:
Expand All @@ -20,6 +23,7 @@ jobs:
matrix: ${{ steps.set-all-matrix.outputs.matrix }}
integrations: ${{ steps.set-all-matrix.outputs.integrations }}
core: ${{ steps.set-all-matrix.outputs.core }}
infra: ${{ steps.set-all-matrix.outputs.infra}}
steps:
- name: Checkout Repo
uses: actions/checkout@v4
Expand All @@ -42,6 +46,8 @@ jobs:
- 'integrations/**'
- '!integrations/**/*.md'
- '!integrations/_infra/*'
infra:
- 'integrations/_infra/*'
- name: Set integrations and all matrix
id: set-all-matrix
Expand All @@ -50,8 +56,11 @@ jobs:
HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }}
echo "Core changes : ${HAS_CORE}"
MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))")
HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }}
echo "Infra changes : ${HAS_INFRA}"
echo "Integration changes : ${INTEGRATIONS}"
echo "All changes : ${MATRIX}"
echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT
echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT
echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT
49 changes: 21 additions & 28 deletions .github/workflows/docker-images-security-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,34 @@ on:
# This is a bit annoying, there's no real way to display the integrations dynamically in a dropdown for the action dispatcher
options:
- all
- argocd
- aws
- azure
- azure-devops
- backstage
- datadog
- dynatrace
- fake-integration
- gcp
- jenkins
- kafka
- launchdarkly
- newrelic
- opencost
- pagerduty
- servicenow
- sonarqube
- terraform-cloud
- argocd
- azure
- datadog
- firehydrant
- gcp
- gitlab
- jenkins
- jira
- kafka
- kubecost
- launchdarkly
- linear
- newrelic
- octopus
- opencost
- opsgenie
- pagerduty
- sentry
- servicenow
- snyk
- sonarqube
- statuspage
- terraform-cloud
- wiz

jobs:
Expand Down Expand Up @@ -77,13 +78,6 @@ jobs:
- name: Checkout Repo
uses: actions/checkout@v4

- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/amd64,linux/arm64

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Extract version and image tag
id: enrich-version
Expand All @@ -97,17 +91,16 @@ jobs:
echo "identifier=${IDENTIFIER}" >> ${GITHUB_OUTPUT}
echo "image_tag=${IMAGE_FULL_TAG}" >> ${GITHUB_OUTPUT}
- name: Build
uses: docker/build-push-action@v6
- name: Build Docker Image
uses: ./.github/workflows/actions/build-docker-image
with:
context: .
file: ./integrations/_infra/Dockerfile
dockerfile: ./integrations/_infra/Dockerfile
platforms: linux/amd64
push: false
skip-push: 'yupp'
tags: ${{ steps.enrich-version.outputs.image_tag }}
load: true
cache-from: type=gha
cache-to: type=gha,mode=max
load-created-image: 'yupp'
docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
build-args: |
BUILD_CONTEXT=./integrations/${{ steps.enrich-version.outputs.integration }}
INTEGRATION_VERSION=${{ steps.enrich-version.outputs.version }}
Expand Down
36 changes: 36 additions & 0 deletions .github/workflows/infra.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: Build infra images
on:
pull_request:
workflow_dispatch:

jobs:
detect-changes:
uses: ./.github/workflows/detect-changes-matrix.yml
build-infra:
# runs-on: ${{ matrix.platform == 'linux/arm64' && 'macos-13' || 'ubuntu-latest' }}
runs-on: 'ubuntu-latest'
needs: detect-changes
if: ${{ needs.detect-changes.outputs.infra == 'true' }}
steps:
- name: Check out code
uses: actions/checkout@v4

- name: Build Docker Image
uses: ./.github/workflows/actions/build-docker-image
with:
dockerfile: ./integrations/_infra/Dockerfile.base.builder
platforms: linux/amd64,linux/arm64
tags: ghcr.io/port-labs/port-ocean-base-builder:latest
docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}


- name: Build Docker Image
uses: ./.github/workflows/actions/build-docker-image
with:
dockerfile: ./integrations/_infra/Dockerfile.base.runner
platforms: linux/amd64,linux/arm64
tags: ghcr.io/port-labs/port-ocean-base-runner:latest
docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
skip-init: 'yupp'
30 changes: 7 additions & 23 deletions .github/workflows/release-integrations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.DOCKER_MACHINE_USER }}
password: ${{ secrets.DOCKER_MACHINE_TOKEN }}

- name: Prepare matrix
id: prepare-matrix
run: |
Expand Down Expand Up @@ -63,18 +58,7 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/amd64,linux/arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.DOCKER_MACHINE_USER }}
password: ${{ secrets.DOCKER_MACHINE_TOKEN }}

- name: Prepare Docker images tags
id: prepare_tags
run: |
Expand Down Expand Up @@ -108,17 +92,17 @@ jobs:
echo "is_dev_version=false" >> $GITHUB_OUTPUT
fi
- name: Build and push
uses: docker/build-push-action@v6
- name: Build Docker Image
uses: ./.github/workflows/actions/build-docker-image
with:
context: .
file: ${{ steps.prepare_tags.outputs.dockerfile_path }}
dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.prepare_tags.outputs.tags }}
build-args: |
BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}

upload-specs:
runs-on: ubuntu-latest
Expand Down
Loading

0 comments on commit 13c390a

Please sign in to comment.