Skip to content
Build Jwst

feat: sync y-octo's logic #2139

Sign in to view logs

feat: sync y-octo's logic

feat: sync y-octo's logic #2139

Workflow file for this run

.github/workflows/jwst.yml at 9fb6c2f
name: Build Jwst
on:
workflow_dispatch:
push:
branches: [master]
paths:
# - "apps/cloud/**"
- "apps/homepage/**"
- "apps/keck/**"
- "libs/**"
# - ".github/deployment/Dockerfile-cloud"
- ".github/deployment/Dockerfile-jwst"
# - ".github/deployment/Dockerfile-mt-cloud"
- ".github/workflows/jwst.yml"
pull_request:
branches: [master]
paths:
# - "apps/cloud/**"
- "apps/homepage/**"
- "apps/playground/**"
- "apps/keck/**"
- "libs/**"
# - ".github/deployment/Dockerfile-cloud"
- ".github/deployment/Dockerfile-jwst"
# - ".github/deployment/Dockerfile-mt-cloud"
- ".github/workflows/jwst.yml"
# Cancels all previous workflow runs for pull requests that have not completed.
# See https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
# The concurrency group contains the workflow name and the branch name for
# pull requests or the commit hash for any other events.
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
cancel-in-progress: true
env:
REGISTRY: ghcr.io
NAMESPACE: toeverything
APIPROXY_IMAGE_NAME: apiproxy
CLOUD_IMAGE_NAME: cloud-self-hosted
JWST_IMAGE_NAME: jwst
IMAGE_TAG: canary-${{ github.sha }}
IMAGE_TAG_LATEST: nightly-latest
LOCAL_CACHE: localhost:5000/toeverything/relocate:latest
IMAGE_TAG_MT: canary-mt-${{ github.sha }}
IMAGE_TAG_MT_LATEST: mt-latest
jobs:
# aio-cloud-baseline:
# if: github.ref == 'refs/heads/master'
# runs-on: [self-hosted, linux, x64]
# environment: development
# permissions:
# contents: read
# packages: write
# services:
# registry:
# image: registry:2
# ports:
# - 5000:5000
# steps:
# - name: Checkout
# uses: actions/checkout@v2
# - name: Install Rust
# uses: actions-rs/toolchain@v1
# with:
# profile: minimal
# toolchain: stable
# override: true
# - uses: Swatinem/rust-cache@v2
# - name: Set up pnpm cache
# uses: actions/cache@v3
# continue-on-error: false
# with:
# path: ./node_modules/.pnpm-store
# key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
# restore-keys: ${{ runner.os }}-pnpm-
# - name: Cargo Vendor
# run: cargo vendor > .cargo/config
# - name: Log in to the Container registry
# uses: docker/login-action@v2
# if: github.ref == 'refs/heads/master'
# with:
# registry: ${{ env.REGISTRY }}
# username: ${{ github.actor }}
# password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v2
# - name: Set up Docker Buildx
# id: buildx
# uses: docker/setup-buildx-action@v2
# with:
# driver-opts: network=host
# - name: Build and push Docker image (cloud-mt-baseline)
# uses: docker/build-push-action@v3
# if: github.ref == 'refs/heads/master'
# with:
# context: .
# file: ./.github/deployment/Dockerfile-mt-cloud
# push: true
# tags: ${{ env.LOCAL_CACHE }}
# target: jwst
# - name: Get current time
# id: time
# run: echo "::set-output name=time::$(date +'%Y%m%d-%H%M')"
# - name: Extract metadata (tags, labels) for Docker (mt-cloud)
# id: meta_mt_cloud
# uses: docker/metadata-action@v4
# with:
# images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.CLOUD_IMAGE_NAME }}
# tags: |
# ${{ env.IMAGE_TAG_MT }}-${{ steps.time.outputs.time }}
# ${{ env.IMAGE_TAG_MT_LATEST }}
# - name: Build and push Docker image (mt-cloud)
# uses: docker/build-push-action@v3
# with:
# context: .
# platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
# file: ./.github/deployment/Dockerfile-mt-cloud-prod
# push: ${{ github.ref == 'refs/heads/master' && true || false }}
# tags: ${{ steps.meta_mt_cloud.outputs.tags }}
# labels: ${{ steps.meta_mt_cloud.outputs.labels }}
# target: affine
# build-args: |
# BASE_IMAGE=${{ env.LOCAL_CACHE }}
# cloud:
# runs-on: [self-hosted, linux, x64]
# environment: development
# permissions:
# contents: read
# packages: write
# steps:
# - name: Checkout
# uses: actions/checkout@v2
# - name: Install Rust
# uses: actions-rs/toolchain@v1
# with:
# profile: minimal
# toolchain: stable
# override: true
# - uses: Swatinem/rust-cache@v2
# - name: Set up pnpm cache
# uses: actions/cache@v3
# continue-on-error: false
# with:
# path: ./node_modules/.pnpm-store
# key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
# restore-keys: ${{ runner.os }}-pnpm-
# - name: Cargo Vendor
# run: cargo vendor > .cargo/config
# - name: Log in to the Container registry
# uses: docker/login-action@v2
# if: github.ref == 'refs/heads/master'
# with:
# registry: ${{ env.REGISTRY }}
# username: ${{ github.actor }}
# password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}
# - name: Get current time
# id: time
# run: echo "::set-output name=time::$(date +'%H%M')"
# - name: Extract metadata (tags, labels) for Docker (cloud)
# id: meta_cloud
# uses: docker/metadata-action@v4
# with:
# images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.CLOUD_IMAGE_NAME }}
# tags: |
# ${{ env.IMAGE_TAG }}-${{ steps.time.outputs.time }}
# ${{ env.IMAGE_TAG_LATEST }}
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v2
# - name: Set up Docker Buildx
# id: buildx
# uses: docker/setup-buildx-action@v2
# with:
# driver-opts: network=host
# - name: Build Docker image (pull request)
# uses: docker/build-push-action@v3
# if: github.ref != 'refs/heads/master'
# with:
# context: .
# file: ./.github/deployment/Dockerfile-cloud
# tags: ${{ steps.meta_cloud.outputs.tags }}
# labels: ${{ steps.meta_cloud.outputs.labels }}
# target: cloud
# - name: Build and push Docker image (cloud)
# uses: docker/build-push-action@v3
# if: github.ref == 'refs/heads/master'
# with:
# context: .
# file: ./.github/deployment/Dockerfile-cloud
# push: true
# tags: ${{ steps.meta_cloud.outputs.tags }}
# labels: ${{ steps.meta_cloud.outputs.labels }}
# target: cloud
jwst:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: pnpm/action-setup@v2
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
target: x86_64-unknown-linux-musl
override: true
- uses: Swatinem/rust-cache@v2
- name: Set up pnpm cache
uses: actions/cache@v3
continue-on-error: false
with:
path: ./node_modules/.pnpm-store
key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: ${{ runner.os }}-pnpm-
- name: Build & Check
run: |
sudo apt install -yq musl-tools musl-dev
pnpm i --frozen-lockfile --store=node_modules/.pnpm-store
cargo vendor > .cargo/config
cargo build --profile fast-release --package keck --target x86_64-unknown-linux-musl
env:
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: x86_64-linux-musl-gcc
CARGO_TERM_COLOR: always
- name: Log in to the Container registry
if: github.ref == 'refs/heads/master'
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}
- name: Get current time
id: time
run: echo "::set-output name=time::$(date +'%H%M')"
- name: Extract metadata (tags, labels) for Docker
id: meta_jwst
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.JWST_IMAGE_NAME }}
tags: |
${{ env.IMAGE_TAG }}-${{ steps.time.outputs.time }}
${{ env.IMAGE_TAG_LATEST }}
- name: Build and push Docker image (jwst)
uses: docker/build-push-action@v3
with:
context: .
file: ./.github/deployment/Dockerfile-jwst
push: ${{ github.ref == 'refs/heads/master' && true || false }}
tags: ${{ steps.meta_jwst.outputs.tags }}
labels: ${{ steps.meta_jwst.outputs.labels }}
target: jwst
network: host
# apiproxy:
# if: github.ref == 'refs/heads/master'
# runs-on: ubuntu-latest
# environment: development
# permissions:
# contents: read
# packages: write
# steps:
# - name: Checkout
# uses: actions/checkout@v2
# - name: Log in to the Container registry
# uses: docker/login-action@v2
# with:
# registry: ${{ env.REGISTRY }}
# username: ${{ github.actor }}
# password: ${{ secrets.ACTIONS_PACKAGE_PUBLISH }}
# - name: Extract metadata (tags, labels) for Docker (apiproxy)
# id: meta_apiproxy
# uses: docker/metadata-action@v4
# with:
# images: ${{ env.REGISTRY }}/${{ env.NAMESPACE }}/${{ env.APIPROXY_IMAGE_NAME }}
# tags: |
# ${{ env.IMAGE_TAG }}
# ${{ env.IMAGE_TAG_LATEST }}
# - name: Build and push Docker image (apiproxy)
# uses: docker/build-push-action@v3
# with:
# context: .
# file: ./.github/deployment/Dockerfile-apiproxy
# push: ${{ github.ref == 'refs/heads/master' && true || false }}
# tags: ${{ steps.meta_apiproxy.outputs.tags }}
# labels: ${{ steps.meta_apiproxy.outputs.labels }}
# target: apiproxy
lint:
runs-on: ubuntu-latest
permissions:
contents: read
security-events: write
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
components: clippy
override: true
- uses: Swatinem/rust-cache@v2
- name: Set up pnpm cache
uses: actions/cache@v3
continue-on-error: false
with:
path: ./node_modules/.pnpm-store
key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: ${{ runner.os }}-pnpm-
- name: Install required cargo components
uses: taiki-e/install-action@v2
with:
tool: clippy-sarif,sarif-fmt
- name: Build & Check
run: |
cargo vendor > .cargo/config
cargo clippy --all-features --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt
RUSTDOCFLAGS="-D rustdoc::broken-intra-doc-links" cargo doc --workspace --all-features --no-deps
env:
CARGO_TERM_COLOR: always
- name: Upload analysis results to GitHub
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: rust-clippy-results.sarif
wait-for-processing: true
test:
name: test & collect coverage
runs-on: ubuntu-latest
continue-on-error: true
env:
RUSTFLAGS: -D warnings
CARGO_TERM_COLOR: always
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview
- uses: Swatinem/rust-cache@v2
- name: Install latest nextest release
uses: taiki-e/install-action@nextest
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
- name: Collect coverage data
run: cargo llvm-cov nextest --all-targets --lcov --output-path lcov.info
- name: Upload coverage data to codecov
uses: codecov/codecov-action@v3
with:
name: tests
files: lcov.info
loom:
name: loom thread test
runs-on: ubuntu-latest
continue-on-error: true
env:
RUSTFLAGS: --cfg loom
RUST_BACKTRACE: full
CARGO_TERM_COLOR: always
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Install latest nextest release
uses: taiki-e/install-action@nextest
- name: Loom Thread Test
run: |
cargo nextest run -p jwst-codec --lib
fuzzing:
name: fuzzing
runs-on: ubuntu-latest
continue-on-error: true
env:
RUSTFLAGS: -D warnings
CARGO_TERM_COLOR: always
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@master
with:
toolchain: nightly-2023-05-23
- uses: Swatinem/rust-cache@v2
- name: fuzzing
working-directory: ./libs/jwst-codec-utils
run: |
cargo install cargo-fuzz
cargo fuzz run apply_update -- -max_total_time=30
cargo fuzz run codec_doc_any_struct -- -max_total_time=30
cargo fuzz run codec_doc_any -- -max_total_time=30
cargo fuzz run decode_bytes -- -max_total_time=30
cargo fuzz run i32_decode -- -max_total_time=30
cargo fuzz run i32_encode -- -max_total_time=30
cargo fuzz run ins_del_text -- -max_total_time=30
cargo fuzz run sync_message -- -max_total_time=30
cargo fuzz run u64_decode -- -max_total_time=30
cargo fuzz run u64_encode -- -max_total_time=30
cargo fuzz run apply_update -- -max_total_time=30
- name: upload fuzz artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: fuzz-artifact
path: ./lib/jwst-codec-utils/fuzz/artifacts/**/*