userbenchmark-a100-bisection.yml

name: TorchBench A100 bisection
on:
  workflow_dispatch:
    inputs:
      start_commit:
        description: "Start PyTorch commit hash"
        required: true
      end_commit:
        description: "End PyTorch commit hash"
        required: true
      userbenchmark:
        description: "Userbenchmark name"
        required: true
      userbenchmark_args:
        description: "Userbenchmark arguments"
        required: true

jobs:
  bisection:
    environment: docker-s3-upload
    env:
      BASE_CONDA_ENV: "torchbench"
      CONDA_ENV: "bisection-ci-a100"
      PLATFORM_NAME: "gcp_a100"
      SETUP_SCRIPT: "/workspace/setup_instance.sh"
      BISECT_WORKDIR: ".userbenchmark/${{ github.event.inputs.userbenchmark }}/bisection"
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
    if: ${{ github.repository_owner == 'pytorch' }}
    runs-on: linux.aws.a100
    timeout-minutes: 2880 # 48 hours
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          path: benchmark
      - name: Checkout pytorch
        uses: actions/checkout@v3
        with:
          repository: pytorch/pytorch
          path: srcs/pytorch
          fetch-depth: 0
      - name: Checkout torchvision
        uses: actions/checkout@v3
        with:
          repository: pytorch/vision
          path: srcs/vision
          fetch-depth: 0
      - name: Checkout torchaudio
        uses: actions/checkout@v3
        with:
          repository: pytorch/audio
          path: srcs/audio
          fetch-depth: 0
      - name: Install devel packages
        run: |
          sudo apt-get update -y
          sudo apt-get update -y
          sudo apt-get install -y libjpeg-dev zlib1g-dev libpng-dev
          sudo ldconfig
          sudo ldconfig
      - name: Setup conda env
        run: |
          CONDA_ENV=${BASE_CONDA_ENV} . "${SETUP_SCRIPT}"
          cd benchmark
          mkdir -p "${BISECT_WORKDIR}"
          PYTORCH_GIT_HASH=$(python -c 'import torch; print(torch.version.git_version)')
          python run_benchmark.py ${{ github.event.inputs.userbenchmark }} ${{ github.event.inputs.userbenchmark_args }} --dryrun \
                 --output "${BISECT_WORKDIR}/metrics-control.json"
          sed -i "s/${PYTORCH_GIT_HASH}/${{ github.event.inputs.start_commit }}/g" "${BISECT_WORKDIR}/metrics-control.json"
          python run_benchmark.py ${{ github.event.inputs.userbenchmark }} ${{ github.event.inputs.userbenchmark_args }} --dryrun \
                 --output "${BISECT_WORKDIR}/metrics-treatment.json"
          sed -i "s/${PYTORCH_GIT_HASH}/${{ github.event.inputs.end_commit }}/g" "${BISECT_WORKDIR}/metrics-treatment.json"
          python regression_detector.py \
                 --control "${BISECT_WORKDIR}/metrics-control.json" --treatment "${BISECT_WORKDIR}/metrics-treatment.json" \
                 --output "${BISECT_WORKDIR}/regression-gh${GITHUB_RUN_ID}.yaml"
          python ./utils/python_utils.py --create-conda-env "${CONDA_ENV}"
      - name: Setup bisection environment
        run: |
          . "${SETUP_SCRIPT}"; cd benchmark
          python utils/cuda_utils.py --install-torch-build-deps
          python utils/cuda_utils.py --install-torchbench-deps
      - name: Bisection
        run: |
          . "${SETUP_SCRIPT}"; cd benchmark
          python bisection.py --work-dir "${BISECT_WORKDIR}" --torch-repos-path "${PWD}/../srcs" \
                --torchbench-repo-path "${PWD}" --config "${BISECT_WORKDIR}/regression-gh${GITHUB_RUN_ID}.yaml" \
                --output "${BISECT_WORKDIR}/bisect-output-gh${GITHUB_RUN_ID}.json"
          cp -r "${BISECT_WORKDIR}" ../bisection-result
      - name: Upload artifact
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: Bisection result
          path: bisection-result/
      - name: Clean up Conda env
        if: always()
        run: |
          . "${SETUP_SCRIPT}"
          conda deactivate && conda deactivate
          conda remove -n "${CONDA_ENV}" --all