Metadata Extract #61
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Metadata Extract | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'The environment to run the workflow in' | |
| required: true | |
| type: choice | |
| default: 'preview' | |
| options: | |
| - 'preview' | |
| - 'production' | |
| schedule: | |
| - cron: '0 9 * * 1' # Run at midnight UTC on Mondays | |
| jobs: | |
| get_variants_list: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| sdk_ex_matrix: ${{ steps.setmatrix_sdk_ex.outputs.sdk_ex_matrix }} | |
| sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }} | |
| airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }} | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Install hub-utils | |
| run: pipx install git+https://github.com/meltano/hub-utils.git | |
| - name: Get Variants List - SDK Extractors | |
| id: get-variants-list-sdk-ex | |
| run: echo "VARIANTS_SDK_EX=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk)" >> $GITHUB_OUTPUT | |
| - name: Set Dynamic Matrix - SDK Extractors | |
| id: setmatrix_sdk_ex | |
| run: | | |
| matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex.outputs.VARIANTS_SDK_EX }}}" | |
| echo "::set-output name=sdk_ex_matrix::$matrixStringifiedObject" | |
| - name: Get Variants List - SDK Loaders | |
| id: get-variants-list-sdk-loaders | |
| run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT | |
| - name: Set Dynamic Matrix - SDK Loaders | |
| id: setmatrix_sdk_loaders | |
| run: | | |
| matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}" | |
| echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject" | |
| - name: Get Variants List - Airbyte | |
| id: get-variants-list-airbyte | |
| run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT | |
| - name: Set Dynamic Airbyte Matrix | |
| id: setmatrix_airbyte | |
| run: | | |
| matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}" | |
| echo "::set-output name=airbyte_matrix::$matrixStringifiedObject" | |
| metadata_extract_sdk_ex: | |
| name: SDK Extractors - Metadata Extract | |
| needs: get_variants_list | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} | |
| env: | |
| AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_matrix) }} | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Configure AWS credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| aws-region: us-west-2 | |
| role-session-name: "GitHubActions" | |
| - name: Install hub-utils | |
| run: pipx install git+https://github.com/meltano/hub-utils.git | |
| - name: Install Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Extract Metadata (3.10) | |
| id: extract_310 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.9 (Only if 3.10 failed) | |
| if: always() && (steps.extract_310.outcome == 'failure') | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Extract Metadata (3.9) | |
| if: always() && (steps.extract_310.outcome == 'failure') | |
| id: extract_39 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.8 (Only if 3.9 failed) | |
| if: always() && (steps.extract_39.outcome == 'failure') | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| - name: Extract Metadata (3.8) | |
| if: always() && (steps.extract_39.outcome == 'failure') | |
| id: extract_38 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| metadata_extract_sdk_loaders: | |
| name: SDK Loaders - Metadata Extract | |
| needs: get_variants_list | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} | |
| env: | |
| AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }} | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Configure AWS credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| aws-region: us-west-2 | |
| role-session-name: "GitHubActions" | |
| - name: Install hub-utils | |
| run: pipx install git+https://github.com/meltano/hub-utils.git | |
| - name: Install Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Extract Metadata (3.10) | |
| id: extract_310 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.9 (Only if 3.10 failed) | |
| if: always() && (steps.extract_310.outcome == 'failure') | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Extract Metadata (3.9) | |
| if: always() && (steps.extract_310.outcome == 'failure') | |
| id: extract_39 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.8 (Only if 3.9 failed) | |
| if: always() && (steps.extract_39.outcome == 'failure') | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| - name: Extract Metadata (3.8) | |
| if: always() && (steps.extract_39.outcome == 'failure') | |
| id: extract_38 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| metadata_extract_airbyte_p1: | |
| name: Airbyte - Part 1 Metadata Extract | |
| needs: get_variants_list | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} | |
| container: | |
| image: airbyte/${{ matrix.source-name }}:latest | |
| steps: | |
| - name: Get Airbyte Spec | |
| run: | | |
| $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json | |
| id: get-airbyte-stdout | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: extractor--${{ matrix.source-name }}--airbyte.json | |
| path: extractor--${{ matrix.source-name }}--airbyte.json | |
| metadata_extract_airbyte_p2: | |
| name: Airbyte - Part 2 Metadata Extract | |
| if: ${{ always() }} | |
| needs: [metadata_extract_airbyte_p1, get_variants_list] | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} | |
| env: | |
| AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Configure AWS credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| aws-region: us-west-2 | |
| role-session-name: "GitHubActions" | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: extractor--${{ matrix.source-name }}--airbyte.json | |
| - name: Install hub-utils | |
| run: pipx install git+https://github.com/meltano/hub-utils.git | |
| - name: Upload Metadata | |
| run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json |