-
Notifications
You must be signed in to change notification settings - Fork 68
95 lines (83 loc) · 3.23 KB
/
metadata-extract-airbyte.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
name: Metadata Extract - Airbyte
on:
workflow_dispatch:
inputs:
environment:
description: 'The environment to run the workflow in'
required: true
type: choice
default: 'preview'
options:
- 'preview'
- 'production'
schedule:
- cron: '0 9 * * 1' # Run at midnight UTC on Mondays
jobs:
get_variants_list:
runs-on: ubuntu-latest
outputs:
airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }}
steps:
- uses: actions/[email protected]
- uses: astral-sh/setup-uv@v4
with:
version: ">=0.4.30"
- name: Install hub-utils
run: uv tool install git+https://github.com/meltano/hub-utils.git@main
- name: Get Variants List - Airbyte
id: get-variants-list-airbyte
run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT
- name: Set Dynamic Airbyte Matrix
id: setmatrix_airbyte
run: |
matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}"
echo "airbyte_matrix=$matrixStringifiedObject" >> $GITHUB_OUTPUT
metadata_extract_airbyte_p1:
name: Airbyte - Part 1 Metadata Extract
needs: get_variants_list
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
container:
image: airbyte/${{ matrix.source-name }}:latest
steps:
- name: Get Airbyte Spec
run: |
$AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json
id: get-airbyte-stdout
- uses: actions/upload-artifact@v3
with:
name: extractor--${{ matrix.source-name }}--airbyte.json
path: extractor--${{ matrix.source-name }}--airbyte.json
metadata_extract_airbyte_p2:
name: Airbyte - Part 2 Metadata Extract
if: ${{ always() }}
needs: [metadata_extract_airbyte_p1, get_variants_list]
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
env:
AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
permissions:
id-token: write # This is required for requesting the JWT
steps:
- uses: actions/[email protected]
- name: Configure AWS credentials
uses: aws-actions/[email protected]
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
aws-region: us-west-2
role-session-name: "GitHubActions"
- uses: actions/download-artifact@v3
with:
name: extractor--${{ matrix.source-name }}--airbyte.json
- uses: astral-sh/setup-uv@v4
with:
version: ">=0.4.30"
- name: Install hub-utils
run: uv tool install git+https://github.com/meltano/hub-utils.git@main
- name: Upload Metadata
run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json