-
Notifications
You must be signed in to change notification settings - Fork 169
218 lines (198 loc) · 10.1 KB
/
OSSCI.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
name: InfraCI - OSS cluster
on:
#Run on Manual execution
workflow_dispatch:
inputs:
environment:
description: 'Which GitHub Environment to deploy to'
required: true
default: "csu"
type: environment
ingressController:
description: "Which ingress controller to deploy"
type: choice
options:
- All
- Nginx
- Contour
- Traefik
required: true
default: Traefik
#Run when PR's are made to main, where the changes are in the bicep directory or this workflow file itself
#pull_request:
# branches: [main]
# paths:
# - "bicep/*"
# - ".github/workflows/OSSCI.yml"
#Run on a weekly schedule
schedule:
# At 9pm, every month.
- cron: "0 21 1 1-12 *"
env:
RG: "AksBicepAcc-Ci-OssCluster" #The resource group we're deploying to.
RESNAME: "AksOss" #Used in Azure Resource Naming, overrides the default in the parameter file
DEPNAME: "Dep${{ github.run_number }}" #Deployment Name
AZCLIVERSION: 2.53.0 #2.43.0 #2.34.1 #2.29.2 #2.26.0 #latest
permissions:
id-token: write
contents: read
concurrency: "OSSCI-${{ github.event.inputs.Environment != '' && github.event.inputs.Environment || 'csu' }}-AksBicepAcc-Ci-OssCluster"
jobs:
ReusableWF:
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.Environment }}
outputs:
RG: ${{ env.RG }}
ENVIRONMENT: ${{ github.event.inputs.Environment != '' && github.event.inputs.Environment || 'csu' }}
RESNAME: ${{ env.RESNAME }}
PARAMFILE: ${{ env.ParamFilePath }}
ExistingDnsDomainName: aksc.msftcsu.net
ExistingDnsDomainRg: aksbicepacc-ci-deployvnet
LatestAkscVersionTag: ${{ steps.AkscTags.outputs.LATEST}}
steps:
- name: Dummy step
run: echo "Resuable workflows can't be directly reference ENV/INPUTS (yet), so we need this job to proxy"
- name: Get latest AKSC version
id: AkscTags
run: |
LATEST=$(curl https://api.github.com/repos/Azure/AKS-Construction/releases/latest | jq '.tag_name' -r)
echo "LATEST=$LATEST" >> $GITHUB_OUTPUT
ContourDeploy:
uses: ./.github/workflows/AKSC_Deploy.yml
needs: [ReusableWF]
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.ingressController == 'Contour' || github.event.inputs.ingressController == 'All') || github.event_name != 'workflow_dispatch'
with:
environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
templateVersion: ${{ needs.ReusableWF.outputs.LatestAkscVersionTag }} #"0.9.3-preview3"
rg: ${{ needs.ReusableWF.outputs.RG }}
resourceName: azcontour
templateParams: resourceName=az-contour agentCount=2 JustUseSystemPool=true custom_vnet=true enable_aad=true enableAzureRBAC=true adminPrincipalId=_USER_OBJECT_ID_ registries_sku=Standard acrPushRolePrincipalId=_USER_OBJECT_ID_ networkPolicy=azure azurepolicy=audit dnsZoneId=_DNS_ZONE_ID_ keyVaultAksCSI=true keyVaultCreate=true keyVaultOfficerRolePrincipalId=_USER_OBJECT_ID_ enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }}
postScriptParams: "ingress=contour,ingressEveryNode=true,dnsZoneId=_DNS_ZONE_ID_,[email protected],certClusterIssuer=letsencrypt-staging,monitor=oss,enableMonitorIngress=true,grafanaHostname=grafanacnt"
secrets:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
USER_OBJECT_ID: ${{ secrets.USER_OBJECT_ID }}
DNS_ZONE_ID: ${{ secrets.BYODNSZONEID }}
ContourGrafanaVerify:
runs-on: ubuntu-latest
name: Verify Grafana Dashboard (Contour)
environment: ${{ github.event.inputs.environment }}
needs: [ContourDeploy]
env:
URL: "https://grafanacnt.aksc.msftcsu.net"
steps:
- name: Curl Grafana domain (on 80)
env:
HOSTNAME: grafanacnt.aksc.msftcsu.net
run: |
sleep 60s
curl $HOSTNAME
- name: Verify Grafana dashboard available TLS
timeout-minutes: 5
run: |
echo "curl $URL [$(date +"%T")]"
curlcommand="curl --connect-timeout 2 --retry 25 --retry-delay 20 --no-keepalive --no-tcp-nodelay -X GET --insecure --write-out %{http_code} --silent --fail --output /dev/null $URL -v --trace-time"
echo "Running curl command $curlcommand with retry"
respcode=$($curlcommand || sleep 1m; $curlcommand)
echo $respcode
curl --insecure $URL
- name: Verify Grafana Certificate
run: |
curl --insecure -vvI $APPURL 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }'
NginxDeploy:
uses: ./.github/workflows/AKSC_Deploy.yml
needs: [ReusableWF]
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.ingressController == 'Nginx' || github.event.inputs.ingressController == 'All') || github.event_name != 'workflow_dispatch'
with:
environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
templateVersion: ${{ needs.ReusableWF.outputs.LatestAkscVersionTag }} #"0.9.3-preview3"
rg: ${{ needs.ReusableWF.outputs.RG }}
resourceName: azcontour
templateParams: resourceName=az-nginx agentCount=2 JustUseSystemPool=true custom_vnet=true enable_aad=true enableAzureRBAC=true adminPrincipalId=_USER_OBJECT_ID_ registries_sku=Standard acrPushRolePrincipalId=_USER_OBJECT_ID_ networkPolicy=azure azurepolicy=audit dnsZoneId=_DNS_ZONE_ID_ keyVaultAksCSI=true keyVaultCreate=true keyVaultOfficerRolePrincipalId=_USER_OBJECT_ID_ enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }}
postScriptParams: "ingress=nginx,ingressEveryNode=true,dnsZoneId=_DNS_ZONE_ID_,[email protected],monitor=oss,enableMonitorIngress=true,grafanaHostname=grafanangx"
secrets:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
USER_OBJECT_ID: ${{ secrets.USER_OBJECT_ID }}
DNS_ZONE_ID: ${{ secrets.BYODNSZONEID }}
NginxGrafanaVerify:
runs-on: ubuntu-latest
name: Verify Grafana Dashboard (Nginx)
environment: ${{ github.event.inputs.environment }}
needs: [NginxDeploy]
env:
URL: "https://grafanangx.aksc.msftcsu.net"
steps:
- name: Curl Grafana domain (on 80)
env:
HOSTNAME: grafanangx.aksc.msftcsu.net
run: |
sleep 60s
curl $HOSTNAME
- name: Verify Grafana dashboard available TLS
timeout-minutes: 5
run: |
echo "curl $URL [$(date +"%T")]"
curlcommand="curl --connect-timeout 2 --retry 25 --retry-delay 20 --no-keepalive --no-tcp-nodelay -X GET --insecure --write-out %{http_code} --silent --fail --output /dev/null $URL -v --trace-time"
echo "Running curl command $curlcommand with retry"
respcode=$($curlcommand || sleep 1m; $curlcommand)
echo $respcode
curl --insecure $URL
- name: Verify Grafana Certificate
run: |
curl --insecure -vvI $APPURL 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }'
TraefikDeploy:
uses: ./.github/workflows/AKSC_Deploy.yml
needs: [ReusableWF]
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.ingressController == 'Traefik' || github.event.inputs.ingressController == 'All') || github.event_name != 'workflow_dispatch'
with:
environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
templateVersion: ${{ needs.ReusableWF.outputs.LatestAkscVersionTag }} #"0.9.3-preview3"
rg: ${{ needs.ReusableWF.outputs.RG }}
resourceName: azcontour
templateParams: resourceName=az-traefik agentCount=2 JustUseSystemPool=true custom_vnet=true enable_aad=true enableAzureRBAC=true adminPrincipalId=_USER_OBJECT_ID_ registries_sku=Standard acrPushRolePrincipalId=_USER_OBJECT_ID_ networkPolicy=azure azurepolicy=audit dnsZoneId=_DNS_ZONE_ID_ keyVaultAksCSI=true keyVaultCreate=true keyVaultOfficerRolePrincipalId=_USER_OBJECT_ID_ enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }}
postScriptParams: "ingress=traefik,ingressEveryNode=true,dnsZoneId=_DNS_ZONE_ID_,[email protected],certClusterIssuer=letsencrypt-staging,monitor=oss,enableMonitorIngress=true,grafanaHostname=grafanatrae"
secrets:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
USER_OBJECT_ID: ${{ secrets.USER_OBJECT_ID }}
DNS_ZONE_ID: ${{ secrets.BYODNSZONEID }}
TraefikGrafanaVerify:
runs-on: ubuntu-latest
name: Verify Grafana Dashboard (Traefik)
environment: ${{ github.event.inputs.environment }}
needs: [TraefikDeploy]
env:
URL: "https://grafanatrae.aksc.msftcsu.net"
steps:
- name: Curl Grafana domain (on 80)
env:
HOSTNAME: grafanatrae.aksc.msftcsu.net
run: |
sleep 60s
curl $HOSTNAME
- name: Verify Grafana dashboard available TLS
timeout-minutes: 5
run: |
echo "curl $URL [$(date +"%T")]"
curlcommand="curl --connect-timeout 2 --retry 25 --retry-delay 20 --no-keepalive --no-tcp-nodelay -X GET --insecure --write-out %{http_code} --silent --fail --output /dev/null $URL -v --trace-time"
echo "Running curl command $curlcommand with retry"
respcode=$($curlcommand || sleep 1m; $curlcommand)
echo $respcode
curl --insecure $URL
- name: Verify Grafana Certificate
run: |
curl --insecure -vvI $APPURL 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }'
Cleanup:
uses: ./.github/workflows/cleanupRg.yml
needs: [ContourGrafanaVerify, NginxGrafanaVerify, TraefikGrafanaVerify]
with:
environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
rg: ${{ needs.ReusableWF.outputs.RG }}
secrets:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}