InfraCI - OSS cluster #120
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: InfraCI - OSS cluster | |
| on: | |
| #Run on Manual execution | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Which GitHub Environment to deploy to' | |
| required: true | |
| default: "csu" | |
| type: environment | |
| ingressController: | |
| description: "Which ingress controller to deploy" | |
| type: choice | |
| options: | |
| - All | |
| - Nginx | |
| - Contour | |
| - Traefik | |
| required: true | |
| default: Traefik | |
| #Run when PR's are made to main, where the changes are in the bicep directory or this workflow file itself | |
| #pull_request: | |
| # branches: [main] | |
| # paths: | |
| # - "bicep/*" | |
| # - ".github/workflows/OSSCI.yml" | |
| #Run on a weekly schedule | |
| schedule: | |
| # At 9pm, every month. | |
| - cron: "0 21 1 1-12 *" | |
| env: | |
| RG: "AksBicepAcc-Ci-OssCluster" #The resource group we're deploying to. | |
| RESNAME: "AksOss" #Used in Azure Resource Naming, overrides the default in the parameter file | |
| DEPNAME: "Dep${{ github.run_number }}" #Deployment Name | |
| AZCLIVERSION: 2.53.0 #2.43.0 #2.34.1 #2.29.2 #2.26.0 #latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| concurrency: "OSSCI-${{ github.event.inputs.Environment != '' && github.event.inputs.Environment || 'csu' }}-AksBicepAcc-Ci-OssCluster" | |
| jobs: | |
| ReusableWF: | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.Environment }} | |
| outputs: | |
| RG: ${{ env.RG }} | |
| ENVIRONMENT: ${{ github.event.inputs.Environment != '' && github.event.inputs.Environment || 'csu' }} | |
| RESNAME: ${{ env.RESNAME }} | |
| PARAMFILE: ${{ env.ParamFilePath }} | |
| ExistingDnsDomainName: aksc.msftcsu.net | |
| ExistingDnsDomainRg: aksbicepacc-ci-deployvnet | |
| LatestAkscVersionTag: ${{ steps.AkscTags.outputs.LATEST}} | |
| steps: | |
| - name: Dummy step | |
| run: echo "Resuable workflows can't be directly reference ENV/INPUTS (yet), so we need this job to proxy" | |
| - name: Get latest AKSC version | |
| id: AkscTags | |
| run: | | |
| LATEST=$(curl https://api.github.com/repos/Azure/AKS-Construction/releases/latest | jq '.tag_name' -r) | |
| echo "LATEST=$LATEST" >> $GITHUB_OUTPUT | |
| ContourDeploy: | |
| uses: ./.github/workflows/AKSC_Deploy.yml | |
| needs: [ReusableWF] | |
| if: github.event_name == 'workflow_dispatch' && (github.event.inputs.ingressController == 'Contour' || github.event.inputs.ingressController == 'All') || github.event_name != 'workflow_dispatch' | |
| with: | |
| environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| templateVersion: ${{ needs.ReusableWF.outputs.LatestAkscVersionTag }} #"0.9.3-preview3" | |
| rg: ${{ needs.ReusableWF.outputs.RG }} | |
| resourceName: azcontour | |
| templateParams: resourceName=az-contour agentCount=2 JustUseSystemPool=true custom_vnet=true enable_aad=true enableAzureRBAC=true adminPrincipalId=_USER_OBJECT_ID_ registries_sku=Standard acrPushRolePrincipalId=_USER_OBJECT_ID_ networkPolicy=azure azurepolicy=audit dnsZoneId=_DNS_ZONE_ID_ keyVaultAksCSI=true keyVaultCreate=true keyVaultOfficerRolePrincipalId=_USER_OBJECT_ID_ enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }} | |
| postScriptParams: "ingress=contour,ingressEveryNode=true,dnsZoneId=_DNS_ZONE_ID_,[email protected],certClusterIssuer=letsencrypt-staging,monitor=oss,enableMonitorIngress=true,grafanaHostname=grafanacnt" | |
| secrets: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| USER_OBJECT_ID: ${{ secrets.USER_OBJECT_ID }} | |
| DNS_ZONE_ID: ${{ secrets.BYODNSZONEID }} | |
| ContourGrafanaVerify: | |
| runs-on: ubuntu-latest | |
| name: Verify Grafana Dashboard (Contour) | |
| environment: ${{ github.event.inputs.environment }} | |
| needs: [ContourDeploy] | |
| env: | |
| URL: "https://grafanacnt.aksc.msftcsu.net" | |
| steps: | |
| - name: Curl Grafana domain (on 80) | |
| env: | |
| HOSTNAME: grafanacnt.aksc.msftcsu.net | |
| run: | | |
| sleep 60s | |
| curl $HOSTNAME | |
| - name: Verify Grafana dashboard available TLS | |
| timeout-minutes: 5 | |
| run: | | |
| echo "curl $URL [$(date +"%T")]" | |
| curlcommand="curl --connect-timeout 2 --retry 25 --retry-delay 20 --no-keepalive --no-tcp-nodelay -X GET --insecure --write-out %{http_code} --silent --fail --output /dev/null $URL -v --trace-time" | |
| echo "Running curl command $curlcommand with retry" | |
| respcode=$($curlcommand || sleep 1m; $curlcommand) | |
| echo $respcode | |
| curl --insecure $URL | |
| - name: Verify Grafana Certificate | |
| run: | | |
| curl --insecure -vvI $APPURL 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }' | |
| NginxDeploy: | |
| uses: ./.github/workflows/AKSC_Deploy.yml | |
| needs: [ReusableWF] | |
| if: github.event_name == 'workflow_dispatch' && (github.event.inputs.ingressController == 'Nginx' || github.event.inputs.ingressController == 'All') || github.event_name != 'workflow_dispatch' | |
| with: | |
| environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| templateVersion: ${{ needs.ReusableWF.outputs.LatestAkscVersionTag }} #"0.9.3-preview3" | |
| rg: ${{ needs.ReusableWF.outputs.RG }} | |
| resourceName: azcontour | |
| templateParams: resourceName=az-nginx agentCount=2 JustUseSystemPool=true custom_vnet=true enable_aad=true enableAzureRBAC=true adminPrincipalId=_USER_OBJECT_ID_ registries_sku=Standard acrPushRolePrincipalId=_USER_OBJECT_ID_ networkPolicy=azure azurepolicy=audit dnsZoneId=_DNS_ZONE_ID_ keyVaultAksCSI=true keyVaultCreate=true keyVaultOfficerRolePrincipalId=_USER_OBJECT_ID_ enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }} | |
| postScriptParams: "ingress=nginx,ingressEveryNode=true,dnsZoneId=_DNS_ZONE_ID_,[email protected],monitor=oss,enableMonitorIngress=true,grafanaHostname=grafanangx" | |
| secrets: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| USER_OBJECT_ID: ${{ secrets.USER_OBJECT_ID }} | |
| DNS_ZONE_ID: ${{ secrets.BYODNSZONEID }} | |
| NginxGrafanaVerify: | |
| runs-on: ubuntu-latest | |
| name: Verify Grafana Dashboard (Nginx) | |
| environment: ${{ github.event.inputs.environment }} | |
| needs: [NginxDeploy] | |
| env: | |
| URL: "https://grafanangx.aksc.msftcsu.net" | |
| steps: | |
| - name: Curl Grafana domain (on 80) | |
| env: | |
| HOSTNAME: grafanangx.aksc.msftcsu.net | |
| run: | | |
| sleep 60s | |
| curl $HOSTNAME | |
| - name: Verify Grafana dashboard available TLS | |
| timeout-minutes: 5 | |
| run: | | |
| echo "curl $URL [$(date +"%T")]" | |
| curlcommand="curl --connect-timeout 2 --retry 25 --retry-delay 20 --no-keepalive --no-tcp-nodelay -X GET --insecure --write-out %{http_code} --silent --fail --output /dev/null $URL -v --trace-time" | |
| echo "Running curl command $curlcommand with retry" | |
| respcode=$($curlcommand || sleep 1m; $curlcommand) | |
| echo $respcode | |
| curl --insecure $URL | |
| - name: Verify Grafana Certificate | |
| run: | | |
| curl --insecure -vvI $APPURL 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }' | |
| TraefikDeploy: | |
| uses: ./.github/workflows/AKSC_Deploy.yml | |
| needs: [ReusableWF] | |
| if: github.event_name == 'workflow_dispatch' && (github.event.inputs.ingressController == 'Traefik' || github.event.inputs.ingressController == 'All') || github.event_name != 'workflow_dispatch' | |
| with: | |
| environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| templateVersion: ${{ needs.ReusableWF.outputs.LatestAkscVersionTag }} #"0.9.3-preview3" | |
| rg: ${{ needs.ReusableWF.outputs.RG }} | |
| resourceName: azcontour | |
| templateParams: resourceName=az-traefik agentCount=2 JustUseSystemPool=true custom_vnet=true enable_aad=true enableAzureRBAC=true adminPrincipalId=_USER_OBJECT_ID_ registries_sku=Standard acrPushRolePrincipalId=_USER_OBJECT_ID_ networkPolicy=azure azurepolicy=audit dnsZoneId=_DNS_ZONE_ID_ keyVaultAksCSI=true keyVaultCreate=true keyVaultOfficerRolePrincipalId=_USER_OBJECT_ID_ enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }} | |
| postScriptParams: "ingress=traefik,ingressEveryNode=true,dnsZoneId=_DNS_ZONE_ID_,[email protected],certClusterIssuer=letsencrypt-staging,monitor=oss,enableMonitorIngress=true,grafanaHostname=grafanatrae" | |
| secrets: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| USER_OBJECT_ID: ${{ secrets.USER_OBJECT_ID }} | |
| DNS_ZONE_ID: ${{ secrets.BYODNSZONEID }} | |
| TraefikGrafanaVerify: | |
| runs-on: ubuntu-latest | |
| name: Verify Grafana Dashboard (Traefik) | |
| environment: ${{ github.event.inputs.environment }} | |
| needs: [TraefikDeploy] | |
| env: | |
| URL: "https://grafanatrae.aksc.msftcsu.net" | |
| steps: | |
| - name: Curl Grafana domain (on 80) | |
| env: | |
| HOSTNAME: grafanatrae.aksc.msftcsu.net | |
| run: | | |
| sleep 60s | |
| curl $HOSTNAME | |
| - name: Verify Grafana dashboard available TLS | |
| timeout-minutes: 5 | |
| run: | | |
| echo "curl $URL [$(date +"%T")]" | |
| curlcommand="curl --connect-timeout 2 --retry 25 --retry-delay 20 --no-keepalive --no-tcp-nodelay -X GET --insecure --write-out %{http_code} --silent --fail --output /dev/null $URL -v --trace-time" | |
| echo "Running curl command $curlcommand with retry" | |
| respcode=$($curlcommand || sleep 1m; $curlcommand) | |
| echo $respcode | |
| curl --insecure $URL | |
| - name: Verify Grafana Certificate | |
| run: | | |
| curl --insecure -vvI $APPURL 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }' | |
| Cleanup: | |
| uses: ./.github/workflows/cleanupRg.yml | |
| needs: [ContourGrafanaVerify, NginxGrafanaVerify, TraefikGrafanaVerify] | |
| with: | |
| environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| rg: ${{ needs.ReusableWF.outputs.RG }} | |
| secrets: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} |