Merge branch 'master' into bpf-iface-autodetect

.gitignore

@@ -19,6 +19,7 @@ node/windows-packaging/nssm.exe
 _output
 builder.coverprofile
 *.log
+*.pprof
 .release-*.*
 
 /* Created by local kind cluster */

.semaphore/release/hashrelease.yml

@@ -2,48 +2,71 @@ version: v1.0
 name: Publish hashrelease
 agent:
   machine:
-    type: f1-standard-2
+    type: f1-standard-4
     os_image: ubuntu2004
 execution_time_limit:
   hours: 4
 
 global_job_config:
   secrets:
+    - name: oss-release-secrets
     # Github SSH secret for pulling private repositories.
     - name: private-repo
     # Secret for GitHub API access.
     - name: marvin-github-token
     # Secret for pushing to the docs box.
     - name: docs-ssh
-    # Secret for pulling images from GCR.
-    - name: gcloud-registry-access
-    # Secret for the docker auth
-    - name: hashrelease-docker-auth
-    # Secret for Image Scanning Service
+    # Secret for image registries
+    - name: quay-hashrelease
+    - name: docker
     - name: iss-image-scanning
     # Secrets for Slack notifications
     - name: releasebot-slack
   prologue:
     commands:
       - chmod 0600 ~/.keys/*
       - ssh-add ~/.keys/*
-      - export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/keys/.registry-viewer-serviceaccount.json
-      - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
-      - docker login
+      # For some reason, /mnt is 100 GB and has a qemu-nbd image file.
+      # Let's delete it and use it for our own purposes (building calico
+      # without running out of space)
+      - sudo killall qemu-nbd || true
+      - sudo rm -f /mnt/docker.qcow2
+      - sudo chown $(id -u):$(id -g) /mnt/
+      - mkdir calico
+      - sudo mount --bind /mnt calico
+      # Checkout the code and unshallow it.
+      # (this is going to throw an error because it can't remove
+      # the `calico` directory, which is a mount, but it will
+      # continue anyway)
       - checkout
-      # Unshallow the git repository to get latest tags
       - retry git fetch --quiet --unshallow
+      # Semaphore mounts a copy-on-write FS as /var/lib/docker in order to provide a pre-loaded cache of
+      # some images. However, the cache is not useful to us and the copy-on-write FS is a big problem given
+      # how much we churn docker containers during the build.  Disable it.
+      - sudo systemctl stop docker
+      - sudo umount /var/lib/docker && sudo killall qemu-nbd || true
+      - sudo systemctl start docker
+      # Log in to container registries needed for release.
+      - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin
+      - echo $QUAY_TOKEN | docker login --username "$QUAY_USER" --password-stdin quay.io
+      # Credentials for accessing gcloud, needed to push images to gcr
+      - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/gcr-credentials.json
+      - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+      # Manually log in to GCR until we can test the gcr credentials helper
+      - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://gcr.io
+      - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://eu.gcr.io
+      - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://asia.gcr.io
+      - cat ${GOOGLE_APPLICATION_CREDENTIALS} | docker login -u _json_key --password-stdin https://us.gcr.io
 
 blocks:
   - name: Publish hashrelease
     task:
       jobs:
         - name: Build and publish hashrelease
           commands:
-            - if [[ ${SEMAPHORE_WORKFLOW_TRIGGERED_BY_SCHEDULE} == "true" ]]; then export BUILD_IMAGES=true; export SKIP_PUBLISH_IMAGES=false; fi
+            - if [[ ${SEMAPHORE_WORKFLOW_TRIGGERED_BY_SCHEDULE} == "true" ]]; then export BUILD_CONTAINER_IMAGES=true; export PUBLISH_IMAGES=true; fi
             - make hashrelease
       prologue:
         commands:
           - export GITHUB_TOKEN=${MARVIN_GITHUB_TOKEN}
           - cd release
-          - make build

.semaphore/semaphore.yml.d/blocks/20-felix.yml

@@ -88,44 +88,44 @@
         commands:
           - cd felix
           - make bin/calico-felix.exe fv/win-fv.exe
-# TODO: disable the Windows FV capz for the moment. Re-enable after they're fixed and passing.
-# yamllint disable-line rule:comments
-#- name: "Felix: Windows FV capz"
-#  run:
-#    when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"
-#  dependencies: ["Felix: Build Windows binaries"]
-#  task:
-#    secrets:
-#      - name: banzai-secrets
-#      - name: private-repo
-#    prologue:
-#      commands:
-#        - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none
-#        - export REPORT_DIR=/home/semaphore/report
-#        - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID
-#        - export AZURE_TENANT_ID=$AZ_TENANT_ID
-#        - export AZURE_CLIENT_ID=$AZ_SP_ID
-#        - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD
-#        - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
-#        - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')"
-#        - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')"
-#        - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')"
-#        - cd felix
-#    epilogue:
-#      always:
-#        commands:
-#          - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
-#    env_vars:
-#      - name: FV_PROVISIONER
-#        value: "capz"
-#      - name: FV_TYPE
-#        value: "calico-felix"
-#      - name: SEMAPHORE_ARTIFACT_EXPIRY
-#        value: 2w
-#    jobs:
-#      - name: CAPZ - Windows FV
-#        commands:
-#          - ./.semaphore/run-win-fv
+- name: "Felix: Windows FV capz"
+  run:
+    when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"
+  dependencies: ["Felix: Build Windows binaries"]
+  task:
+    secrets:
+      - name: banzai-secrets
+      - name: private-repo
+    prologue:
+      commands:
+        - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none
+        - export REPORT_DIR=/home/semaphore/report
+        - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID
+        - export AZURE_TENANT_ID=$AZ_TENANT_ID
+        - export AZURE_CLIENT_ID=$AZ_SP_ID
+        - export AZURE_CLIENT_SECRET=$AZ_SP_PASSWORD
+        - export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZ_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
+        - export AZURE_TENANT_ID_B64="$(echo -n "$AZ_TENANT_ID" | base64 | tr -d '\n')"
+        - export AZURE_CLIENT_ID_B64="$(echo -n "$AZ_SP_ID" | base64 | tr -d '\n')"
+        - export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZ_SP_PASSWORD" | base64 | tr -d '\n')"
+        - cd felix
+    epilogue:
+      always:
+        commands:
+          - artifact push job ${REPORT_DIR} --destination test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
+    env_vars:
+      - name: FV_PROVISIONER
+        value: "capz"
+      - name: FV_TYPE
+        value: "calico-felix"
+      - name: SEMAPHORE_ARTIFACT_EXPIRY
+        value: 2w
+      - name: CONTAINERD_VERSION
+        value: 1.7.22
+    jobs:
+      - name: CAPZ - Windows FV
+        commands:
+          - ./.semaphore/run-win-fv
 - name: "Felix: FV Tests"
   run:
     when: "${FORCE_RUN} or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/cni-plugin/pkg/dataplane/linux/dataplane_linux.go'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"