canonical · swetha1654 · Dec 11, 2024 · Dec 2, 2024 · Dec 2, 2024 · Dec 2, 2024
@@ -0,0 +1,105 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: Allure Report Generation
+
+on: [workflow_call]
+
+jobs:
+  allure-report:
+    name: Publish Allure report
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    if: always() && !cancelled()
+    steps:
+      - name: Download Allure
+        # Following instructions from https://allurereport.org/docs/gettingstarted-installation/#install-via-the-system-package-manager-for-linux
+        run: gh release download --repo allure-framework/allure2 --pattern 'allure_*.deb'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Allure
+        run: |
+          sudo apt-get update
+          sudo apt-get install ./allure_*.deb -y
+      # For first run, manually create branch with no history
+      # (e.g.
+      # git checkout --orphan gh-pages
+      # git rm -rf .
+      # touch .nojekyll
+      # git add .nojekyll
+      # git commit -m "Initial commit"
+      # git push origin gh-pages
+      # )
+      - name: Checkout GitHub pages branch
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          path: repo/
+      - name: Download default test results
+        uses: actions/download-artifact@v4
+        with:
+          path: allure-collection-default-results/
+          pattern: allure-default-results*
+          merge-multiple: true
+      - name: Download actual test results
+        uses: actions/download-artifact@v4
+        with:
+          path: allure-results/
+          pattern: allure-results*
+          merge-multiple: true
+      - name: Install CLI
+        run: pipx install git+https://github.com/canonical/operator-workflows@ISD-2620-allure#subdirectory=python/cli
+      - name: Combine Allure default results & actual results
+        # For every test: if actual result available, use that. Otherwise, use default result
+        # So that, if actual result not available, Allure report will show "unknown"/"failed" test result
+        # instead of omitting the test
+        run: allure-add-default-for-missing-results --allure-results-dir=allure-results --allure-collection-default-results-dir=allure-collection-default-results
+      - name: Load test report history
+        run: |
+          if [[ -d repo/_latest/history/ ]]
+          then
+            echo 'Loading history'
+            cp -r repo/_latest/history/ allure-results/
+          fi
+      - name: Create executor.json
+        shell: python
+        run: |
+          # Reverse engineered from https://github.com/simple-elf/allure-report-action/blob/eca283b643d577c69b8e4f048dd6cd8eb8457cfd/entrypoint.sh
+          import json
+
+          DATA = {
+              "name": "GitHub Actions",
+              "type": "github",
+              "buildOrder": ${{ github.run_number }},  # TODO future improvement: use run ID
+              "buildName": "Run ${{ github.run_id }}",
+              "buildUrl": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+              "reportUrl": "../${{ github.run_number }}/",
+          }
+          with open("allure-results/executor.json", "w") as file:
+              json.dump(DATA, file)
+      - name: Generate Allure report
+        run: allure generate
+      - name: Create index.html
+        shell: python
+        run: |
+          DATA = f"""<!DOCTYPE html>
+          <meta charset="utf-8">
+          <meta http-equiv="cache-control" content="no-cache">
+          <meta http-equiv="refresh" content="0; url=${{ github.run_number }}">
+          """
+          with open("repo/index.html", "w") as file:
+              file.write(DATA)
+      - name: Update GitHub pages branch
+        working-directory: repo/
+        # TODO future improvement: commit message
+        run: |
+          mkdir '${{ github.run_number }}'
+          rm -f _latest
+          ln -s '${{ github.run_number }}' _latest
+          cp -r ../allure-report/. _latest/
+          git add .
+          git config user.name "GitHub Actions"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -m "Allure report ${{ github.run_number }}"
+          # Uses token set in checkout step
+          git push origin gh-pages
@@ -171,6 +171,55 @@ jobs:
         )) || inputs.runs-on
       }}
     steps:
+      - name: Plan Integration
+        uses: canonical/operator-workflows/internal/plan-integration@main
+        id: plan-integration
+        with:
+          plan: ${{ inputs.plan }}
+      - name: Integration tests variable setting
+        working-directory: ${{ inputs.working-directory }}/${{ inputs.charm-directory }}
+        run: |
+          CHARM_NAME="$([ -f metadata.yaml ] && yq '.name' metadata.yaml || echo UNKNOWN)"
+          if [ "$CHARM_NAME" == "UNKNOWN" ]; then
+            CHARM_NAME="$([ -f charmcraft.yaml ] && yq '.name' charmcraft.yaml || echo UNKNOWN)"
+          fi
+          echo "CHARM_NAME=$CHARM_NAME" >> $GITHUB_ENV
+
+          args="${{ steps.plan-integration.outputs.args }}"
+          echo "ARGS=$args" >> $GITHUB_ENV
+
+          allure_artifact_suffix=$(uuidgen)
+          series=""
+          if [ ! -z ${{ matrix.series }} ]; then
+            series="--series ${{ matrix.series }}"
+            allure_artifact_suffix=$(allure_artifact_suffix)-${{ matrix.series }}
+          fi
+          echo "SERIES=$series" >> $GITHUB_ENV
+          module=""
+          if [ ! -z ${{ matrix.modules }} ]; then
+            module="-k ${{ matrix.modules }}"
+            allure_artifact_suffix=$(allure_artifact_suffix)-${{ matrix.modules }}
+          fi
+          echo "MODULE=$module" >> $GITHUB_ENV
+          echo "ALLURE_ARTIFACT_SUFFIX=$allure_artifact_suffix" >> $GITHUB_ENV
+      - name: Install pytest plugin for Allure
+        run: pip install git+https://github.com/canonical/operator-workflows@ISD-2620-allure#subdirectory=python/pytest_plugins/allure_pytest_collection_report
+      - name: Collect tests for Allure (k8s)
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.provider == 'microk8s' }}
+        run: |
+          tox -e ${{ inputs.test-tox-env }} -- --model testing --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} --allure-collection-dir=allure-default ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} 
+      - name: Collect tests for Allure (lxd)
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.provider == 'lxd' }}
+        run: |
+          tox -e ${{ inputs.test-tox-env }} -- --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} --allure-collection-dir=allure-default ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }}
+      - name: Upload Default Allure results
+        timeout-minutes: 3
+        uses: actions/upload-artifact@v4
+        with:
+          name: allure-default-results-${{ env.ALLURE_ARTIFACT_SUFFIX }}
+          path: allure-default/
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
         with:
@@ -267,43 +316,23 @@ jobs:
 
       - run: sudo apt install skopeo -y
 
-      - name: Plan Integration
-        uses: canonical/operator-workflows/internal/plan-integration@main
-        id: plan-integration
-        with:
-          plan: ${{ inputs.plan }}
-
-      - name: Integration tests variable setting
-        working-directory: ${{ inputs.working-directory }}/${{ inputs.charm-directory }}
-        run: |
-          CHARM_NAME="$([ -f metadata.yaml ] && yq '.name' metadata.yaml || echo UNKNOWN)"
-          if [ "$CHARM_NAME" == "UNKNOWN" ]; then
-            CHARM_NAME="$([ -f charmcraft.yaml ] && yq '.name' charmcraft.yaml || echo UNKNOWN)"
-          fi
-          echo "CHARM_NAME=$CHARM_NAME" >> $GITHUB_ENV
-
-          args="${{ steps.plan-integration.outputs.args }}"
-          echo "ARGS=$args" >> $GITHUB_ENV
-          series=""
-          if [ ! -z ${{ matrix.series }} ]; then
-            series="--series ${{ matrix.series }}"
-          fi
-          echo "SERIES=$series" >> $GITHUB_ENV
-          module=""
-          if [ ! -z ${{ matrix.modules }} ]; then
-            module="-k ${{ matrix.modules }}"
-          fi
-          echo "MODULE=$module" >> $GITHUB_ENV
       - name: Run k8s integration tests
         working-directory: ${{ inputs.working-directory }}
         if: ${{ inputs.provider == 'microk8s' }}
         run: |
-          tox -e ${{ inputs.test-tox-env }} -- --model testing --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }}
+          tox -e ${{ inputs.test-tox-env }} -- --model testing --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} --alluredir=allure-results ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }} 
       - name: Run lxd integration tests
         working-directory: ${{ inputs.working-directory }}
         if: ${{ inputs.provider == 'lxd' }}
         run: |
-          tox -e ${{ inputs.test-tox-env }} -- --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }}
+          tox -e ${{ inputs.test-tox-env }} -- --keep-models ${{ env.SERIES }} ${{ env.MODULE }} ${{ env.ARGS }} --alluredir=allure-results ${{ inputs.extra-arguments }} ${{ secrets.INTEGRATION_TEST_ARGS }}
+      - name: Upload Allure results
+        timeout-minutes: 3
+        if: always() && !cancelled()
+        uses: actions/upload-artifact@v4
+        with:
+          name: allure-results-${{ env.ALLURE_ARTIFACT_SUFFIX }}
+          path: ${{ inputs.working-directory }}allure-results/
       - name: Tmate debugging session (self-hosted)
         if: ${{ failure() && (inputs.tmate-debug || runner.debug) && inputs.self-hosted-runner }}
         uses: canonical/action-tmate@main

@@ -132,3 +132,14 @@ jobs:
           [ '${{ needs.integration-craft.result }}' = 'success' ] || (echo integration-craft failed && false)
           [ '${{ needs.publish.result }}' != 'failure' ] || (echo publish failed && false)
           [ '${{ needs.publish-artifact.result }}' != 'failure' ] || (echo publish failed && false)
+  allure-report:
+    if: always() && !cancelled()
+    needs:
+      - integration
+      - integration-juju3
+      - integration-artifact
+      - integration-self-hosted
+      - integration-rock
+      - integration-rock-artifact
+      - integration-craft
+    uses: ./.github/workflows/allure_report.yaml
diff --git a/python/cli/README.md b/python/cli/README.md
@@ -0,0 +1,3 @@
+The `allure_add_default_for_missing_results` cli combines the Allure default results & actual results before generating the report. 
+
+Currently, if the result of a test is not available (for reasons such as setup failure), allure omits the test. This CLI ensures that for every test, if actual result is available, it will use that. Otherwise, it uses the default result `unknown`.
diff --git a/python/cli/allure_add_default_for_missing_results/__init__.py b/python/cli/allure_add_default_for_missing_results/__init__.py
diff --git a/python/cli/allure_add_default_for_missing_results/allure_add_default_for_missing_results.py b/python/cli/allure_add_default_for_missing_results/allure_add_default_for_missing_results.py
@@ -0,0 +1,49 @@
+import argparse
+import dataclasses
+import json
+import pathlib
+
+
+@dataclasses.dataclass(frozen=True)
+class Result:
+    test_case_id: str
+    path: pathlib.Path
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            return False
+        return self.test_case_id == other.test_case_id
+
+
+def main():
+    """Combine Allure default results & actual results
+
+    For every test: if actual result available, use that. Otherwise, use default result
+
+    So that, if actual result not available, Allure report will show "unknown"/"failed" test result
+    instead of omitting the test
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--allure-results-dir", required=True)
+    parser.add_argument("--allure-collection-default-results-dir", required=True)
+    args = parser.parse_args()
+
+    actual_results = pathlib.Path(args.allure_results_dir)
+    default_results = pathlib.Path(args.allure_collection_default_results_dir)
+
+    results: dict[pathlib.Path, set[Result]] = {
+        actual_results: set(),
+        default_results: set(),
+    }
+    for directory, results_ in results.items():
+        for path in directory.glob("*-result.json"):
+            with path.open("r") as file:
+                id_ = json.load(file)["testCaseId"]
+            results_.add(Result(id_, path))
+
+    actual_results.mkdir(exist_ok=True)
+
+    missing_results = results[default_results] - results[actual_results]
+    for default_result in missing_results:
+        # Move to `actual_results` directory
+        default_result.path.rename(actual_results / default_result.path.name)
diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml
@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "workflows-cli"
+# Version unused; repository has its own versioning system. (See .github/workflows/__release.yaml)
+version = "0.1.0"
+description = ""
+license = "Apache-2.0"
+authors = ["Carl Csaposs <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.scripts]
+allure-add-default-for-missing-results = "workflows_cli.allure_add_default_for_missing_results:main"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+pyyaml = "^6.0.1"
+requests = "^2.31.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/python/pytest_plugins/allure_pytest_collection_report/README.md b/python/pytest_plugins/allure_pytest_collection_report/README.md
@@ -0,0 +1 @@
+The `allure-pytest` plugin marks all the test statuses as unknown during collection time.
diff --git a/python/pytest_plugins/allure_pytest_collection_report/__init__.py b/python/pytest_plugins/allure_pytest_collection_report/__init__.py
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		The `allure_add_default_for_missing_results` cli combines the Allure default results & actual results before generating the report.

		Currently, if the result of a test is not available (for reasons such as setup failure), allure omits the test. This CLI ensures that for every test, if actual result is available, it will use that. Otherwise, it uses the default result `unknown`.
swetha1654 marked this conversation as resolved. Show resolved Hide resolved
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		The `allure-pytest` plugin marks all the test statuses as unknown during collection time.