Fuzzbucket in CI

.github/workflows/pull-request.yaml

@@ -313,58 +313,7 @@ jobs:
         with:
           name: agent-artifacts
           path: bin/
-
-  ux-e2e-tests:
-    needs: build-agent
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Setup Just
-        uses: extractions/setup-just@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Restore Cache docker image
-        id: cache-docker-image
-        uses: actions/cache/restore@v3
-        env:
-          cache-name: cache-docker-image
-        with:
-          path: /tmp/build-image.tar
-          key: ${{ runner.os }}-docker-image-${{ env.cache-name }}-${{ hashFiles('/tmp/build-image.tar') }}
-      -
-        name: Load Docker Image
-        run: |
-          docker load --input /tmp/build-image.tar
-      -
-        name: Download built agents
-        uses: actions/download-artifact@v3
-        with:
-          name: agent-artifacts
-          path: ./bin
-      -
-        name: Make agents executable
-        run: |
-          chmod -R +x ./bin/**/connect-client
-      -
-        name: Restore Cache UX node_modules
-        id: node_modules
-        uses: actions/cache/restore@v3
-        env:
-          cache-name: node-modules
-        with:
-          path: ./web/node_modules
-          key: ${{ runner.os }}-node-modules-${{ env.cache-name }}-${{ hashFiles('./web/package.json') }}
-      -
-        name: UX E2E Tests
-        run: |
-          PATH="${PATH}:/usr/local/bin" just web/test-e2e
-
+
   test-windows:
     needs: build-agent
     runs-on: windows-latest
@@ -394,29 +343,35 @@ jobs:
           # bats reads the load libraries from relative path only in Windows
           BATS_SUPPORT_LIB: ..\..\libs\bats-support\load.bash
           BATS_ASSERT_LIB: ..\..\libs\bats-assert\load
-        # WARNING: Docker run will only fail step if last command fails...
         run: |
           just test/bats-install
           $env:Path += ";$env:GITHUB_WORKSPACE\test\libs\bats-core\bin"
           $env:BINARY_PATH = "$env:GITHUB_WORKSPACE\bin\windows-amd64\connect-client"
           cd $env:GITHUB_WORKSPACE\test
-          just run-client $env:os
+          just test-cli windows-amd64
       -
         name: Run Windows Cypress Tests
         env:
           os: windows-latest
           DOCKER: "false"
+          FUZZBUCKET_SSH_KEY: ${{ secrets.FUZZBUCKET_SSH_KEY }}
+          FUZZBUCKET_URL: ${{ secrets.FUZZBUCKET_URL }}
+          FUZZBUCKET_CREDENTIALS: ${{ secrets.FUZZBUCKET_CREDENTIALS }}
         run: |
-          just web/build-and-test-ci-e2e $env:os
-
-
+          $env:CONNECT_API_KEY="$(python ./test/setup/gen_apikey.py 'admin')"
+          just test/ui-tests windows-amd64
   test-macos:
     needs: build-agent
     runs-on: macos-latest
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
+      -
+        name: python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11        
       -
         name: Setup Just
         uses: extractions/setup-just@v1
@@ -444,21 +399,22 @@ jobs:
           just test/bats-install
           PATH="$PATH:${{ github.workspace }}/test/libs/bats-core/bin"
           cd ${{ github.workspace }}/test
-          just run-client ${os}
+          just test-cli darwin-amd64
       -
         name: Run MacOS Cypress Tests
         env:
           os: macos-latest
           DOCKER: "false"
         run: |
-          just web/build-and-test-ci-e2e ${os}
+          export CONNECT_API_KEY="$(python ./test/setup/gen_apikey.py 'admin')"
+          just test/ui-tests darwin-amd64
 
   build-linux-amd64:
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
-    needs: build-agent
+    # needs: build-agent
     steps:
       -
         name: Checkout
@@ -513,7 +469,7 @@ jobs:
           key:  ${{ env.cache-name }}-${{ hashFiles('test/docker/Dockerfile') }}
 
   test-linux-amd64:
-    needs: build-linux-amd64
+    needs:  [build-agent, build-linux-amd64]
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -555,21 +511,21 @@ jobs:
         env:
           os: linux-amd64
         run: |
-          just test/run-client ${os}
+          just test/docker_test-cli ${os}
       -
         name: Run linux-amd64 Cypress Tests
         env:
           os: linux-amd64
           DOCKER: "false"
         run: |
-          just test/ui-client ${os}
+          export CONNECT_API_KEY="$(python ./test/setup/gen_apikey.py 'admin')"
+          just test/ui-tests ${os}
 
   build-linux-arm64:
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
-    needs: build-agent
     steps:
       -
         name: Checkout
@@ -624,7 +580,7 @@ jobs:
           key:  ${{ env.cache-name }}-${{ hashFiles('test/docker/Dockerfile') }}
 
   test-linux-arm64:
-    needs: build-linux-arm64
+    needs:  [build-agent, build-linux-arm64]
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -676,12 +632,11 @@ jobs:
         env:
           os: linux-arm64
         run: |
-          just test/run-client ${os}
+          just test/docker_test-cli ${os}
       # -
       #   name: Run linux-arm64 Cypress Tests
       #   env:
       #     os: linux-arm64
       #     DOCKER: "false"
       #   run: |
-      #     just web/build && \
-      #     just test/ui-client ${os}
+      #     just test/docker_publishing-tests ${os}

justfile

@@ -183,9 +183,7 @@ start-agent-for-e2e:
     echo "Working directory is $(pwd)"
 
     ./bin/$GOOS-$GOARCH/connect-client publish-ui \
-        ./test/sample-content/fastapi-simple \
-        --listen=127.0.0.1:9000 \
-        --token=abc123
+        ${CMD_ARGS}
 
 [private]
 _with_docker *args:

test/README.md

@@ -1,37 +1,80 @@
 # connect-client tests
 
-These tests will use the `connect-client` binary in the following environments:
+These tests will run the `connect-client` binary in the following environments:
 * windows-amd64 (Github Actions only)
 * macos-amd64 (Github Actions only)
 * linux-amd64 (Local Docker & Github Actions)
 * linux-arm64 (Local Docker & Github Actions)
 
-You can also run the tests against your local environment. 
 
-First, build the docker containers needed:
+
+## Binaries
+First you'll need the `connect-client` binary to test with. If testing on your local OS, outside of Docker, in the root directory run:
+```
+just build
+```
+
+If you want to run tests against a linux binary in Docker, in this directory run:
+
+```
+just build-binary linux-amd64
+```
+(also works with `linux-arm64`)
+
+## CLI Tests
+
+To run CLI tests run the following command (replace `darwin-arm64` with your local OS and architecture).  This will run the CLI tests located in `./bats/`:
+
+```
+just test-cli darwin-amd64
+```
+
+To run against a Linux OS, first, build the docker containers needed:
+```
+just build linux-amd64
+```
+
+Now you can run the tests:
+
+```
+just docker_test-cli linux-amd64
+```
+(also works with `linux-arm64`)
+
+## Publishing UI Tests to Connect
+
+To run these tests locally, you'll need:
+* access to Connect Fuzzbucket Credentials
+* environment variables set for `FUZZBUCKET_CREDENTIALS` and `FUZZBUCKET_URL`.
+
+To run the Publishing UI tests run the following command (replace `darwin-arm64` with your local OS and architecture).  This will start Connect in a fuzzbucket instance and run the Publish UI tests located in `../web/cypress/test`:
+
 ```
-just build all
+just init-connect-and-publish darwin-arm64
 ```
-Or if you just want to build one service, you can specify it:
+
+To run against a Linux OS first, build the docker containers needed (also works with `linux-arm64`):
 ```
 just build linux-amd64
 ```
 
-Run against linux-amd64 and linux-arm64:
+Now you can run the tests:
+
 ```
-just run-client linux-amd64 
-just run-client linux-arm64
+just docker_init-connect-and-publish linux-amd64
 ```
 
-These commands will build/use a docker container running the specified linux platform.
+By default the `justfile` sets `TEST_SCENARIO=basic`. This loads the `environment/.basic` environment which sets the command arguments used to start the `connect-client`. By default we only deploy the `fastapi-sample` content. If you'd like to run against all content available set `CONTENT=all_content` which will deploy all the content under `./sample-content/`. If you want to run a single deployment, set `CONTENT` to the path of any content you'd like to deploy.
 
-To run the tests against your local environment (replace `darwin-amd64` with the proper binary):
+All Content Test:
+```
+CONTENT=all_content just init-connect-and-publish darwin-arm64
+```
 
+Single Content Test:
 ```
-just run-client-local darwin-amd64
+CONTENT="test/sample-content/python/stock-dashboard-python/" just init-connect-and-publish darwin-arm64
 ```
-Running locally installs BATS locally in the `test/libs` directory (.gitignore'd).
 
-Github Actions will use the virtual environment runners to test against `windows-amd64` and `darwin-amd64`.
 
 

test/deploy_helper.py

@@ -0,0 +1,10 @@
+import os
+
+# Specify the directory path
+directory_path = "./sample-content/python/"
+
+# Get a list of directories
+subdirectories = [d for d in os.listdir(directory_path) if os.path.isdir(os.path.join(directory_path, d))]
+
+# Print the list of directories
+print(str(subdirectories).replace("[","").replace("]","").replace("'","").replace(",",""))

test/docker/Dockerfile

@@ -1,8 +1,16 @@
 FROM ubuntu:jammy-20230425
+EXPOSE 9000
 VOLUME ${PWD}/../:/publishing-client
 WORKDIR /publishing-client/test
 ARG platform
-ENV platform $platform
+ARG FUZZBUCKET_SSH_KEY
+ARG FUZZBUCKET_URL
+ARG FUZZBUCKET_CREDENTIALS
+
+ENV platform=$platform
+ENV FUZZBUCKET_SSH_KEY=$FUZZBUCKET_SSH_KEY
+ENV FUZZBUCKET_URL=$FUZZBUCKET_URL
+ENV FUZZBUCKET_CREDENTIALS=$FUZZBUCKET_CREDENTIALS
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get clean && \
@@ -24,14 +32,19 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     libasound2 \
     libxtst6 \
     xauth \
-    xvfb
+    xvfb \
+    python3 \
+    python3-pip \
+    python-is-python3 \
+    jq
 
-RUN export GOLANG_VERSION=1.20.4 \
+RUN export GOLANG_VERSION=1.21.0 \
     && export GOLANG_DOWNLOAD_SHA256=698ef3243972a51ddb4028e4a1ac63dc6d60821bf18e59a807e051fee0a385bd \
     && curl -fsSL "https://go.dev/dl/go${GOLANG_VERSION}.${platform}.tar.gz" -o golang.tar.gz \
     && tar -C /usr/local -xzf golang.tar.gz \
     && rm golang.tar.gz
 ENV PATH="$PATH:/usr/local/go/bin"
+ENV PATH="$PATH:"
 
 RUN npm install -g n && \
     n stable

test/environments/.basic

@@ -0,0 +1 @@
+CMD_ARGS="${CONTENT} --listen=localhost:9000 --token=abc123 -n env"