feat: Add integration testing of schema mismatch behaviour (#232)

* chore: update catalyst-ci version

* feat: add intregation test target for schema mismatch behaviour

* fix: update deny.toml

* chore: simplify pyproject

* fix: add name to docker-compose

* chore: tweak Earthfiles to play nice with CI

* fix: spelling

.config/dictionaries/project.dic

@@ -5,6 +5,7 @@ adminer
 androidx
 appspot
 asyncio
+asyncpg
 auditability
 bluefireteam
 BROTLI
@@ -31,6 +32,7 @@ dotglob
 drep
 dreps
 encryptor
+fetchval
 fontawesome
 formz
 Formz
@@ -46,11 +48,13 @@ iphoneos
 jetbrains
 jorm
 jormungandr
+Joaquín
 Jörmungandr
 kroki
 lcov
 Leshiy
 localizable
+loguru
 mdlint
 mitigations
 moderations
@@ -76,6 +80,7 @@ psql
 Ptarget
 pubkey
 pubspec
+pytest
 rapidoc
 redoc
 reqwest
@@ -120,4 +125,4 @@ nextest
 testcov
 testdocs
 fmtchk
-fmtfix
+fmtfix

catalyst-gateway/Earthfile

@@ -4,7 +4,7 @@ VERSION --try --global-cache 0.7
 
 # Set up our target toolchains, and copy our files.
 builder:
-    DO github.com/input-output-hk/catalyst-ci/earthly/rust:v2.6.0+SETUP
+    DO github.com/input-output-hk/catalyst-ci/earthly/rust:v2.7.0+SETUP
 
     COPY --dir .cargo .config Cargo.* clippy.toml deny.toml rustfmt.toml bin crates tests .
 

catalyst-gateway/deny.toml

@@ -108,6 +108,7 @@ allow = [
     "Apache-2.0",
     "Unicode-DFS-2016",
     "BSD-3-Clause",
+    "BSD-2-Clause",
     "BlueOak-1.0.0",
     "Apache-2.0 WITH LLVM-exception"
 ]
@@ -279,4 +280,4 @@ allow-git = [
 # 1 or more gitlab.com organizations to allow git sources for
 #gitlab = [""]
 # 1 or more bitbucket.org organizations to allow git sources for
-#bitbucket = [""]
+#bitbucket = [""]

catalyst-gateway/event-db/Earthfile

@@ -9,7 +9,7 @@ VERSION 0.7
 # Internal: builder is our Event db builder target. Prepares all necessary artifacts.
 #   CI target : dependency
 builder:
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.15+BUILDER \
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.7.0+BUILDER \
         --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff
 
     COPY ./../../+repo-config-2/repo/.sqlfluff .
@@ -21,30 +21,30 @@ builder:
 check:
     FROM +builder
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.15+CHECK
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.7.0+CHECK
 
 
 # format all SQL files in the current project.  Local developers tool.
 #   CI target : false
 format:
     LOCALLY
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.15+FORMAT --src=$(echo ${PWD}/../../)   
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.7.0+FORMAT --src=$(echo ${PWD}/../../)
 
 # build - an event db docker image.
 #   CI target : true
 build:
     FROM +builder
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.15+BUILD --image_name=event-db
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.15+DOCS --image_name=event-db
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.7.0+BUILD --image_name=event-db
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.7.0+DOCS --image_name=event-db
 
 # test the event db database schema
 #   CI target : true
 #test:
-#    FROM github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.15+postgres-base
+#    FROM github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.7.0+postgres-base
 
-#    COPY github.com/input-output-hk/catalyst-ci/earthly/utils:v2.0.15+shell-assert/assert.sh .
+#    COPY github.com/input-output-hk/catalyst-ci/earthly/utils:v2.7.0+shell-assert/assert.sh .
 
 #    COPY ./docker-compose.yml .
 #    WITH DOCKER \

catalyst-gateway/tests/Earthfile

@@ -5,7 +5,7 @@ build:
     RUN cargo build -p cat-gateway --release
     SAVE ARTIFACT target/release/cat-gateway cat-gateway
 
-cat-gateway-publish:
+package-cat-gateway:
     FROM alpine:3.19
     ARG tag="latest"
     ARG address
@@ -16,7 +16,7 @@ cat-gateway-publish:
     ENTRYPOINT ./cat-gateway run --address $address --database-url $db_url --log-level $log_level
     SAVE IMAGE cat-gateway:$tag
 
-schemathesis-publish:
+package-schemathesis:
     FROM python:3.12-alpine3.19
     ARG tag="latest"
     ARG max_examples=1000
@@ -40,9 +40,9 @@ fuzzer-api-test:
     COPY integration/docker-compose.yml .
     WITH DOCKER \
         --compose docker-compose.yml \
-        --load schemathesis:latest=(+schemathesis-publish --openapi_spec="http://127.0.0.1:3030/docs/cat-gateway.json") \
+        --load schemathesis:latest=(+package-schemathesis --openapi_spec="http://127.0.0.1:3030/docs/cat-gateway.json") \
         --load event-db:latest=(../event-db+build --with_historic_data=false) \
-        --load cat-gateway:latest=(+cat-gateway-publish --address="127.0.0.1:3030" \
+        --load cat-gateway:latest=(+package-cat-gateway --address="127.0.0.1:3030" \
                                         --db_url="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev") \
         --service event-db \
         --service cat-gateway \
@@ -86,4 +86,4 @@ integration-test:
     IF [ -f fail ]
         RUN echo "Integration tests failed" && \
             exit 1
-    END
+    END

catalyst-gateway/tests/schema-mismatch/Earthfile

@@ -0,0 +1,34 @@
+VERSION --global-cache 0.7
+
+builder:
+    FROM github.com/input-output-hk/catalyst-ci/earthly/python:v2.7.0+python-base
+
+    COPY --dir ./schema_mismatch README.md .
+    DO github.com/input-output-hk/catalyst-ci/earthly/python:v2.7.0+BUILDER
+
+package-tester:
+    FROM +builder
+
+    CMD poetry run pytest
+    # The following is useful for debugging the tests
+    # CMD poetry run pytest -vvvv --capture tee-sys --show-capture=stderr
+
+test:
+    FROM earthly/dind:alpine
+
+    ARG DB_URL="postgres://catalyst-event-dev:CHANGE_ME@event-db/CatalystEventDev"
+    ARG CAT_ADDRESS="0.0.0.0:3030"
+
+    WORKDIR /default
+    COPY ./docker-compose.yml .
+
+    WITH DOCKER \
+        --compose docker-compose.yml \
+        --load event-db:latest=(../../event-db+build --with_historic_data=false) \
+        --load cat-gateway:latest=(../+package-cat-gateway --address=$CAT_ADDRESS --db_url=$DB_URL) \
+        --load test:latest=(+package-tester) \
+        --service event-db \
+        --service cat-gateway \
+        --allow-privileged
+        RUN docker run --network=default_default test
+    END

catalyst-gateway/tests/schema-mismatch/README.md

@@ -0,0 +1,14 @@
+# Integration testing for DB Schema Version Mismatch behavior
+
+Sets up a containerized environment with the `EventDB` and `catalyst-gateway` services running.
+
+Integration tests are run in this environment that probe the behavior of the `catalyst-gateway` service in situations
+where the DB schema version changes during execution, and creates a mismatch with the version that gateway service expects.
+
+## Running
+
+To run:
+
+```bash
+earthly -P +test
+```

catalyst-gateway/tests/schema-mismatch/docker-compose.yml

@@ -0,0 +1,35 @@
+version: "3"
+
+services:
+  event-db:
+    image: event-db:latest
+    environment:
+      # Required environment variables for migrations
+      - DB_HOST=localhost
+      - DB_PORT=5432
+      - DB_NAME=CatalystEventDev
+      - DB_DESCRIPTION="Catalyst Event DB"
+      - DB_SUPERUSER=postgres
+      - DB_SUPERUSER_PASSWORD=postgres
+      - DB_USER=catalyst-event-dev
+      - DB_USER_PASSWORD=CHANGE_ME
+
+      - INIT_AND_DROP_DB=true
+      - WITH_MIGRATIONS=true
+      - WITH_SEED_DATA=true
+    ports:
+      - 5432:5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${DB_SUPERUSER} -d $${DB_SUPERUSER_PASSWORD}"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  cat-gateway:
+    image: cat-gateway:latest
+    hostname: gateway
+    ports:
+      - 3030:3030
+    depends_on:
+      event-db:
+        condition: service_healthy