Provide development environment via nix (#2274)

I am only keeping the meaningful commit messages: * Add bash completion in development shells It turns out that nix shells have a nice way to load completions already. This even uses "bash" in the installation, so that it does not interfere with other shells that may load the nix environment with direnv. https://www.github.com/NixOS/nixpkgs/pull/103501 https://www.github.com/scop/bash-completion/blob/7f1165018a72f837d59b3cea7071efa0108841a2/README.md Since I now used the "lazy" approach for loading the completions, the completion on `python manage.py <Tab>` does not work anymore, because the builtin completions are found first. We could overwrite BASH_COMPLETION_USER_DIR to have the highest precedence, but I think that most users use `./manage.py` anyways (it is what we recommend in our README) and the current approach seems to be the most canonical way to do this within nix packages. * Use unix domain sockets instead of TCP for postgres and redis * Don't use `projectDir` This should avoid that editing some views.py file changes the hash of the poetry environment and thus save on some rebuilds. (The rebuilds would only create some symlinks, all actually expensive steps are cached, but this change won't hurt). * Remove `poetry` from packages While `poetry2nix` adds an overlay on the Python package set that disables running the tests, this was not true for the additional poetry from nixpkgs directly. With the override, this poetry had to be rebuilt and thus the tests ran again. * Add gettext to npm-ci pc process too This should fix msgfmt errors for `./manage.py compilemessages`.
e-valuation · Oct 13, 2024 · 38ce586 · 38ce586
1 parent bfd3d4b
commit 38ce586
Show file tree

Hide file tree

Showing 25 changed files with 1,853 additions and 487 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -39,16 +39,7 @@ updates:
     labels:
       - "[T] Dependencies"
   - package-ecosystem: "github-actions"
-    directory: "/.github/setup_nodejs"
-    schedule:
-      interval: "weekly"
-    groups:
-      actions:
-        patterns: ["*"]
-    labels:
-      - "[T] Dependencies"
-  - package-ecosystem: "github-actions"
-    directory: "/.github/setup_python"
+    directory: "/.github/setup_evap"
     schedule:
       interval: "weekly"
     groups:

diff --git a/.github/setup_evap/action.yml b/.github/setup_evap/action.yml
@@ -0,0 +1,39 @@
+name: "Setup EvaP Testing Environment"
+description: "Sets up the nix environment with caching and copy localsettings"
+
+inputs:
+  shell:
+    description: "name of development shell to use"
+    required: false
+    default: .#evap-dev
+  start-db:
+    description: "whether or not to run the database in the background"
+    required: false
+    default: false
+  npm-ci:
+    description: "whether or not to run `npm ci`"
+    required: false
+    default: false
+
+runs:
+  using: "composite"
+  steps:
+    - uses: DeterminateSystems/nix-installer-action@main
+    - uses: DeterminateSystems/magic-nix-cache-action@main
+    - uses: nicknovitski/nix-develop@v1
+      with:
+        arguments: "${{ inputs.shell }}"
+
+    - name: Add localsettings
+      run: cp evap/settings_test.py evap/localsettings.py
+      shell: bash
+
+    - name: Install Node dependencies
+      run: npm ci
+      shell: bash
+      if: ${{ inputs.npm-ci }}
+
+    - name: Start database
+      run: nix run .#services -- --detached && nix run .#wait-for-pc
+      shell: bash
+      if: ${{ inputs.start-db }}
diff --git a/.github/setup_nodejs/action.yml b/.github/setup_nodejs/action.yml
diff --git a/.github/setup_python/action.yml b/.github/setup_python/action.yml
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -12,23 +12,11 @@ jobs:
 
     runs-on: ubuntu-22.04
 
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: evap
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - name: Setup python
-        uses: ./.github/setup_python
+      - uses: actions/checkout@v4
+      - uses: ./.github/setup_evap
+        with:
+          start-db: true
 
       - name: Run tests
         run: coverage run manage.py test
@@ -42,23 +30,11 @@ jobs:
 
     runs-on: ubuntu-22.04
 
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: evap
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - name: Setup python
-        uses: ./.github/setup_python
+      - uses: actions/checkout@v4
+      - uses: ./.github/setup_evap
+        with:
+          start-db: true
 
       - name: Run tests
         run: python manage.py test --shuffle
@@ -69,12 +45,8 @@ jobs:
     name: MyPy
 
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - name: Setup python
-        uses: ./.github/setup_python
-
+      - uses: actions/checkout@v4
+      - uses: ./.github/setup_evap
       - name: Run MyPy
         run: mypy
 
@@ -84,33 +56,23 @@ jobs:
     name: Linter
 
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - name: Setup python
-        uses: ./.github/setup_python
-
+      - uses: actions/checkout@v4
+      - uses: ./.github/setup_evap
       - name: Run ruff
         run: ruff check .
-
       - name: Run pylint
         run: pylint evap tools
 
-
   formatter:
     runs-on: ubuntu-22.04
 
     name: Formatting
 
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - name: Setup python
-        uses: ./.github/setup_python
-
-      - name: Setup nodejs
-        uses: ./.github/setup_nodejs
+      - uses: actions/checkout@v4
+      - uses: ./.github/setup_evap
+        with:
+          npm-ci: true
 
       - name: Check code formatting
         run: black --check .
@@ -120,41 +82,19 @@ jobs:
       - name: Check TypeScript formatting
         run: npx prettier --list-different --loglevel debug 'evap/static/ts/**/*.ts'
 
-
   backup-process:
     runs-on: ubuntu-22.04
 
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: evap
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-      redis:
-        image: redis
-        options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
-        ports:
-          - 6379:6379
-
     name: Backup process
 
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
-
-      - name: Setup python
-        uses: ./.github/setup_python
+      - uses: ./.github/setup_evap
         with:
-          requirements-file: requirements.txt
-
-      - name: Setup nodejs
-        uses: ./.github/setup_nodejs
+          shell: .#evap # no dev-dependencies
+          start-db: true
 
       - name: Install additional dependencies
         run: sudo apt-get update && sudo apt-get install gettext
@@ -182,13 +122,12 @@ jobs:
     name: Compile Scss
 
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
-
-      - name: Setup nodejs
-        uses: ./.github/setup_nodejs
+      - uses: ./.github/setup_evap
+        with:
+          npm-ci: true
 
       - name: Compile Scss
         run: npx sass evap/static/scss/evap.scss evap/static/css/evap.css
@@ -198,34 +137,16 @@ jobs:
           name: css
           path: evap/static/css/evap.css
 
-
   render_pages:
     runs-on: ubuntu-22.04
 
     name: Render Html pages
 
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: evap
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-      redis:
-        image: redis
-        options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
-        ports:
-          - 6379:6379
-
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - name: Setup python
-        uses: ./.github/setup_python
+      - uses: actions/checkout@v4
+      - uses: ./.github/setup_evap
+        with:
+          start-db: true
 
       - name: Render pages
         run: coverage run manage.py ts render_pages
@@ -240,7 +161,6 @@ jobs:
           name: rendered-pages
           path: evap/static/ts/rendered
 
-
   typescript:
     runs-on: ubuntu-22.04
 
@@ -249,13 +169,14 @@ jobs:
     name: Test Typescript
 
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
+      - uses: ./.github/setup_evap
+        with:
+          npm-ci: true
 
-      - name: Setup nodejs
-        uses: ./.github/setup_nodejs
+      - run: npx puppeteer browsers install chrome
 
       - name: Compile Typescript
         run: npx tsc --project evap/static/ts/tsconfig.compile.json

diff --git a/.gitignore b/.gitignore
@@ -56,5 +56,6 @@ todo
 # flake8 configuration
 setup.cfg
 
-# User specific vagrant overrides (needs extra config)
-.Vagrantfile.local
+/data/
+/result
+.envrc