treewide: Improve docker container build, CI caching and python dependencies

.github/workflows/build-docker.yml

@@ -25,16 +25,16 @@ jobs:
       - name: Disk usage after freeing up space
         run: df -h
       # Actually build the Docker container
-      - uses: actions/checkout@v2
-      - uses: docker/setup-buildx-action@v1
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
       - name: GHCR Log-in
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: util/container/Dockerfile

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     name: Build documentation
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:pyproject
     steps:
       - uses: actions/checkout@v2
       - name: Build docs
@@ -29,7 +29,7 @@ jobs:
     name: Python unit tests
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:pyproject
     steps:
       - uses: actions/checkout@v2
       - name: Run pytest
@@ -43,7 +43,7 @@ jobs:
     name: Simulate SW on Snitch Cluster w/ Verilator
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:pyproject
     steps:
       - uses: actions/checkout@v2
         with:
@@ -77,7 +77,7 @@ jobs:
     name: Simulate SW on Snitch Cluster w/ Banshee
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:pyproject
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/lint.yml

@@ -43,7 +43,7 @@ jobs:
     name: Check Sources Up-to-Date
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:pyproject
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/publish-docs.yml

@@ -11,7 +11,7 @@ jobs:
     name: Deploy documentation
     runs-on: ubuntu-22.04
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:pyproject
     steps:
       - uses: actions/checkout@v2
       # For some reason, the checkout is done by a different user,

.gitlab-ci.yml

@@ -16,14 +16,6 @@ variables:
 before_script:
   source iis-setup.sh
 
-##############
-# Build docs #
-##############
-
-docs:
-  script:
-    - make docs
-
 #####################
 # Python unit tests #
 #####################

docs/ug/documentation.md

@@ -7,7 +7,7 @@ theme](https://squidfunk.github.io/mkdocs-material/). Before building the
 documentation, make sure you have the required dependencies installed:
 
 ```shell
-pip install -r docs/requirements.txt
+pip install .[docs]
 ```
 
 After everything is installed, you can build a static copy of the `html` documentation by

iis-setup.sh

@@ -15,12 +15,10 @@ export LLVM_BINROOT=/usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin
 # Create Python virtual environment with required packages
 /usr/local/anaconda3-2022.05/bin/python3 -m venv .venv
 source .venv/bin/activate
-# Unpack packages in a local temporary directory which can be safely cleaned
-# after installation. Also protects against "No space left on device" errors
+# Install local packages in editable mode and unpack packages in a
+# local temporary directory which can be safely cleaned after installation.
+# Also protects against "No space left on device" errors
 # occurring when the /tmp folder is filled by other processes.
 mkdir tmp
-TMPDIR=tmp pip install -r python-requirements.txt
+TMPDIR=tmp pip install -e .
 rm -rf tmp
-
-# Install local packages in editable mode.
-pip install -e .

pyproject.toml

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: SHL-0.51
 
 [build-system]
-requires = ["setuptools>=42"]
+requires = ["setuptools>=64"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -12,8 +12,52 @@ authors = [
     {name = "Luca Colagrande", email = "[email protected]"}
 ]
 dynamic = ["version"]
+dependencies = [
+    # Keep sorted.
+    # General dependencies.
+    "bin2coe",
+    "dataclasses",
+    "editorconfig-checker==2.3.51",
+    "flake8",
+    "gitpython",
+    "hjson",
+    "humanize",
+    "json5",
+    "jsonref",
+    "jsonschema",
+    "mako",
+    "matplotlib",
+    "mkdocs-material",
+    "pandas",
+    "prettytable",
+    "progressbar2",
+    "psutil",
+    "pyelftools",
+    "pyflexfloat",
+    "pytablewriter",
+    "pytest",
+    "pyyaml",
+    "scikit-learn",
+    "tabulate",
+    "termcolor",
+    "yamllint",
+    # DNN lib dependencies.
+    "numpy",
+    "torch",
+    "json5"
+]
+
+[project.optional-dependencies]
+docs = [
+    "mkdocs",
+    "mkdocs-material",
+    "mkdocs-include-markdown-plugin",
+    "mkdocs-macros-plugin",
+    "mkdocstrings",
+    "mkdocstrings-python",
+]
 
 [tool.setuptools.package-dir]
 "snitch.dnn" = "sw/dnn"
 "snitch.blas" = "sw/blas"
-"snitch.util" = "util"
+"snitch.util" = "util"

util/container/Dockerfile

@@ -55,7 +55,7 @@ RUN cd Python-${PYTHON_VERSION} && \
 
 # Build Rust tools
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-ENV PATH "/root/.cargo/bin:${PATH}"
+ENV PATH="/root/.cargo/bin:${PATH}"
 RUN rustup install 1.70.0
 RUN rustup override set 1.70.0
 
@@ -75,7 +75,7 @@ WORKDIR /tools
 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz && \
     tar -x -f cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz --strip-components=1 -C . && \
     rm -rf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-ENV PATH "/tools/bin:${PATH}"
+ENV PATH="/tools/bin:${PATH}"
 
 # Install `banshee` (needs cmake)
 RUN rustup install 1.67.0
@@ -87,10 +87,6 @@ RUN cargo install --path /tmp/banshee
 RUN wget https://github.com/pulp-platform/bender/releases/download/v${BENDER_VERSION}/bender-${BENDER_VERSION}-x86_64-linux-gnu-ubuntu18.04.tar.gz
 RUN tar xzf bender-${BENDER_VERSION}-x86_64-linux-gnu-ubuntu18.04.tar.gz
 
-# Install spike-dasm
-RUN wget https://github.com/pulp-platform/riscv-isa-sim/releases/download/snitch-v${SPIKE_DASM_VERSION}/snitch-spike-dasm-${SPIKE_DASM_VERSION}-x86_64-linux-gnu-ubuntu18.04.tar.gz
-RUN tar xzf snitch-spike-dasm-${SPIKE_DASM_VERSION}-x86_64-linux-gnu-ubuntu18.04.tar.gz
-
 # Install Doxygen
 RUN wget https://www.doxygen.nl/files/doxygen-${DOXYGEN_VERSION}.linux.bin.tar.gz
 RUN tar xzf doxygen-${DOXYGEN_VERSION}.linux.bin.tar.gz
@@ -104,7 +100,7 @@ ARG VERIBLE_VERSION=0.0-3318-g8d254167
 LABEL version="0.1"
 LABEL description="Snitch container for hardware and software development."
 LABEL maintainer="[email protected]"
-LABEL org.opencontainers.image.source https://github.com/pulp-platform/snitch_cluster
+LABEL org.opencontainers.image.source=https://github.com/pulp-platform/snitch_cluster
 
 WORKDIR /tools
 
@@ -135,13 +131,13 @@ RUN latest_tag=`curl -s -H "Accept: application/vnd.github.v3+json" https://api.
     echo "SNITCH_LLVM_VERSION=${SNITCH_LLVM_VERSION} LLVM_TAR=${LLVM_TAR} latest_tag=${latest_tag}" && \
     wget -qO- https://github.com/pulp-platform/llvm-project/releases/download/${SNITCH_LLVM_VERSION}/${LLVM_TAR} | \
     tar xvz --strip-components=1 -C riscv-llvm
-ENV LLVM_BINROOT "/tools/riscv-llvm/bin"
+ENV LLVM_BINROOT="/tools/riscv-llvm/bin"
 
 # Install Verible
 RUN wget https://github.com/chipsalliance/verible/releases/download/v${VERIBLE_VERSION}/verible-v${VERIBLE_VERSION}-linux-static-x86_64.tar.gz && \
     tar -x -f verible-v${VERIBLE_VERSION}-linux-static-x86_64.tar.gz --strip-components=1 -C . && \
     rm -rf verible-v${VERIBLE_VERSION}-linux-static-x86_64.tar.gz
-ENV PATH "/tools/bin:${PATH}"
+ENV PATH="/tools/bin:${PATH}"
 
 # Install git>=2.18, required by Github checkout action to recurse submodules
 RUN apt-get update && apt-get install software-properties-common -y && \
@@ -151,27 +147,26 @@ RUN apt-get update && apt-get install software-properties-common -y && \
 
 # Copy artifacts from stage 1.
 COPY --from=builder /tools/bender bin/
-COPY --from=builder /tools/spike-dasm bin/
 COPY --from=builder /root/.cargo/bin/banshee bin/
 COPY --from=builder /opt/python /opt/python
 COPY --from=builder /tools/verilator /tools/verilator/
 COPY --from=builder /tools/doxygen/bin/doxygen bin/
 
 # Create and activate virtual environment
-ENV VIRTUAL_ENV "/root/.venvs/snitch_cluster"
+ENV VIRTUAL_ENV="/root/.venvs/snitch_cluster"
 RUN /opt/python/bin/python3 -m venv ${VIRTUAL_ENV}
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 # Install Python requirements
-COPY python-requirements.txt /tmp/python-requirements.txt
-COPY docs/requirements.txt /tmp/docs/requirements.txt
-COPY sw/dnn/requirements.txt /tmp/sw/dnn/requirements.txt
-RUN pip install -r /tmp/python-requirements.txt
-
+COPY pyproject.toml .
+COPY sw/dnn ./sw/dnn
+COPY sw/blas ./sw/blas
+COPY util ./util
+RUN pip install .[docs]
 # Add Verilator to PATH
-ENV PATH "/tools/verilator/bin:${PATH}"
-ENV VLT_ROOT "/tools/verilator/share/verilator/"
+ENV PATH="/tools/verilator/bin:${PATH}"
+ENV VLT_ROOT="/tools/verilator/share/verilator/"
 
 # Set locale to UTF-8, required because Python 3.6 defaults on ASCII encoding.
 # See https://click.palletsprojects.com/en/8.1.x/unicode-support/
-ENV LC_ALL "C.UTF-8"
-ENV LANG   "C.UTF-8"
+ENV LC_ALL="C.UTF-8"
+ENV LANG="C.UTF-8"