upgrade to pyproject.toml (#704)

.github/workflows/CI.yml

@@ -70,34 +70,29 @@ jobs:
     - uses: actions/checkout@v2
     - name: docker build
       run: |
-        mkdir focal_self_test
-        cp requirements.txt focal_self_test
-
-        cat << 'EOF' > focal_self_test/Dockerfile
+        # overwriting Dockerfile!
+        cat << 'EOF' > Dockerfile
           FROM ubuntu:20.04
           ENV LC_ALL C.UTF-8
           ENV LANG C.UTF-8
-          RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y python3-pip
+          RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y python3-pip git-core
 
           RUN useradd -ms /bin/bash -u 1337 wdler
           USER wdler
           WORKDIR /home/wdler
           RUN mkdir miniwdl
 
-          COPY requirements.txt /home/wdler/
-          RUN pip3 install --user -r requirements.txt
-
-          ENV PYTHONPATH $PYTHONPATH:/home/wdler/.local/lib/python3.6
           ENV PATH $PATH:/home/wdler/.local/bin
+          COPY . /home/wdler/miniwdl
+          RUN cd miniwdl && pip3 install --user .
 
-          # expectation -- mount miniwdl source tree at /home/wdler/miniwdl
-          CMD env -C miniwdl python3 -m WDL run_self_test
+          CMD miniwdl run_self_test
         EOF
-        docker build -t miniwdl_focal_self_test focal_self_test
+        docker build -t miniwdl_focal_self_test .
     - name: miniwdl run_self_test
       run: |
         docker run \
           --group-add $(stat -c %g /var/run/docker.sock) -v /var/run/docker.sock:/var/run/docker.sock \
-          -v $(pwd):/home/wdler/miniwdl -v /tmp:/tmp \
+          -v /tmp:/tmp \
           -e AWS_EC2_METADATA_DISABLED=true \
           miniwdl_focal_self_test

.gitignore

@@ -9,3 +9,4 @@ build/
 .eggs/
 .venv/
 venv/
+WDL/_version.py

.readthedocs.yml

@@ -10,5 +10,7 @@ sphinx:
 python:
   version: 3.8
   install:
-    - requirements: requirements.dev.txt
-    - requirements: requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - dev

CONTRIBUTING.md

@@ -12,7 +12,7 @@ Feedback and contributions to miniwdl are welcome, via issues and pull requests
 To set up your Linux development environment,
 
 1. `git clone --recursive` this repository or your fork thereof, and `cd` into it
-2. Install dependencies as illustrated in the [Dockerfile](https://github.com/chanzuckerberg/miniwdl/blob/main/Dockerfile) (OS packages + `pip3 install --user -r` both `requirements.txt` and `requirements.dev.txt`)
+2. Install dependencies as illustrated in the [Dockerfile](https://github.com/chanzuckerberg/miniwdl/blob/main/Dockerfile) (OS packages) and [pyproject.toml](pyproject.toml)  (PyPI dependencies).
 3. Invoking user must have [permission to control Docker](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)
 4. Try `python3 -m WDL run_self_test` to test the configuration
 

Dockerfile

@@ -1,4 +1,5 @@
-# builds docker image for running test suite for the contextual miniwdl source tree
+# dev docker image for running test suite on the contextual miniwdl source tree
+# (this is NOT a docker image for miniwdl end users!)
 #    docker build -t miniwdl .
 # run the full test suite -- notice configuration needed for it to command the host dockerd
 #    docker run  \
@@ -13,7 +14,7 @@ ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
 RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y \
     python3-pip python3-setuptools tzdata wget zip git-core default-jre jq shellcheck docker.io
-RUN pip3 install -U pip  # due to infamous pyca/cryptography#5771
+RUN pip3 install yq
 
 # add and become 'wdler' user -- it's useful to run the test suite as some arbitrary uid, because
 # the runner has numerous file permissions-related constraints
@@ -24,11 +25,10 @@ RUN mkdir miniwdl
 # https://github.com/actions/checkout/issues/760
 RUN git config --global --add safe.directory /home/wdler/miniwdl
 
-# install pip requirements
-COPY requirements.txt requirements.dev.txt /home/wdler/
-RUN bash -o pipefail -c "pip3 install --user -r requirements.dev.txt" && rm requirements.*
-ENV PYTHONPATH $PYTHONPATH:/home/wdler/.local/lib/python3.6
 ENV PATH $PATH:/home/wdler/.local/bin
+COPY pyproject.toml /home/wdler/miniwdl
+RUN tomlq -r '(.project.dependencies + .project["optional-dependencies"].dev)[]' miniwdl/pyproject.toml \
+    | xargs pip3 install --user && rm miniwdl/pyproject.toml
 
 # expectation -- mount miniwdl source tree at /home/wdler/miniwdl
-CMD make -C miniwdl
+CMD make -C miniwdl unit_tests

Makefile

@@ -71,7 +71,7 @@ pypi: bdist
 # build dist
 bdist:
 	rm -rf dist/
-	python3 setup.py sdist bdist_wheel
+	python -m build
 
 # sphinx
 doc:

docs/add_functions.md

@@ -5,8 +5,7 @@ This lab will show how to add new functions to the WDL standard library, thus in
 ```bash
 git clone --recursive https://github.com/chanzuckerberg/miniwdl.git  # or your own fork
 cd miniwdl
-pip3 install -r requirements.txt
-pip3 install -r requirements.dev.txt
+pip3 install '.[dev]'  # to install dev dependencies
 ```
 
 The standard library is implemented in `WDL/StdLib.py`, except for some details specific to the task runtime environment.

pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=6.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "miniwdl"
+dynamic = ["version"]
+description = "Workflow Description Language (WDL) local runner & developer toolkit"
+readme = "README.md"
+requires-python = ">=3.8"
+license = {text = "MIT"}
+authors = [
+    {name = "CZI", email = "[email protected]"}
+]
+urls = { "Homepage" = "https://github.com/chanzuckerberg/miniwdl" }
+classifiers = [
+    "Programming Language :: Python :: 3 :: Only",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+    "Topic :: Software Development :: Interpreters"
+]
+dependencies = [
+    "importlib-metadata>=3.10.1",
+    "regex>=2020.4.4",
+    "xdg>=5.0.0",
+    "docker>=3.4.0",
+    "PyYAML>=5.4.1,<7",
+    "argcomplete>=3,<4",
+    "pygtail~=0.14",
+    "coloredlogs>=15,<16",
+    "python-json-logger>=2,<3",
+    "lark~=1.1",
+    "bullet>=2,<3",
+    "psutil>=5,<6"
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff",
+    "mypy",
+    "pytest",
+    "testfixtures",
+    "pytest-xdist",
+    "pytest-cov",
+    "coverage",
+    "python-coveralls",
+    "build",
+    "wheel",
+    "twine",
+    "sphinx",
+    "sphinx-autobuild",
+    "sphinx_rtd_theme",
+    "sphinx-argparse",
+    "recommonmark",
+    "graphviz",
+    "boto3"
+]
+
+[project.scripts]
+miniwdl = "WDL.CLI:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["WDL"]
+
+[tool.setuptools_scm]
+write_to = "WDL/_version.py"
+
+[tool.mypy]
+mypy_path = "${MYPY_CONFIG_FILE_DIR}/stubs"
+
+[tool.ruff]
+exclude = ["WDL/_version.py"]