From ac58a50f9557639f5d117e6e5327fc4de4a05771 Mon Sep 17 00:00:00 2001
From: Reid Priedhorsky <1682574+reidpr@users.noreply.github.com>
Date: Tue, 31 Oct 2023 10:43:37 -0600
Subject: [PATCH] add doctests to test suite (#1744)

---
 .gitignore            |  2 ++
 Makefile.am           |  3 +-
 lib/filesystem.py     |  6 ++--
 misc/loc              |  3 +-
 test/Makefile.am      | 22 +++++++++---
 test/doctest-auto     | 32 +++++++++++++++++
 test/doctest.py.in    | 82 +++++++++++++++++++++++++++++++++++++++++++
 test/force-auto.py.in | 10 +++---
 8 files changed, 145 insertions(+), 15 deletions(-)
 create mode 100755 test/doctest-auto
 create mode 100755 test/doctest.py.in

diff --git a/.gitignore b/.gitignore
index a552538e4..f007e7ec9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,8 +26,10 @@ a.out
 /lib/version.py
 /lib/version.sh
 /lib/version.txt
+/test/build/30_doctest-auto.bats
 /test/force-auto.bats
 /test/docs-sane
+/test/doctest
 /test/fixtures/symlink-to-tmp
 /test/force-auto
 /test/make-perms-test
diff --git a/Makefile.am b/Makefile.am
index 833f33497..19c0a47db 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,7 +18,8 @@ install-exec-hook:
 	         $(DESTDIR)@bindir@/ch-image \
 	         $(DESTDIR)@bindir@/ch-run-oci \
 	         $(DESTDIR)@bindir@/ch-test \
-	         $(DESTDIR)@libdir@/charliecloud/base.sh; \
+	         $(DESTDIR)@libdir@/charliecloud/base.sh \
+	         $(DESTDIR)@libexecdir@/charliecloud/doctest; \
 	do \
 	  sed -Ei -e 's|^(ch_lib ?= ?).+/lib"?$$|\1"@libdir@/charliecloud"|' \
 	          -e 's|^(CHTEST_DIR=).+$$|\1@libexecdir@/charliecloud|' \
diff --git a/lib/filesystem.py b/lib/filesystem.py
index 938e67878..6705d6970 100644
--- a/lib/filesystem.py
+++ b/lib/filesystem.py
@@ -594,11 +594,11 @@ def strip(self, left=0, right=0):
 
            >>> a = Path("/a/b/c")
            >>> a.strip(left=1)
-           Path("a/b/c")
+           Path('a/b/c')
            >>> a.strip(right=1)
-           Path("/a/b")
+           Path('/a/b')
            >>> a.strip(left=1, right=1)
-           Path("a/b")
+           Path('a/b')
 
          It is an error if I don’t have at least left + right components,
          i.e., you can strip a path down to nothing but not further."""
diff --git a/misc/loc b/misc/loc
index a77beed33..77fb83f2f 100755
--- a/misc/loc
+++ b/misc/loc
@@ -243,8 +243,9 @@ find ./.github ./examples ./test -type f -a \( \
      -o -path ./test/fixtures/README \
      -o -path ./.github/PERUSEME \
      -o -path ./examples/chtest/printns \
-     -o -path ./test/common.bash \
      -o -path ./test/approved-trailing-whitespace \
+     -o -path ./test/common.bash \
+     -o -path ./test/doctest-auto \
      -o -path ./test/old-storage \
      -o -path ./test/sotest/files_inferrable.txt \
      -o -path ./test/whiteout \) | sort > /tmp/loc.test
diff --git a/test/Makefile.am b/test/Makefile.am
index 976b6d252..0c8cfca85 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -51,6 +51,9 @@ Build.centos7xz \
 Build.docker_pull \
 Build.missing \
 docs-sane \
+doctest \
+doctest-auto \
+force-auto \
 make-perms-test \
 old-storage
 
@@ -68,8 +71,10 @@ sotest/libsotest.so.1.0 \
 sotest/sotest
 
 CLEANFILES = $(sobuilts) \
-             docs-sane force-auto make-perms-test \
-             force-auto.bats
+             docs-sane \
+             doctest build/30_doctest-auto.bats \
+             force-auto force-auto.bats \
+             make-perms-test
 
 if ENABLE_TEST
 nobase_test_DATA = $(testfiles)
@@ -79,6 +84,9 @@ if ENABLE_CH_IMAGE  # this means we have Python
 nobase_test_DATA += force-auto.bats
 force-auto.bats: force-auto
 	./$< > $@
+nobase_test_DATA += build/30_doctest-auto.bats
+build/30_doctest-auto.bats: doctest-auto
+	./$< > $@
 endif
 # See comment about symlinks in examples/Makefile.am.
 all-local:
@@ -93,12 +101,16 @@ uninstall-hook:
 	rmdir $(DESTDIR)$(testdir)/fixtures || true
 	rmdir $$(find $(pkglibexecdir) -type d | sort -r)
 endif
-EXTRA_DIST = $(testfiles) $(testfiles_exec) \
-             docs-sane.py.in force-auto.py.in make-perms-test.py.in
+EXTRA_DIST = $(testfiles) \
+             $(testfiles_exec) \
+             docs-sane.py.in \
+             doctest.py.in \
+             force-auto.py.in \
+	     make-perms-test.py.in
 EXTRA_SCRIPTS = $(sobuilts)
 
 ## Python scripts - need text processing
-docs-sane force-auto make-perms-test: %: %.py.in
+docs-sane doctest force-auto make-perms-test: %: %.py.in
 	rm -f $@
 	sed -E 's|%PYTHON_SHEBANG%|@PYTHON_SHEBANG@|' < $< > $@
 	chmod +rx,-w $@  # respects umask
diff --git a/test/doctest-auto b/test/doctest-auto
new file mode 100755
index 000000000..9b736591c
--- /dev/null
+++ b/test/doctest-auto
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Print (on stdout) BATS tests to run doctests on each file in lib/.
+
+set -e -o pipefail
+
+cat <<EOF
+# AUTOGENERATED -- DO NOT EDIT
+
+load ../common
+
+setup () {
+    scope standard
+    [[ \$CH_TEST_BUILDER = ch-image ]] || skip 'ch-image only'
+}
+
+EOF
+
+
+for f in ../lib/*.py; do
+    m=$(basename "$f")
+    m=${m%%.py}
+    if [[ $m == version ]]; then
+        continue
+    fi
+cat <<EOF
+
+@test 'doctest: $m' {
+    ./doctest $m
+}
+EOF
+done
diff --git a/test/doctest.py.in b/test/doctest.py.in
new file mode 100755
index 000000000..f6f9b0d1d
--- /dev/null
+++ b/test/doctest.py.in
@@ -0,0 +1,82 @@
+#!%PYTHON_SHEBANG%
+
+import doctest
+import fnmatch
+import importlib.util
+import os
+import re
+import sys
+
+usage = """\
+Usage:
+
+  $ test/doctest MODULE [OBJECT_REGEX]
+
+Run doctests in Charliecloud Python MODULE (in lib/). If OBJECT_REGEX given,
+only fun tests on objects with names (excluding the module name) matching that
+regular expression (default all objects). Exits unsuccessfully on first
+failure."""
+
+
+# Command line arguments.
+
+try:
+   module_name = sys.argv[1]
+except IndexError:
+   module_name = "--help"
+try:
+   object_re = sys.argv[2]
+except IndexError:
+   object_re = ".*"
+if (module_name in ("-h", "--help", "-?")):
+   print(usage, file=sys.stderr)
+   sys.exit(1)  # help message is not a successful test
+
+
+# Import target module.
+
+ch_lib = os.path.dirname(os.path.abspath(__file__)) + "/../lib"
+sys.path.insert(0, ch_lib)
+import charliecloud as ch  # avoid circular import problems
+print("imported %s from %s" % (ch.__name__, ch.__file__))
+module = importlib.import_module(module_name)
+print("imported %s from %s" % (module.__name__, module.__file__))
+
+
+# Locate tests to run.
+# see: https://github.com/python/cpython/blob/73a003f/Lib/doctest.py#L1905
+
+tests_all = doctest.DocTestFinder().find(module)
+for test in tests_all:
+   test.name_short = re.sub(r"^[a-z_]+\.", "", test.name)
+tests_nonempty = [i for i in tests_all if len(i.examples) > 0]
+tests = [i for i in tests_nonempty if re.search(object_re, i.name_short)]
+print("will run %d/%d tests" % (len(tests), len(tests_nonempty)))
+
+
+# Run tests.
+
+out = ""
+def out_save(text):
+   global out
+   out += text
+runner = doctest.DocTestRunner(optionflags=(  doctest.DONT_ACCEPT_TRUE_FOR_1
+                                            | doctest.ELLIPSIS))
+for test in tests:
+   print("%s (%d examples) ... " % (test.name_short, len(test.examples)),
+         end="")
+   out = ""
+   results = runner.run(test, out=out_save)
+   assert (results.attempted == len(test.examples))
+   if (results.failed == 0):
+      print("ok")
+   else:
+      print("%d failed" % results.failed)
+      print(out)
+      print("big L, stopping tests")
+      sys.exit(1)
+
+
+# Summarize.
+
+print("all tests passed")
diff --git a/test/force-auto.py.in b/test/force-auto.py.in
index d1144988f..9bfab7a39 100644
--- a/test/force-auto.py.in
+++ b/test/force-auto.py.in
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+#!%PYTHON_SHEBANG%
 
 # This script generates a BATS file to exercise “ch-image build --force”
-# across a variety of distributions. It's used by Makefile.am.
+# across a variety of distributions. It’s used by Makefile.am.
 #
 # About each distribution, we remember:
 #
@@ -25,12 +25,12 @@
 # This would appear to yield 3×2×4 = 24 tests per distribution. However:
 #
 #   1. We only try pre-prepared images for “really need” commands with --force
-#      given, to save time, so it's at most 9 potential tests.
+#      given, to save time, so it’s at most 9 potential tests.
 #
-#   2. The pre-preparation step doesn't make sense for some distros or for
+#   2. The pre-preparation step doesn’t make sense for some distros or for
 #      --force=seccomp.
 #
-#   3. We've not yet determined an “apparently need” command for some distros.
+#   3. We’ve not yet determined an “apparently need” command for some distros.
 #
 # Bottom line, the number of tests per distro varies. See the code below for
 # specific details.