Exit status for tests should be non-zero if test failures (#95)

* Add test script that doesn't rely on docker

* Modify test-words to be able to exit with non-zero exit status for test failures: Luhn exercise only

* Modify test-words to be able to exit with non-zero exit status for test failures: all other exercises

* Tweaks

* be more specific about the exit status

.gitignore

@@ -2,7 +2,6 @@
 .DS_Store
 bin/configlet
 bin/configlet.exe
-bin/**
 *.bak
 chg_test.rr
 .idea/**

bin/test-no-docker

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+usage="Usage: $0 -a|exercise_slug
+
+Test an example solution.
+Use -a to test all exercises.
+
+Example: $0 luhn"
+
+die() { echo "$*" >&2; exit 1; }
+
+make_test_dir() {
+    local exercise_dir=$1
+    local slug=${exercise_dir##*/}
+    local key=$2
+    (
+        cd "$exercise_dir"
+        cp -t "$test_dir" test.8th test-words.8th "${slug}-tests.8th" 
+        while IFS= read -r solution; do
+            cp "$solution" "$test_dir/$slug.8th"
+        done < <(
+            jq -r --arg key "$key" '.files[$key][]' .meta/config.json
+        )
+    )
+}
+
+test_one() {
+    local slug=$1
+    if [[ -d "./exercises/concept/$slug" ]]; then
+        make_test_dir "./exercises/concept/$slug" exemplar
+    elif [[ -d "./exercises/practice/$slug" ]]; then
+        make_test_dir "./exercises/practice/$slug" example
+    else
+        die "no such exercise: $slug"
+    fi
+    (
+        cd "$test_dir" || die "cannot cd to $test_dir"
+        8th test.8th
+    )
+}
+
+cleanup() { rm -rf "$test_dir"; }
+
+test_dir=$(mktemp -d)
+trap cleanup EXIT
+
+all=false
+while getopts :ha opt; do
+    case $opt in
+        h) die "$usage" ;;
+        a) all=true ;;
+        ?) die "unknown option: -$OPTARG" ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if $all; then
+    shopt -s nullglob
+    for dir in ./exercises/concept/* ./exercises/practice/*; do
+        slug=${dir##*/}
+        if test_one "$slug"; then
+            echo "$slug tests exited with $?"
+            echo
+        else
+            die "$slug tests exited with $?"
+        fi
+    done
+else
+    slug=$1
+    [[ -n $slug ]] || die "$usage"
+    test_one "$slug"
+fi

exercises/practice/acronym/acronym-tests.8th

@@ -9,3 +9,5 @@
 "from phrases with consecutive delimiters" "SIMUFTA" ( "Something - I made up from thin air" acronym ) test_eqs
 "from phrases with underscore emphasis" "TRNT" ( "The Road _Not_ Taken" acronym ) test_eqs
 
+
+end-of-tests

exercises/practice/acronym/test-words.8th

@@ -1,114 +1,107 @@
 needs console/loaded
+con:onBlack
 
 -1 var, test-count
+var tests-passed
+var tests-failed
 
 : tests \ n -- 
     test-count ! 
 ;
 
-: decr-test-count \ --
-    test-count @ n:1- test-count !
+: test-passed \ s --
+    tests-passed @ n:1+ tests-passed !
+    con:green . space " ... OK" . con:white cr
+;
+
+: test-failed \ s --
+    tests-failed @ n:1+ tests-failed !
+    con:red . space " ... FAIL" . con:white cr
 ;
 
 : isword?  \ x -- x f
    dup  >kind  ns:w   n:= 
 ;
 
 : test_eq \ s x w -- | s w x --
-   decr-test-count
    isword? !if swap then
    w:exec
-   n:= if
-    con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   n:=
+   if test-passed else test-failed then 
 ;
 
 : test_eqs \ s x w -- | s w x -- 
-   decr-test-count
    isword? !if swap then
    w:exec
-   s:= if
-    con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   s:=
+   if test-passed else test-failed then 
 ;
 
 : test_true \ s w --
-   decr-test-count
    w:exec
-   if
-    con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   if test-passed else test-failed then 
 ;
 
 : test_false \ s w --
-   decr-test-count
    w:exec
-   !if
-    con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   if test-failed else test-passed then 
 ;
 
 : test_null \ s w --
-   decr-test-count
    w:exec
-   null? if
-    drop con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    drop con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   null?
+   if test-passed else test-failed then 
 ;
 
 \ compare arrays by testing elements with string equality
 : test_eqa \ s x w -- | s w x -- 
-   decr-test-count
    isword? !if swap then
    w:exec
-   ' s:= a:= nip nip if
-    con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   ' s:= a:= 2nip
+   if test-passed else test-failed then 
 ;
 
 
-: test_map_eq \ m m -- 
-   decr-test-count
+: test_map_eq \ m w -- | w m -- 
    isword? !if swap then
    w:exec
-   ' n:= m:= if
-    2drop
-    con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    2drop
-    con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   ' n:= m:= 2nip
+   if test-passed else test-failed then 
 ;
 
-: test_map_neq \ m m -- 
-   decr-test-count
+: test_map_neq \ m w -- | w m -- 
    isword? !if swap then
    w:exec
-   ' n:= m:= !if
-    2drop
-    con:green con:onBlack . space " ... OK" . con:white con:onBlack cr
-   else
-    2drop
-    con:red con:onBlack . space " ... FAIL" . con:white con:onBlack cr
-   then 
+   ' n:= m:= 2nip
+   if test-failed else test-passed then 
+;
+
+\ Num passed + num failed should == num tests
+: all-tests-run? \ -- T
+    tests-passed @ tests-failed @ n:+ test-count @ n:=
 ;
 
-( test-count @ 0 n:= 
-  !if 
-  con:red con:onBlack
-  "... FAIL - not all tests completed" . 
-  con:white con:onBlack 
-  cr 
-  then ) onexit
+( all-tests-run?  
+  !if con:red "... FAIL - not all tests completed" . con:white cr then
+) onexit
+
+\ Print a summary of the tests run
+( con:white
+  test-count   @ . space "tests planned - " .
+  tests-passed @ . space "passed - " .
+  tests-failed @ . space "failed" . cr
+) onexit
+
+\ Set the exit status:
+\   0 = all OK
+\   1 = not all tests were run (some error occurred)
+\   2 = some tests failed
+: end-of-tests \ --
+    all-tests-run?
+    if
+      tests-failed @ 0 n:= if 0 else 2 then
+    else
+      1
+    then
+    die
+;

exercises/practice/affine-cipher/affine-cipher-tests.8th

@@ -48,3 +48,5 @@
 "decode with a not coprime to m"
 ( "Test" 13 5 code> )  test_null
 
+
+end-of-tests