tests: Implement jobserver inheritance

6 files changed, 132 insertions, 47 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8e96526..5ca0a75 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -59,7 +59,7 @@ jobs:
       - name: Run tests
         run: |
           jobs=$(sysctl -n hw.ncpu)
-          make -j$jobs distcheck JOBS=-j$jobs
+          make -j$jobs distcheck
 
   freebsd:
     name: FreeBSD
@@ -90,7 +90,7 @@ jobs:
 
           run: |
             chown -R action:action .
-            sudo -u action gmake -j$(nproc) distcheck JOBS=-j$(nproc)
+            sudo -u action gmake -j$(nproc) distcheck
 
   openbsd:
     name: OpenBSD
@@ -120,7 +120,7 @@ jobs:
           run: |
             chown -R action:action .
             jobs=$(sysctl -n hw.ncpu)
-            doas -u action gmake -j$jobs check JOBS=-j$jobs TEST_FLAGS="--sudo=doas --verbose=skipped"
+            doas -u action gmake -j$jobs check TEST_FLAGS="--sudo=doas --verbose=skipped"
 
   netbsd:
     name: NetBSD
@@ -153,7 +153,7 @@ jobs:
             PATH="/sbin:/usr/sbin:$PATH"
             chown -R action:action .
             jobs=$(sysctl -n hw.ncpu)
-            sudo -u action gmake -j$jobs check CC=clang LDFLAGS="-rpath /usr/pkg/lib" JOBS=-j$jobs TEST_FLAGS="--sudo --verbose=skipped"
+            sudo -u action gmake -j$jobs check CC=clang LDFLAGS="-rpath /usr/pkg/lib" TEST_FLAGS="--sudo --verbose=skipped"
 
   dragonflybsd:
     name: DragonFly BSD
@@ -184,7 +184,7 @@ jobs:
           run: |
             chown -R action:action .
             jobs=$(sysctl -n hw.ncpu)
-            sudo -u action gmake -j$jobs check JOBS=-j$jobs TEST_FLAGS="--sudo --verbose=skipped"
+            sudo -u action gmake -j$jobs check TEST_FLAGS="--sudo --verbose=skipped"
 
   omnios:
     name: OmniOS
@@ -216,4 +216,4 @@ jobs:
             PATH="/usr/xpg4/bin:$PATH"
             chown -R action:staff .
             jobs=$(getconf NPROCESSORS_ONLN)
-            sudo -u action gmake -j$jobs check LDFLAGS="-Wl,-rpath,/opt/ooce/lib/amd64" JOBS=-j$jobs TEST_FLAGS="--sudo --verbose=skipped"
+            sudo -u action gmake -j$jobs check LDFLAGS="-Wl,-rpath,/opt/ooce/lib/amd64" TEST_FLAGS="--sudo --verbose=skipped"
diff --git a/GNUmakefile b/GNUmakefile
index 34c7f37..e01e93a 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -304,13 +304,8 @@ check: $(CHECKS)
 check-units: $(BIN)/tests/units
 	$<
 
-JOBS := $(filter -j%,$(MAKEFLAGS))
-ifndef JOBS
-	JOBS := -j1
-endif
-
 $(STRATEGY_CHECKS): check-%: $(BIN)/bfs $(TEST_UTILS)
-	./tests/tests.sh $(JOBS) --bfs="$(BIN)/bfs -S $*" $(TEST_FLAGS)
+	+./tests/tests.sh --make="$(MAKE)" --bfs="$(BIN)/bfs -S $*" $(TEST_FLAGS)
 
 # Custom test flags for distcheck
 DISTCHECK_FLAGS := -s TEST_FLAGS="--sudo --verbose=skipped"
diff --git a/tests/getopts.sh b/tests/getopts.sh
index ac75140..5214e9f 100644
--- a/tests/getopts.sh
+++ b/tests/getopts.sh
@@ -10,6 +10,7 @@ if command -v nproc &>/dev/null; then
 else
     JOBS=1
 fi
+MAKE=
 PATTERNS=()
 SUDO=()
 STOP=0
@@ -24,15 +25,19 @@ VERBOSE_TESTS=0
 usage() {
     local pad=$(printf "%*s" ${#0} "")
     color cat <<EOF
-Usage: ${GRN}$0${RST} [${BLU}-j${RST}${BLD}N${RST}] [${BLU}--bfs${RST}=${MAG}path/to/bfs${RST}] [${BLU}--sudo${RST}[=${BLD}COMMAND${RST}]] [${BLU}--stop${RST}]
-       $pad [${BLU}--no-clean${RST}] [${BLU}--update${RST}] [${BLU}--verbose${RST}[=${BLD}LEVEL${RST}]] [${BLU}--help${RST}]
-       $pad [${BLU}--posix${RST}] [${BLU}--bsd${RST}] [${BLU}--gnu${RST}] [${BLU}--all${RST}] [${BLD}TEST${RST} [${BLD}TEST${RST} ...]]
+Usage: ${GRN}$0${RST}
+           [${BLU}-j${RST}${BLD}N${RST}] [${BLU}--make${RST}=${BLD}MAKE${RST}] [${BLU}--bfs${RST}=${BLD}path/to/bfs${RST}] [${BLU}--sudo${RST}[=${BLD}COMMAND${RST}]]
+           [${BLU}--stop${RST}] [${BLU}--no-clean${RST}] [${BLU}--update${RST}] [${BLU}--verbose${RST}[=${BLD}LEVEL${RST}]] [${BLU}--help${RST}]
+           [${BLU}--posix${RST}] [${BLU}--bsd${RST}] [${BLU}--gnu${RST}] [${BLU}--all${RST}] [${BLD}TEST${RST} [${BLD}TEST${RST} ...]]
 
-  [${BLU}-j${RST}${BLD}N${RST}]
+  ${BLU}-j${RST}${BLD}N${RST}
       Run ${BLD}N${RST} tests in parallel (default: ${BLD}$JOBS${RST})
 
-  ${BLU}--bfs${RST}=${MAG}path/to/bfs${RST}
-      Set the path to the bfs executable to test (default: ${MAG}./bin/bfs${RST})
+  ${BLU}--make${RST}=${BLD}MAKE${RST}
+      Use the jobserver from ${BLD}MAKE${RST}, e.g. ${BLU}--make${RST}=${BLD}"make -j$JOBS"${RST}
+
+  ${BLU}--bfs${RST}=${BLD}path/to/bfs${RST}
+      Set the path to the bfs executable to test (default: ${BLD}./bin/bfs${RST})
 
   ${BLU}--sudo${RST}[=${BLD}COMMAND${RST}]
       Run tests that require root using ${GRN}sudo${RST} or the given ${BLD}COMMAND${RST}
@@ -75,6 +80,9 @@ parse_args() {
             -j?*)
                 JOBS="${arg#-j}"
                 ;;
+            --make=*)
+                MAKE="${arg#*=}"
+                ;;
             --bfs=*)
                 BFS="${arg#*=}"
                 ;;
@@ -138,6 +146,8 @@ parse_args() {
         esac
     done
 
+    read -a MAKE <<<"$MAKE"
+
     # Try to resolve the path to $BFS before we cd, while also supporting
     # --bfs="./bin/bfs -S ids"
     read -a BFS <<<"${BFS:-$BIN/bfs}"
diff --git a/tests/run.sh b/tests/run.sh
index 85d961f..0b975b9 100644
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -27,30 +27,40 @@ debug_err() {
     local ret=$? line func file
     callers | while read -r line func file; do
         if [ "$func" = source ]; then
-            local cmd="$(awk "NR == $line" "$file" 2>/dev/null)" || :
-            debug "$file" $line "${RED}error $ret${RST}" "$cmd" >&$DUPERR
+            debug "$file" $line "${RED}error $ret${RST}" >&$DUPERR
             break
         fi
     done
 }
 
-# Run a single test
-run_test() (
+# Source a test
+source_test() (
     set -eE
     trap debug_err ERR
+
+    if ((${#MAKE[@]})); then
+        # Close the jobserver pipes
+        exec {READY_PIPE}<&- {DONE_PIPE}>&-
+    fi
+
     cd "$TMP"
     source "$@"
 )
 
-# Run a test in the background
-bg_test() {
+# Run a test
+run_test() {
     if ((VERBOSE_ERRORS)); then
-        run_test "$1"
+        source_test "$1"
     else
-        run_test "$1" 2>"$TMP/$TEST.err"
+        source_test "$1" 2>"$TMP/$TEST.err"
     fi
     ret=$?
 
+    if ((${#MAKE[@]})); then
+        # Write one byte to the done pipe
+        printf . >&$DONE_PIPE
+    fi
+
     case $ret in
         0)
             if ((VERBOSE_TESTS)); then
@@ -73,28 +83,87 @@ bg_test() {
     return $ret
 }
 
-# Wait for a background test to finish
-wait_test() {
-    wait -n
-    ret=$?
+# Count the tests running in the background
+BG=0
+
+# Run a test in the background
+bg_test() {
+    run_test "$1" &
+    ((++BG))
+}
+
+# Reap a finished background test
+reap_test() {
     ((BG--))
 
-    case $ret in
+    case "$1" in
         0)
             ((++passed))
-            return 0
             ;;
         $EX_SKIP)
             ((++skipped))
-            return 0
             ;;
         *)
             ((++failed))
-            return $ret
             ;;
     esac
 }
 
+# Wait for a background test to finish
+wait_test() {
+    local pid
+    wait -n -ppid
+    ret=$?
+    if [ -z "${pid:-}" ]; then
+        debug "${BASH_SOURCE[0]}" $((LINENO - 3)) "${RED}error $ret${RST}" >&$DUPERR
+        exit 1
+    fi
+
+    reap_test $ret
+}
+
+# Wait until we're ready to run another test
+wait_ready() {
+    if ((${#MAKE[@]})); then
+        # We'd like to parse the output of jobs -n, but we can't run it in a
+        # subshell or we won't get the right output
+        jobs -n >"$TMP/jobs"
+        while read -r job status ret foo; do
+            case "$status" in
+                Done)
+                    reap_test 0
+                    ;;
+                Exit)
+                    reap_test $ret
+                    ;;
+            esac
+        done <"$TMP/jobs"
+
+        # Read one byte from the ready pipe
+        read -r -N1 -u$READY_PIPE
+    elif ((BG >= JOBS)); then
+        wait_test
+    fi
+}
+
+# Run make as a co-process to use its job control
+comake() {
+    coproc {
+        # We can't just use std{in,out}, due to
+        # https://www.gnu.org/software/make/manual/html_node/Parallel-Input.html
+        exec {DONE_PIPE}<&0 {READY_PIPE}>&1
+        exec "${MAKE[@]}" -s \
+             -f "$TESTS/tests.mk" \
+             DONE=$DONE_PIPE \
+             READY=$READY_PIPE \
+             "${TEST_CASES[@]/#/tests/}" \
+             </dev/null >/dev/null
+    }
+
+    # coproc pipes aren't inherited by subshells, so dup them
+    exec {READY_PIPE}<&${COPROC[0]} {DONE_PIPE}>&${COPROC[1]}
+}
+
 # Run all the tests
 run_tests() {
     if ((VERBOSE_TESTS)); then
@@ -125,17 +194,17 @@ run_tests() {
         TEST_FMT="."
     fi
 
-    BG=0
+    if ((${#MAKE[@]})); then
+        comake
+    fi
 
     # Turn off set -e (but turn it back on in run_test)
     set +e
 
     for TEST in "${TEST_CASES[@]}"; do
-        if ((BG >= JOBS)); then
-            wait_test
-            if (($? && STOP)); then
-                break
-            fi
+        wait_ready
+        if (($? && STOP)); then
+            break
         fi
 
         percent=$((100 * ran / total))
@@ -144,8 +213,8 @@ run_tests() {
         mkdir -p "$TMP/$TEST"
         OUT="$TMP/$TEST.out"
 
-        bg_test "$TESTS/$TEST.sh" &
-        ((++BG, ++ran))
+        bg_test "$TESTS/$TEST.sh"
+        ((++ran))
     done
 
     while ((BG > 0)); do
@@ -185,7 +254,7 @@ skip() {
         caller | {
             read -r line file
             printf "${BOL}"
-            debug "$file" $line "" "$(awk "NR == $line" "$file")" >&$DUPOUT
+            debug "$file" $line "" >&$DUPOUT
         }
     fi
 
diff --git a/tests/tests.mk b/tests/tests.mk
new file mode 100644
index 0000000..5bf4f6c
--- /dev/null
+++ b/tests/tests.mk
@@ -0,0 +1,7 @@
+# Copyright © Tavian Barnes <tavianator@tavianator.com>
+# SPDX-License-Identifier: 0BSD
+
+# GNU makefile that exposes make's job control to tests.sh
+
+tests/%:
+	bash -c 'printf . >&$(READY) && read -r -N1 -u$(DONE)'
diff --git a/tests/util.sh b/tests/util.sh
index ec24958..7dba9fb 100644
--- a/tests/util.sh
+++ b/tests/util.sh
@@ -113,9 +113,13 @@ callers() {
 
 # Print a message including path, line number, and command
 debug() {
-    local file="${1/#*\/tests\//tests/}"
-    set -- "$file" "${@:2}"
-    color printf "${BLD}%s:%d:${RST} %s\n    %s\n" "$@"
+    local file="$1"
+    local line="$2"
+    local msg="$3"
+    local cmd="$(awk "NR == $line" "$file" 2>/dev/null)" || :
+    file="${file/#*\/tests\//tests/}"
+
+    color printf "${BLD}%s:%d:${RST} %s\n    %s\n" "$file" "$line" "$msg" "$cmd"
 }
 
 ## Deferred cleanup
@@ -163,7 +167,7 @@ pop_defer() {
     eval "$cmd" || ret=$?
 
     if ((ret != 0)); then
-        debug "$file" $line "${RED}error $ret${RST}" "defer $cmd" >&$DUPERR
+        debug "$file" $line "${RED}error $ret${RST}" >&$DUPERR
     fi
 
     return $ret