diff --git a/test/sharness.sh b/test/sharness.sh index 7395be4..86d4a30 100644 --- a/test/sharness.sh +++ b/test/sharness.sh @@ -18,33 +18,73 @@ # along with this program. If not, see http://www.gnu.org/licenses/ . # Public: Current version of Sharness. -SHARNESS_VERSION="0.3.0" +SHARNESS_VERSION="1.1.0" export SHARNESS_VERSION # Public: The file extension for tests. By default, it is set to "t". -: ${SHARNESS_TEST_EXTENSION:=t} +: "${SHARNESS_TEST_EXTENSION:=t}" export SHARNESS_TEST_EXTENSION -# Keep the original TERM for say_color -ORIGINAL_TERM=$TERM +# Public: Root directory containing tests. Tests can override this variable, +# e.g. for testing Sharness itself. +if test -z "$SHARNESS_TEST_DIRECTORY" +then + SHARNESS_TEST_DIRECTORY=$(pwd) +else + # ensure that SHARNESS_TEST_DIRECTORY is an absolute path so that it + # is valid even if the current working directory is changed + SHARNESS_TEST_DIRECTORY=$(cd "$SHARNESS_TEST_DIRECTORY" && pwd) || exit 1 +fi +export SHARNESS_TEST_DIRECTORY + +# Reset TERM to original terminal if found, otherwise save original TERM +[ "x" = "x$SHARNESS_ORIG_TERM" ] && + SHARNESS_ORIG_TERM="$TERM" || + TERM="$SHARNESS_ORIG_TERM" +# Public: The unsanitized TERM under which sharness is originally run +export SHARNESS_ORIG_TERM + +# Export SHELL_PATH +: "${SHELL_PATH:=$SHELL}" +export SHELL_PATH + +# if --tee was passed, write the output not only to the terminal, but +# additionally to the file test-results/$BASENAME.out, too. +case "$SHARNESS_TEST_TEE_STARTED, $* " in +done,*) + # do not redirect again + ;; +*' --tee '*|*' --verbose-log '*) + mkdir -p "$SHARNESS_TEST_DIRECTORY/test-results" + BASE="$SHARNESS_TEST_DIRECTORY/test-results/$(basename "$0" ".$SHARNESS_TEST_EXTENSION")" + + # Make this filename available to the sub-process in case it is using + # --verbose-log. + SHARNESS_TEST_TEE_OUTPUT_FILE="$BASE.out" + export SHARNESS_TEST_TEE_OUTPUT_FILE + + # Truncate before calling "tee -a" to get rid of the results + # from any previous runs. + : >"$SHARNESS_TEST_TEE_OUTPUT_FILE" + + (SHARNESS_TEST_TEE_STARTED="done" ${SHELL_PATH} "$0" "$@" 2>&1; + echo $? >"$BASE.exit") | tee -a "$SHARNESS_TEST_TEE_OUTPUT_FILE" + test "$(cat "$BASE.exit")" = 0 + exit + ;; +esac # For repeatability, reset the environment to a known state. +# TERM is sanitized below, after saving color control sequences. LANG=C LC_ALL=C -PAGER=cat +PAGER="cat" TZ=UTC -TERM=dumb EDITOR=: -export LANG LC_ALL PAGER TZ TERM EDITOR +export LANG LC_ALL PAGER TZ EDITOR unset VISUAL CDPATH GREP_OPTIONS -# Line feed -LF=' -' - -[ "x$ORIGINAL_TERM" != "xdumb" ] && ( - TERM=$ORIGINAL_TERM && - export TERM && +[ "x$TERM" != "xdumb" ] && ( [ -t 1 ] && tput bold >/dev/null 2>&1 && tput setaf 1 >/dev/null 2>&1 && @@ -60,6 +100,8 @@ while test "$#" -ne 0; do immediate=t; shift ;; -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests) TEST_LONG=t; export TEST_LONG; shift ;; + --in|--int|--inte|--inter|--intera|--interac|--interact|--interacti|--interactiv|--interactive|--interactive-|--interactive-t|--interactive-te|--interactive-tes|--interactive-test|--interactive-tests): + TEST_INTERACTIVE=t; export TEST_INTERACTIVE; verbose=t; shift ;; -h|--h|--he|--hel|--help) help=t; shift ;; -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose) @@ -68,49 +110,69 @@ while test "$#" -ne 0; do # Ignore --quiet under a TAP::Harness. Saying how many tests # passed without the ok/not ok details is always an error. test -z "$HARNESS_ACTIVE" && quiet=t; shift ;; + --chain-lint) + chain_lint=t; shift ;; + --no-chain-lint) + chain_lint=; shift ;; --no-color) color=; shift ;; + --tee) + shift ;; # was handled already --root=*) root=$(expr "z$1" : 'z[^=]*=\(.*\)') shift ;; + --verbose-log) + verbose_log=t + shift ;; *) echo "error: unknown test option '$1'" >&2; exit 1 ;; esac done if test -n "$color"; then + # Save the color control sequences now rather than run tput + # each time say_color() is called. This is done for two + # reasons: + # * TERM will be changed to dumb + # * HOME will be changed to a temporary directory and tput + # might need to read ~/.terminfo from the original HOME + # directory to get the control sequences + # Note: This approach assumes the control sequences don't end + # in a newline for any terminal of interest (command + # substitutions strip trailing newlines). Given that most + # (all?) terminals in common use are related to ECMA-48, this + # shouldn't be a problem. + say_color_error=$(tput bold; tput setaf 1) # bold red + say_color_skip=$(tput setaf 4) # blue + say_color_warn=$(tput setaf 3) # brown/yellow + say_color_pass=$(tput setaf 2) # green + say_color_info=$(tput setaf 6) # cyan + say_color_reset=$(tput sgr0) + say_color_raw="" # no formatting for normal text say_color() { - ( - TERM=$ORIGINAL_TERM - export TERM + test -z "$1" && test -n "$quiet" && return case "$1" in - error) - tput bold; tput setaf 1;; # bold red - skip) - tput setaf 4;; # blue - warn) - tput setaf 3;; # brown/yellow - pass) - tput setaf 2;; # green - info) - tput setaf 6;; # cyan - *) - test -n "$quiet" && return;; + error) say_color_color=$say_color_error ;; + skip) say_color_color=$say_color_skip ;; + warn) say_color_color=$say_color_warn ;; + pass) say_color_color=$say_color_pass ;; + info) say_color_color=$say_color_info ;; + *) say_color_color=$say_color_raw ;; esac shift - printf "%s" "$*" - tput sgr0 - echo - ) + printf '%s%s%s\n' "$say_color_color" "$*" "$say_color_reset" } else say_color() { test -z "$1" && test -n "$quiet" && return shift - printf "%s\n" "$*" + printf '%s\n' "$*" } fi +TERM=dumb +export TERM + error() { say_color error "error: $*" EXIT_OK=t @@ -121,7 +183,7 @@ say() { say_color info "$*" } -test -n "$test_description" || error "Test script did not set test_description." +test -n "${test_description:-}" || error "Test script did not set test_description." if test "$help" = "t"; then echo "$test_description" @@ -130,7 +192,11 @@ fi exec 5>&1 exec 6<&0 -if test "$verbose" = "t"; then +if test "$verbose_log" = "t" +then + exec 3>>"$SHARNESS_TEST_TEE_OUTPUT_FILE" 4>&3 +elif test "$verbose" = "t" +then exec 4>&2 3>&1 else exec 4>/dev/null 3>/dev/null @@ -161,7 +227,7 @@ trap 'die' EXIT # implicitly by specifying the prerequisite name in calls to test_expect_success # or test_expect_failure. # -# $1 - Name of prerequiste (a simple word, in all capital letters by convention) +# $1 - Name of prerequisite (a simple word, in all capital letters by convention) # # Examples # @@ -198,7 +264,7 @@ test_have_prereq() { # prerequisites can be concatenated with ',' save_IFS=$IFS IFS=, - set -- $* + set -- $@ IFS=$save_IFS total_prereq=0 @@ -215,7 +281,7 @@ test_have_prereq() { negative_prereq= esac - total_prereq=$(($total_prereq + 1)) + total_prereq=$((total_prereq + 1)) case "$satisfied_prereq" in *" $prerequisite "*) satisfied_this_prereq=t @@ -226,7 +292,7 @@ test_have_prereq() { case "$satisfied_this_prereq,$negative_prereq" in t,|,t) - ok_prereq=$(($ok_prereq + 1)) + ok_prereq=$((ok_prereq + 1)) ;; *) # Keep a list of missing prerequisites; restore @@ -247,12 +313,12 @@ test_have_prereq() { # the text_expect_* functions instead. test_ok_() { - test_success=$(($test_success + 1)) - say_color "" "ok $test_count - $@" + test_success=$((test_success + 1)) + say_color "" "ok $test_count - $*" } test_failure_() { - test_failure=$(($test_failure + 1)) + test_failure=$((test_failure + 1)) say_color error "not ok $test_count - $1" shift echo "$@" | sed -e 's/^/# /' @@ -260,13 +326,13 @@ test_failure_() { } test_known_broken_ok_() { - test_fixed=$(($test_fixed + 1)) - say_color error "ok $test_count - $@ # TODO known breakage vanished" + test_fixed=$((test_fixed + 1)) + say_color error "ok $test_count - $* # TODO known breakage vanished" } test_known_broken_failure_() { - test_broken=$(($test_broken + 1)) - say_color warn "not ok $test_count - $@ # TODO known breakage" + test_broken=$((test_broken + 1)) + say_color warn "not ok $test_count - $* # TODO known breakage" } # Public: Execute commands in debug mode. @@ -287,10 +353,29 @@ test_debug() { test "$debug" = "" || eval "$1" } +# Public: Stop execution and start a shell. +# +# This is useful for debugging tests and only makes sense together with "-v". +# Be sure to remove all invocations of this command before submitting. +test_pause() { + if test "$verbose" = t; then + "$SHELL_PATH" <&6 >&3 2>&4 + else + error >&5 "test_pause requires --verbose" + fi +} + test_eval_() { # This is a separate function because some tests use # "return" to end a test_expect_success block early. - eval &3 2>&4 "$*" + case ",$test_prereq," in + *,INTERACTIVE,*) + eval "$*" + ;; + *) + eval &3 2>&4 "$*" + ;; + esac } test_run_() { @@ -299,6 +384,13 @@ test_run_() { test_eval_ "$1" eval_ret=$? + if test "$chain_lint" = "t"; then + test_eval_ "(exit 117) && $1" + if test "$?" != 117; then + error "bug in the test script: broken &&-chain: $1" + fi + fi + if test -z "$immediate" || test $eval_ret = 0 || test -n "$expecting_failure"; then test_eval_ "$test_cleanup" fi @@ -309,7 +401,7 @@ test_run_() { } test_skip_() { - test_count=$(($test_count + 1)) + test_count=$((test_count + 1)) to_skip= for skp in $SKIP_TESTS; do case $this_test.$test_count in @@ -328,7 +420,7 @@ test_skip_() { of_prereq=" of $test_prereq" fi - say_color skip >&3 "skipping test: $@" + say_color skip >&3 "skipping test: $*" say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" : true ;; @@ -426,6 +518,44 @@ test_expect_failure() { echo >&3 "" } +# Public: Run test commands and expect anything from them. Used when a +# test is not stable or not finished for some reason. +# +# When the test passed, an "ok" message is printed, but the number of +# fixed tests is not incremented. +# +# When it failed, a "not ok ... # TODO known breakage" message is +# printed, and the number of tests still broken is incremented. +# +# Failures from these tests won't cause --immediate to stop. +# +# Usually takes two arguments: +# $1 - Test description +# $2 - Commands to be executed. +# +# With three arguments, the first will be taken to be a prerequisite: +# $1 - Comma-separated list of test prerequisites. The test will be skipped if +# not all of the given prerequisites are set. To negate a prerequisite, +# put a "!" in front of it. +# $2 - Test description +# $3 - Commands to be executed. +# +# Returns nothing. +test_expect_unstable() { + test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= + test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_unstable" + export test_prereq + if ! test_skip_ "$@"; then + say >&3 "checking unstable test: $2" + if test_run_ "$2" unstable; then + test_ok_ "$1" + else + test_known_broken_failure_ "$1" + fi + fi + echo >&3 "" +} + # Public: Run command and ensure that it fails in a controlled way. # # Use it instead of "! ". For example, when dies due to a @@ -518,7 +648,7 @@ test_expect_code() { shift "$@" exit_code=$? - if test $exit_code = $want_code; then + if test "$exit_code" = "$want_code"; then return 0 fi @@ -528,7 +658,7 @@ test_expect_code() { # Public: Compare two files to see if expected output matches actual output. # -# The TEST_CMP variable defines the command used for the comparision; it +# The TEST_CMP variable defines the command used for the comparison; it # defaults to "diff -u". Only when the test script was started with --verbose, # will the command's output, the diff, be printed to the standard output. # @@ -551,6 +681,79 @@ test_cmp() { ${TEST_CMP:-diff -u} "$@" } +# Public: portably print a sequence of numbers. +# +# seq is not in POSIX and GNU seq might not be available everywhere, +# so it is nice to have a seq implementation, even a very simple one. +# +# $1 - Starting number. +# $2 - Ending number. +# +# Examples +# +# test_expect_success 'foo works 10 times' ' +# for i in $(test_seq 1 10) +# do +# foo || return +# done +# ' +# +# Returns 0 if all the specified numbers can be displayed. +test_seq() { + i="$1" + j="$2" + while test "$i" -le "$j" + do + echo "$i" || return + i=$(("$i" + 1)) + done +} + +# Public: Check if the file expected to be empty is indeed empty, and barfs +# otherwise. +# +# $1 - File to check for emptiness. +# +# Returns 0 if file is empty, 1 otherwise. +test_must_be_empty() { + if test -s "$1" + then + echo "'$1' is not empty, it contains:" + cat "$1" + return 1 + fi +} + +# debugging-friendly alternatives to "test [-f|-d|-e]" +# The commands test the existence or non-existence of $1. $2 can be +# given to provide a more precise diagnosis. +test_path_is_file () { + if ! test -f "$1" + then + echo "File $1 doesn't exist. $2" + false + fi +} + +test_path_is_dir () { + if ! test -d "$1" + then + echo "Directory $1 doesn't exist. $2" + false + fi +} + +# Check if the directory exists and is empty as expected, barf otherwise. +test_dir_is_empty () { + test_path_is_dir "$1" && + if test -n "$(find "$1" -mindepth 1 -maxdepth 1)" + then + echo "Directory '$1' is not empty, it contains:" + ls -la "$1" + return 1 + fi +} + # Public: Schedule cleanup commands to be run unconditionally at the end of a # test. # @@ -576,6 +779,23 @@ test_when_finished() { } && (exit \"\$eval_ret\"); eval_ret=\$?; $test_cleanup" } +# Public: Schedule cleanup commands to be run unconditionally when all tests +# have run. +# +# This can be used to clean up things like test databases. It is not needed to +# clean up temporary files, as test_done already does that. +# +# Examples: +# +# cleanup mysql -e "DROP DATABASE mytest" +# +# Returns the exit code of the last cleanup command executed. +final_cleanup= +cleanup() { + final_cleanup="{ $* + } && (exit \"\$eval_ret\"); eval_ret=\$?; $final_cleanup" +} + # Public: Summarize test results and exit with an appropriate error code. # # Must be called at the end of each test script. @@ -602,7 +822,7 @@ test_done() { if test -z "$HARNESS_ACTIVE"; then test_results_dir="$SHARNESS_TEST_DIRECTORY/test-results" mkdir -p "$test_results_dir" - test_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.counts" + test_results_path="$test_results_dir/$this_test.$$.counts" cat >>"$test_results_path" <<-EOF total $test_count @@ -621,7 +841,7 @@ test_done() { say_color warn "# still have $test_broken known breakage(s)" fi if test "$test_broken" != 0 || test "$test_fixed" != 0; then - test_remaining=$(( $test_count - $test_broken - $test_fixed )) + test_remaining=$((test_count - test_broken - test_fixed)) msg="remaining $test_remaining test(s)" else test_remaining=$test_count @@ -641,6 +861,8 @@ test_done() { fi say "1..$test_count$skip_all" + test_eval_ "$final_cleanup" + test -d "$remove_trash" && cd "$(dirname "$remove_trash")" && rm -rf "$(basename "$remove_trash")" @@ -656,14 +878,15 @@ test_done() { esac } -# Public: Root directory containing tests. Tests can override this variable, -# e.g. for testing Sharness itself. -: ${SHARNESS_TEST_DIRECTORY:=$(pwd)} -export SHARNESS_TEST_DIRECTORY +# Public: Source directory of test code and sharness library. +# This directory may be different from the directory in which tests are +# being run. +: "${SHARNESS_TEST_SRCDIR:=$(cd "$(dirname "$0")" && pwd)}" +export SHARNESS_TEST_SRCDIR # Public: Build directory that will be added to PATH. By default, it is set to # the parent directory of SHARNESS_TEST_DIRECTORY. -: ${SHARNESS_BUILD_DIRECTORY:="$SHARNESS_TEST_DIRECTORY/.."} +: "${SHARNESS_BUILD_DIRECTORY:="$SHARNESS_TEST_DIRECTORY/.."}" PATH="$SHARNESS_BUILD_DIRECTORY:$PATH" export PATH SHARNESS_BUILD_DIRECTORY @@ -672,19 +895,43 @@ SHARNESS_TEST_FILE="$0" export SHARNESS_TEST_FILE # Prepare test area. -test_dir="trash directory.$(basename "$SHARNESS_TEST_FILE" ".$SHARNESS_TEST_EXTENSION")" -test -n "$root" && test_dir="$root/$test_dir" -case "$test_dir" in -/*) SHARNESS_TRASH_DIRECTORY="$test_dir" ;; - *) SHARNESS_TRASH_DIRECTORY="$SHARNESS_TEST_DIRECTORY/$test_dir" ;; +SHARNESS_TRASH_DIRECTORY="trash directory.$(basename "$SHARNESS_TEST_FILE" ".$SHARNESS_TEST_EXTENSION")" +test -n "$root" && SHARNESS_TRASH_DIRECTORY="$root/$SHARNESS_TRASH_DIRECTORY" +case "$SHARNESS_TRASH_DIRECTORY" in +/*) ;; # absolute path is good + *) SHARNESS_TRASH_DIRECTORY="$SHARNESS_TEST_DIRECTORY/$SHARNESS_TRASH_DIRECTORY" ;; esac test "$debug" = "t" || remove_trash="$SHARNESS_TRASH_DIRECTORY" -rm -rf "$test_dir" || { +rm -rf "$SHARNESS_TRASH_DIRECTORY" || { EXIT_OK=t echo >&5 "FATAL: Cannot prepare test area" exit 1 } + +# +# Load any extensions in $srcdir/sharness.d/*.sh +# +if test -d "${SHARNESS_TEST_SRCDIR}/sharness.d" +then + for file in "${SHARNESS_TEST_SRCDIR}"/sharness.d/*.sh + do + # Ensure glob was not an empty match: + test -e "${file}" || break + + if test -n "$debug" + then + echo >&5 "sharness: loading extensions from ${file}" + fi + . "${file}" + if test $? != 0 + then + echo >&5 "sharness: Error loading ${file}. Aborting." + exit 1 + fi + done +fi + # Public: Empty trash directory, the test area, provided for each test. The HOME # variable is set to that directory too. export SHARNESS_TRASH_DIRECTORY @@ -692,10 +939,10 @@ export SHARNESS_TRASH_DIRECTORY HOME="$SHARNESS_TRASH_DIRECTORY" export HOME -mkdir -p "$test_dir" || exit 1 +mkdir -p "$SHARNESS_TRASH_DIRECTORY" || exit 1 # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). -cd -P "$test_dir" || exit 1 +cd -P "$SHARNESS_TRASH_DIRECTORY" || exit 1 this_test=${SHARNESS_TEST_FILE##*/} this_test=${this_test%.$SHARNESS_TEST_EXTENSION} @@ -708,4 +955,10 @@ for skp in $SKIP_TESTS; do esac done +test -n "$TEST_LONG" && test_set_prereq EXPENSIVE +test -n "$TEST_INTERACTIVE" && test_set_prereq INTERACTIVE + +# Make sure this script ends with code 0 +: + # vi: set ts=4 sw=4 noet :