sharness.sh (18531B)
1 #!/usr/bin/env bash 2 # 3 # Copyright (c) 2011-2012 Mathias Lafeldt 4 # Copyright (c) 2005-2012 Git project 5 # Copyright (c) 2005-2012 Junio C Hamano 6 # 7 # This program is free software: you can redistribute it and/or modify 8 # it under the terms of the GNU General Public License as published by 9 # the Free Software Foundation, either version 2 of the License, or 10 # (at your option) any later version. 11 # 12 # This program is distributed in the hope that it will be useful, 13 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # GNU General Public License for more details. 16 # 17 # You should have received a copy of the GNU General Public License 18 # along with this program. If not, see http://www.gnu.org/licenses/ . 19 20 # Public: Current version of Sharness. 21 SHARNESS_VERSION="0.3.0" 22 export SHARNESS_VERSION 23 24 # Public: The file extension for tests. By default, it is set to "t". 25 : ${SHARNESS_TEST_EXTENSION:=t} 26 export SHARNESS_TEST_EXTENSION 27 28 # Keep the original TERM for say_color 29 ORIGINAL_TERM=$TERM 30 31 # For repeatability, reset the environment to a known state. 32 LANG=C 33 LC_ALL=C 34 PAGER=cat 35 TZ=UTC 36 TERM=dumb 37 EDITOR=: 38 export LANG LC_ALL PAGER TZ TERM EDITOR 39 unset VISUAL CDPATH GREP_OPTIONS 40 41 # Line feed 42 LF=' 43 ' 44 45 [ "x$ORIGINAL_TERM" != "xdumb" ] && ( 46 TERM=$ORIGINAL_TERM && 47 export TERM && 48 [ -t 1 ] && 49 tput bold >/dev/null 2>&1 && 50 tput setaf 1 >/dev/null 2>&1 && 51 tput sgr0 >/dev/null 2>&1 52 ) && 53 color=t 54 55 while test "$#" -ne 0; do 56 case "$1" in 57 -d|--d|--de|--deb|--debu|--debug) 58 debug=t; shift ;; 59 -i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate) 60 immediate=t; shift ;; 61 -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests) 62 TEST_LONG=t; export TEST_LONG; shift ;; 63 -h|--h|--he|--hel|--help) 64 help=t; shift ;; 65 -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose) 66 verbose=t; shift ;; 67 -q|--q|--qu|--qui|--quie|--quiet) 68 # Ignore --quiet under a TAP::Harness. Saying how many tests 69 # passed without the ok/not ok details is always an error. 70 test -z "$HARNESS_ACTIVE" && quiet=t; shift ;; 71 --no-color) 72 color=; shift ;; 73 --root=*) 74 root=$(expr "z$1" : 'z[^=]*=\(.*\)') 75 shift ;; 76 *) 77 echo "error: unknown test option '$1'" >&2; exit 1 ;; 78 esac 79 done 80 81 if test -n "$color"; then 82 say_color() { 83 ( 84 TERM=$ORIGINAL_TERM 85 export TERM 86 case "$1" in 87 error) 88 tput bold; tput setaf 1;; # bold red 89 skip) 90 tput setaf 4;; # blue 91 warn) 92 tput setaf 3;; # brown/yellow 93 pass) 94 tput setaf 2;; # green 95 info) 96 tput setaf 6;; # cyan 97 *) 98 test -n "$quiet" && return;; 99 esac 100 shift 101 printf "%s" "$*" 102 tput sgr0 103 echo 104 ) 105 } 106 else 107 say_color() { 108 test -z "$1" && test -n "$quiet" && return 109 shift 110 printf "%s\n" "$*" 111 } 112 fi 113 114 error() { 115 say_color error "error: $*" 116 EXIT_OK=t 117 exit 1 118 } 119 120 say() { 121 say_color info "$*" 122 } 123 124 test -n "$test_description" || error "Test script did not set test_description." 125 126 if test "$help" = "t"; then 127 echo "$test_description" 128 exit 0 129 fi 130 131 exec 5>&1 132 exec 6<&0 133 if test "$verbose" = "t"; then 134 exec 4>&2 3>&1 135 else 136 exec 4>/dev/null 3>/dev/null 137 fi 138 139 test_failure=0 140 test_count=0 141 test_fixed=0 142 test_broken=0 143 test_success=0 144 145 die() { 146 code=$? 147 if test -n "$EXIT_OK"; then 148 exit $code 149 else 150 echo >&5 "FATAL: Unexpected exit with code $code" 151 exit 1 152 fi 153 } 154 155 EXIT_OK= 156 trap 'die' EXIT 157 158 # Public: Define that a test prerequisite is available. 159 # 160 # The prerequisite can later be checked explicitly using test_have_prereq or 161 # implicitly by specifying the prerequisite name in calls to test_expect_success 162 # or test_expect_failure. 163 # 164 # $1 - Name of prerequiste (a simple word, in all capital letters by convention) 165 # 166 # Examples 167 # 168 # # Set PYTHON prerequisite if interpreter is available. 169 # command -v python >/dev/null && test_set_prereq PYTHON 170 # 171 # # Set prerequisite depending on some variable. 172 # test -z "$NO_GETTEXT" && test_set_prereq GETTEXT 173 # 174 # Returns nothing. 175 test_set_prereq() { 176 satisfied_prereq="$satisfied_prereq$1 " 177 } 178 satisfied_prereq=" " 179 180 # Public: Check if one or more test prerequisites are defined. 181 # 182 # The prerequisites must have previously been set with test_set_prereq. 183 # The most common use of this is to skip all the tests if some essential 184 # prerequisite is missing. 185 # 186 # $1 - Comma-separated list of test prerequisites. 187 # 188 # Examples 189 # 190 # # Skip all remaining tests if prerequisite is not set. 191 # if ! test_have_prereq PERL; then 192 # skip_all='skipping perl interface tests, perl not available' 193 # test_done 194 # fi 195 # 196 # Returns 0 if all prerequisites are defined or 1 otherwise. 197 test_have_prereq() { 198 # prerequisites can be concatenated with ',' 199 save_IFS=$IFS 200 IFS=, 201 set -- $* 202 IFS=$save_IFS 203 204 total_prereq=0 205 ok_prereq=0 206 missing_prereq= 207 208 for prerequisite; do 209 case "$prerequisite" in 210 !*) 211 negative_prereq=t 212 prerequisite=${prerequisite#!} 213 ;; 214 *) 215 negative_prereq= 216 esac 217 218 total_prereq=$(($total_prereq + 1)) 219 case "$satisfied_prereq" in 220 *" $prerequisite "*) 221 satisfied_this_prereq=t 222 ;; 223 *) 224 satisfied_this_prereq= 225 esac 226 227 case "$satisfied_this_prereq,$negative_prereq" in 228 t,|,t) 229 ok_prereq=$(($ok_prereq + 1)) 230 ;; 231 *) 232 # Keep a list of missing prerequisites; restore 233 # the negative marker if necessary. 234 prerequisite=${negative_prereq:+!}$prerequisite 235 if test -z "$missing_prereq"; then 236 missing_prereq=$prerequisite 237 else 238 missing_prereq="$prerequisite,$missing_prereq" 239 fi 240 esac 241 done 242 243 test $total_prereq = $ok_prereq 244 } 245 246 # You are not expected to call test_ok_ and test_failure_ directly, use 247 # the text_expect_* functions instead. 248 249 test_ok_() { 250 test_success=$(($test_success + 1)) 251 say_color "" "ok $test_count - $@" 252 } 253 254 test_failure_() { 255 test_failure=$(($test_failure + 1)) 256 say_color error "not ok $test_count - $1" 257 shift 258 echo "$@" | sed -e 's/^/# /' 259 test "$immediate" = "" || { EXIT_OK=t; exit 1; } 260 } 261 262 test_known_broken_ok_() { 263 test_fixed=$(($test_fixed + 1)) 264 say_color error "ok $test_count - $@ # TODO known breakage vanished" 265 } 266 267 test_known_broken_failure_() { 268 test_broken=$(($test_broken + 1)) 269 say_color warn "not ok $test_count - $@ # TODO known breakage" 270 } 271 272 # Public: Execute commands in debug mode. 273 # 274 # Takes a single argument and evaluates it only when the test script is started 275 # with --debug. This is primarily meant for use during the development of test 276 # scripts. 277 # 278 # $1 - Commands to be executed. 279 # 280 # Examples 281 # 282 # test_debug "cat some_log_file" 283 # 284 # Returns the exit code of the last command executed in debug mode or 0 285 # otherwise. 286 test_debug() { 287 test "$debug" = "" || eval "$1" 288 } 289 290 test_eval_() { 291 # This is a separate function because some tests use 292 # "return" to end a test_expect_success block early. 293 eval </dev/null >&3 2>&4 "$*" 294 } 295 296 test_run_() { 297 test_cleanup=: 298 expecting_failure=$2 299 test_eval_ "$1" 300 eval_ret=$? 301 302 if test -z "$immediate" || test $eval_ret = 0 || test -n "$expecting_failure"; then 303 test_eval_ "$test_cleanup" 304 fi 305 if test "$verbose" = "t" && test -n "$HARNESS_ACTIVE"; then 306 echo "" 307 fi 308 return "$eval_ret" 309 } 310 311 test_skip_() { 312 test_count=$(($test_count + 1)) 313 to_skip= 314 for skp in $SKIP_TESTS; do 315 case $this_test.$test_count in 316 $skp) 317 to_skip=t 318 break 319 esac 320 done 321 if test -z "$to_skip" && test -n "$test_prereq" && ! test_have_prereq "$test_prereq"; then 322 to_skip=t 323 fi 324 case "$to_skip" in 325 t) 326 of_prereq= 327 if test "$missing_prereq" != "$test_prereq"; then 328 of_prereq=" of $test_prereq" 329 fi 330 331 say_color skip >&3 "skipping test: $@" 332 say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})" 333 : true 334 ;; 335 *) 336 false 337 ;; 338 esac 339 } 340 341 # Public: Run test commands and expect them to succeed. 342 # 343 # When the test passed, an "ok" message is printed and the number of successful 344 # tests is incremented. When it failed, a "not ok" message is printed and the 345 # number of failed tests is incremented. 346 # 347 # With --immediate, exit test immediately upon the first failed test. 348 # 349 # Usually takes two arguments: 350 # $1 - Test description 351 # $2 - Commands to be executed. 352 # 353 # With three arguments, the first will be taken to be a prerequisite: 354 # $1 - Comma-separated list of test prerequisites. The test will be skipped if 355 # not all of the given prerequisites are set. To negate a prerequisite, 356 # put a "!" in front of it. 357 # $2 - Test description 358 # $3 - Commands to be executed. 359 # 360 # Examples 361 # 362 # test_expect_success \ 363 # 'git-write-tree should be able to write an empty tree.' \ 364 # 'tree=$(git-write-tree)' 365 # 366 # # Test depending on one prerequisite. 367 # test_expect_success TTY 'git --paginate rev-list uses a pager' \ 368 # ' ... ' 369 # 370 # # Multiple prerequisites are separated by a comma. 371 # test_expect_success PERL,PYTHON 'yo dawg' \ 372 # ' test $(perl -E 'print eval "1 +" . qx[python -c "print 2"]') == "4" ' 373 # 374 # Returns nothing. 375 test_expect_success() { 376 test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= 377 test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_success" 378 export test_prereq 379 if ! test_skip_ "$@"; then 380 say >&3 "expecting success: $2" 381 if test_run_ "$2"; then 382 test_ok_ "$1" 383 else 384 test_failure_ "$@" 385 fi 386 fi 387 echo >&3 "" 388 } 389 390 # Public: Run test commands and expect them to fail. Used to demonstrate a known 391 # breakage. 392 # 393 # This is NOT the opposite of test_expect_success, but rather used to mark a 394 # test that demonstrates a known breakage. 395 # 396 # When the test passed, an "ok" message is printed and the number of fixed tests 397 # is incremented. When it failed, a "not ok" message is printed and the number 398 # of tests still broken is incremented. 399 # 400 # Failures from these tests won't cause --immediate to stop. 401 # 402 # Usually takes two arguments: 403 # $1 - Test description 404 # $2 - Commands to be executed. 405 # 406 # With three arguments, the first will be taken to be a prerequisite: 407 # $1 - Comma-separated list of test prerequisites. The test will be skipped if 408 # not all of the given prerequisites are set. To negate a prerequisite, 409 # put a "!" in front of it. 410 # $2 - Test description 411 # $3 - Commands to be executed. 412 # 413 # Returns nothing. 414 test_expect_failure() { 415 test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq= 416 test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_failure" 417 export test_prereq 418 if ! test_skip_ "$@"; then 419 say >&3 "checking known breakage: $2" 420 if test_run_ "$2" expecting_failure; then 421 test_known_broken_ok_ "$1" 422 else 423 test_known_broken_failure_ "$1" 424 fi 425 fi 426 echo >&3 "" 427 } 428 429 # Public: Run command and ensure that it fails in a controlled way. 430 # 431 # Use it instead of "! <command>". For example, when <command> dies due to a 432 # segfault, test_must_fail diagnoses it as an error, while "! <command>" would 433 # mistakenly be treated as just another expected failure. 434 # 435 # This is one of the prefix functions to be used inside test_expect_success or 436 # test_expect_failure. 437 # 438 # $1.. - Command to be executed. 439 # 440 # Examples 441 # 442 # test_expect_success 'complain and die' ' 443 # do something && 444 # do something else && 445 # test_must_fail git checkout ../outerspace 446 # ' 447 # 448 # Returns 1 if the command succeeded (exit code 0). 449 # Returns 1 if the command died by signal (exit codes 130-192) 450 # Returns 1 if the command could not be found (exit code 127). 451 # Returns 0 otherwise. 452 test_must_fail() { 453 "$@" 454 exit_code=$? 455 if test $exit_code = 0; then 456 echo >&2 "test_must_fail: command succeeded: $*" 457 return 1 458 elif test $exit_code -gt 129 -a $exit_code -le 192; then 459 echo >&2 "test_must_fail: died by signal: $*" 460 return 1 461 elif test $exit_code = 127; then 462 echo >&2 "test_must_fail: command not found: $*" 463 return 1 464 fi 465 return 0 466 } 467 468 # Public: Run command and ensure that it succeeds or fails in a controlled way. 469 # 470 # Similar to test_must_fail, but tolerates success too. Use it instead of 471 # "<command> || :" to catch failures caused by a segfault, for instance. 472 # 473 # This is one of the prefix functions to be used inside test_expect_success or 474 # test_expect_failure. 475 # 476 # $1.. - Command to be executed. 477 # 478 # Examples 479 # 480 # test_expect_success 'some command works without configuration' ' 481 # test_might_fail git config --unset all.configuration && 482 # do something 483 # ' 484 # 485 # Returns 1 if the command died by signal (exit codes 130-192) 486 # Returns 1 if the command could not be found (exit code 127). 487 # Returns 0 otherwise. 488 test_might_fail() { 489 "$@" 490 exit_code=$? 491 if test $exit_code -gt 129 -a $exit_code -le 192; then 492 echo >&2 "test_might_fail: died by signal: $*" 493 return 1 494 elif test $exit_code = 127; then 495 echo >&2 "test_might_fail: command not found: $*" 496 return 1 497 fi 498 return 0 499 } 500 501 # Public: Run command and ensure it exits with a given exit code. 502 # 503 # This is one of the prefix functions to be used inside test_expect_success or 504 # test_expect_failure. 505 # 506 # $1 - Expected exit code. 507 # $2.. - Command to be executed. 508 # 509 # Examples 510 # 511 # test_expect_success 'Merge with d/f conflicts' ' 512 # test_expect_code 1 git merge "merge msg" B master 513 # ' 514 # 515 # Returns 0 if the expected exit code is returned or 1 otherwise. 516 test_expect_code() { 517 want_code=$1 518 shift 519 "$@" 520 exit_code=$? 521 if test $exit_code = $want_code; then 522 return 0 523 fi 524 525 echo >&2 "test_expect_code: command exited with $exit_code, we wanted $want_code $*" 526 return 1 527 } 528 529 # Public: Compare two files to see if expected output matches actual output. 530 # 531 # The TEST_CMP variable defines the command used for the comparision; it 532 # defaults to "diff -u". Only when the test script was started with --verbose, 533 # will the command's output, the diff, be printed to the standard output. 534 # 535 # This is one of the prefix functions to be used inside test_expect_success or 536 # test_expect_failure. 537 # 538 # $1 - Path to file with expected output. 539 # $2 - Path to file with actual output. 540 # 541 # Examples 542 # 543 # test_expect_success 'foo works' ' 544 # echo expected >expected && 545 # foo >actual && 546 # test_cmp expected actual 547 # ' 548 # 549 # Returns the exit code of the command set by TEST_CMP. 550 test_cmp() { 551 ${TEST_CMP:-diff -u} "$@" 552 } 553 554 # Public: Schedule cleanup commands to be run unconditionally at the end of a 555 # test. 556 # 557 # If some cleanup command fails, the test will not pass. With --immediate, no 558 # cleanup is done to help diagnose what went wrong. 559 # 560 # This is one of the prefix functions to be used inside test_expect_success or 561 # test_expect_failure. 562 # 563 # $1.. - Commands to prepend to the list of cleanup commands. 564 # 565 # Examples 566 # 567 # test_expect_success 'test core.capslock' ' 568 # git config core.capslock true && 569 # test_when_finished "git config --unset core.capslock" && 570 # do_something 571 # ' 572 # 573 # Returns the exit code of the last cleanup command executed. 574 test_when_finished() { 575 test_cleanup="{ $* 576 } && (exit \"\$eval_ret\"); eval_ret=\$?; $test_cleanup" 577 } 578 579 # Public: Summarize test results and exit with an appropriate error code. 580 # 581 # Must be called at the end of each test script. 582 # 583 # Can also be used to stop tests early and skip all remaining tests. For this, 584 # set skip_all to a string explaining why the tests were skipped before calling 585 # test_done. 586 # 587 # Examples 588 # 589 # # Each test script must call test_done at the end. 590 # test_done 591 # 592 # # Skip all remaining tests if prerequisite is not set. 593 # if ! test_have_prereq PERL; then 594 # skip_all='skipping perl interface tests, perl not available' 595 # test_done 596 # fi 597 # 598 # Returns 0 if all tests passed or 1 if there was a failure. 599 test_done() { 600 EXIT_OK=t 601 602 if test -z "$HARNESS_ACTIVE"; then 603 test_results_dir="$SHARNESS_TEST_DIRECTORY/test-results" 604 mkdir -p "$test_results_dir" 605 test_results_path="$test_results_dir/${SHARNESS_TEST_FILE%.$SHARNESS_TEST_EXTENSION}.$$.counts" 606 607 cat >>"$test_results_path" <<-EOF 608 total $test_count 609 success $test_success 610 fixed $test_fixed 611 broken $test_broken 612 failed $test_failure 613 614 EOF 615 fi 616 617 if test "$test_fixed" != 0; then 618 say_color error "# $test_fixed known breakage(s) vanished; please update test(s)" 619 fi 620 if test "$test_broken" != 0; then 621 say_color warn "# still have $test_broken known breakage(s)" 622 fi 623 if test "$test_broken" != 0 || test "$test_fixed" != 0; then 624 test_remaining=$(( $test_count - $test_broken - $test_fixed )) 625 msg="remaining $test_remaining test(s)" 626 else 627 test_remaining=$test_count 628 msg="$test_count test(s)" 629 fi 630 631 case "$test_failure" in 632 0) 633 # Maybe print SKIP message 634 if test -n "$skip_all" && test $test_count -gt 0; then 635 error "Can't use skip_all after running some tests" 636 fi 637 [ -z "$skip_all" ] || skip_all=" # SKIP $skip_all" 638 639 if test $test_remaining -gt 0; then 640 say_color pass "# passed all $msg" 641 fi 642 say "1..$test_count$skip_all" 643 644 test -d "$remove_trash" && 645 cd "$(dirname "$remove_trash")" && 646 rm -rf "$(basename "$remove_trash")" 647 648 exit 0 ;; 649 650 *) 651 say_color error "# failed $test_failure among $msg" 652 say "1..$test_count" 653 654 exit 1 ;; 655 656 esac 657 } 658 659 # Public: Root directory containing tests. Tests can override this variable, 660 # e.g. for testing Sharness itself. 661 : ${SHARNESS_TEST_DIRECTORY:=$(pwd)} 662 export SHARNESS_TEST_DIRECTORY 663 664 # Public: Build directory that will be added to PATH. By default, it is set to 665 # the parent directory of SHARNESS_TEST_DIRECTORY. 666 : ${SHARNESS_BUILD_DIRECTORY:="$SHARNESS_TEST_DIRECTORY/.."} 667 PATH="$SHARNESS_BUILD_DIRECTORY:$PATH" 668 export PATH SHARNESS_BUILD_DIRECTORY 669 670 # Public: Path to test script currently executed. 671 SHARNESS_TEST_FILE="./$(basename "$0")" 672 export SHARNESS_TEST_FILE 673 674 # Prepare test area. 675 test_dir="trash directory.$(basename "$SHARNESS_TEST_FILE" ".$SHARNESS_TEST_EXTENSION")" 676 test -n "$root" && test_dir="$root/$test_dir" 677 case "$test_dir" in 678 /*) SHARNESS_TRASH_DIRECTORY="$test_dir" ;; 679 *) SHARNESS_TRASH_DIRECTORY="$SHARNESS_TEST_DIRECTORY/$test_dir" ;; 680 esac 681 test "$debug" = "t" || remove_trash="$SHARNESS_TRASH_DIRECTORY" 682 rm -rf "$test_dir" || { 683 EXIT_OK=t 684 echo >&5 "FATAL: Cannot prepare test area" 685 exit 1 686 } 687 688 # Public: Empty trash directory, the test area, provided for each test. The HOME 689 # variable is set to that directory too. 690 export SHARNESS_TRASH_DIRECTORY 691 692 HOME="$SHARNESS_TRASH_DIRECTORY" 693 export HOME 694 695 mkdir -p "$test_dir" || exit 1 696 # Use -P to resolve symlinks in our working directory so that the cwd 697 # in subprocesses like git equals our $PWD (for pathname comparisons). 698 cd -P "$test_dir" || exit 1 699 700 this_test=${SHARNESS_TEST_FILE##*/} 701 this_test=${this_test%.$SHARNESS_TEST_EXTENSION} 702 for skp in $SKIP_TESTS; do 703 case "$this_test" in 704 $skp) 705 say_color info >&3 "skipping test $this_test altogether" 706 skip_all="skip all tests in $this_test" 707 test_done 708 esac 709 done 710 711 # vi: set ts=4 sw=4 noet :