#!/bin/bash shellfu import jat # # Help perform series of homogenous tests # # This module aims to make it easier to build and maintain reasonable # coverage for your SUT based on various combinations od input factors. # # This is opposed to traditional "linear" style, where you repeat # (copy/paste) the same code over and over. There are multiple # benefits from this design: # # * far less code necessary, # # * more flexible (the enumeration function can employ any logic), # # * "don't run test if setup failed" logic is used, possibly saving # lot of resources, # # * forget about boring formalities such as `jat__pstart*`, # # * splitting code to functions, enables you to short-cut by # `return` keyword (=> clarity AND resource saving), # # * the test is much easier to understand (if done well). # # # =head1 HELP, MY TEST IS FAILING AND I DON'T UNDERSTAND THIS SORCERY! # # If you encounter xcase-based test and want to understand what's # happening, usually the fastest way to get the grip is: # # 1. Look at summary table near the end of test log. Alternatively # there's also 'xcase-results.yaml' attached to the test result. # # 2. Identify the case id that is interesing for you. Write it down! # # 3. Use full-text search to find the case within the test log. There can # up to 5 phases executed for the case: dummy (does not do anything), # 'setup', 'test', 'diag' and 'cleanup'. # # 4. Open the test code. # # For every phase (setup, test, diag and cleanup), there is one function # ("handler") named `xcase__test`, etc. (Actually only 'test' # is mandatory.) # # Most handlers will probably refer to output of function # xcase__id(), which is case id you have written down in # step 2. Sometimes, case ids can have form of variable assignments. # In that case handlers may refer to these variables directly. # # Remember that actual meaning of case id (or its embedded variables) is # entirely defined by test author. Should this information be unclear, # xcase can't help you, the only remaining option is to bug the test author. # # PRO TIP: Many xcase tests will be long to run, so interactive debugging # (e.g. using 1minutetip) could be quite painful. However, function that # controls *what* is executed is `xcase__enum`. You can add # arbitrary filter here (grep...) to speed up the loop! # # # =head1 GETTING STARTED # # Here's what you need to do: # # 1. Construct a list of cases. These can be simple self-explanatory # names or variable lists to represent combinations of various # input factors. # # 2. Implement function `xcase__enum()` that just # lists them, one per line. # # 3. Implement handlers: # # xcase__setup (optional) # xcase__test # xcase__diag (optional) # xcase__cleanup (optional) # # Inside these handlers, case id being currently executed can be # accessed by function `xcase__id()`. # # 4. Finally, run a single "magic" function, `xcase__run()`. # This will: # # 1. Call the enumeration function to collect case ids. # # 2. For each case id, run all implemented handlers (wrap into # phases as needed). # # 3. Add nice report at the end. # # # =head1 EXAMPLE # # xcase__enum() { # echo none # echo empty # echo small # echo normal # echo huge # } # # xcase__setup() { # jat__cmd mkdir /var/ftp || return 1 # case $(xcase__id) in # small|normal|huge|empty) jat__cmd cp "$(xcase__id)" /var/ftp || return 1;; # none) true ;; # esac # jat__cmd useradd joe || return 1 # jat__cmd rlServiceRestart hypothetical_ftp || return 1 # } # # xcase__test() { # local file=$(xcase__id) # jat__cmd su -c 'hypo_ftp localhost 25 <<<"get $file"' - joe # case $(xcase__id) in # small|normal|huge|empty) jat__cmd diff "/var/ftp/$file" "/home/joe/$file" ;; # none) jat__cmd -S 1 test -e "/home/joe/$file";; # esac # } # # xcase__cleanup() { # jat__cmd rm -rf /var/ftp # jat__cmd userdel joe # jat__cmd rlServiceRestart hypothetical_ftp # } # # shellfu import xcase # # jat__pstarts # # some general setup # jat__pend # # xcase__run # # jat__pstartc # # some general cleanup # jat__pend # # Notice that the same test without xcase would probably require repeating all # the setup/test/cleanup code for each of the file size; that would obviously # make test harder to understand and extend. # # Also we were able to make the setup much more responsive to cases when # something is terribly wrong: the setup will now stop (causing test # to be skipped) if any of the commands fail. # # For more practical examples, contact author of this module. # # # =head1 ADVANCED USAGE # # # =head2 Variable auto-setting # # Since version 0.11, xcase can parse and set variables automatically from # the case id. For example, if case ids are: # # size=m,color=red # size=l,color=red # size=xl,color=red # size=m,color=blue # ... # # xcase will, inside each handler, set variables `$size` amnd `$color` to # respective value. This means you can directly do things like: # # xcase__setup() { # case $color in # red) echo "color=#f00" ;; # green) echo "color=#0f0" ;; # blue) echo "color=#00f" ;; # esac > /etc/foo.conf # } # # aiming for even better readability of code and test logs. # # # =head2 Permutation helpers # # The color/size example above is pretty small, but if you have # more variables, doing permutations like that can become tedious. # # xcase__permute() and xcase__per() are # here to help you generate such matrix instead of writing it # yourself or having couple of nested `for` cycles in each of your # enumerators. # # With help of these functions we could rewrite the example as: # # xcase__enum() { # xcase__permute size m l xl \ # | xcase__per color red green blue # } # # Now, xcase__permute() creates list of name=value # pairs, one per line. xcase__per() takes each line # and prints 'red', 'green' and 'blue' version of it, appending # comma and its own name=value pair. # # # Variables treated as arrays # # This variable can be set to a comma-delimited list of variable # names. Each variable named here is then guaranteed to be declared # as Bash array. In case ID, this variable can then list its members # delimited by '+' (plus sign). # # For example, if $XCASE__ARRAYS is set to 'foo,bar,baz', # then following case id: # # foo=foo1+foo2,bar=bar1,baz= # # will set $foo to two-member array, $bar to a single member array, and # $baz to an empty array. # # Note: it's currently not possible to create array with a single item # of an empty string, i.e. `foo=("")`. `foo=` means empty array and # `foo=+` means array with two empty strings. # XCASE__ARRAYS=${XCASE__ARRAYS:-} xcase__id_error() { # # Raise test failure due to value of $1.. or whole case id # # Usage: # xcase__id_error [VARNAME].. # # This convenience function can be used when you detect unknown # value of xcase__id() or, in variable auto-setting # mode, a part of it. # # # For example, instead of: # # case $(xcase__id) in # foo) do_something ;; # bar) do_something_else ;; # esac # # always remember to account for unknown id: # # case $(xcase__id) in # foo) do_something ;; # bar) do_something_else ;; # *) xcase__id_error ;; # esac # # This will ensure that if your xcase__enum() emits # an unknown value (perhaps when you added a case but forgot to # account for it), the incident will not go unnoticed. # # In variable auto-setting mode, you can create more useful error # message by passing name of the actual variable that got unknown # value: # # xcase__enum() { # echo Foo=bar # echo Foo=baz # echo Foo=quux # } # # # ...later in eg. xcase__setup() # # case $Foo in # bar) do_something ;; # baz) do_something_else ;; # *) xcase__id_error Foo ;; # esac # # You can actually name more variables if you are not sure which # was wrong: # # case $Foo:$Bar in # off:off) something 0 0 ;; # on:off) something 0 1 ;; # off:on) something 1 0 ;; # on:on) something 1 1 ;; # *) xcase__id_error Foo Bar ;; # esac # # local var case $# in 0) jat__log_error "unhandled case id value: $(xcase__id)" ;; 1) var=$1 jat__log_error "unhandled case variable: $var=${!var}" ;; *) jat__log_error "unhandled case variable, one of following:" for var in "$@"; do jat__log_error " $var='${!var}'" done ;; esac jat__log_error "update xcase__enum() or ${FUNCNAME[1]}()" } xcase__id() { # # Print current case ID # # Inside handler, this function will output current case id as # given by enumerator. # echo "$__xcase__id" } xcase__per() { # # Permute each line on stdin with variable named $1 at values $2.. # # Usage: # # xcase__per NAME [--] VALUE1 [VALUE2].. # xcase__per NAME -c CMD [ARG].. # # In first form, take line from stdin (if any), and repeat it to # stdout once for each VALUE, adding `,NAME=VALUE` at the end of # the line. # # In second form, do not take VALUEs from arguments but run CMD (witn # any ARGs) and use each line of its stdout as VALUE. # __xcase__doperm per "$@" } xcase__permute() { # # Permute variable named $1 at values $2.. # # Usage: # # xcase__permute NAME [--] VALUE1 [VALUE2].. # xcase__permute NAME -f CMD [ARG].. # # In the first form, produce line `NAME=VALUE` for each VALUE. # # In second form, do not take VALUEs from arguments but run CMD (witn # any ARGs) and use each line of its stdout as VALUE. # __xcase__doperm permute "$@" } xcase__run() { # # Run all cases from xcase__enum() # # Usage: # # xcase__run [-v] [-R] [-T] [-L] [-c path/to/chdir] # # This function is the main launcher for tests. It will perform roughly # following steps: # # 1. create own temporary directory and chdir there, # # 2. for each case id emitted from xcase__enum(): # # 1. create a subdirectory of that name, and chdir there, # # 2. run all available handlers (setup, test, diag, cleanup), # wrapped in JAT phases, # # 3. chdir back # # 4. Submit relics and reports: # # * `xcase-relics.tar.gz` with copy of the mentioned temporary # directory, # # * `xcase-results.yaml` with structured results and time stats, # # * and a nice table in test log # # See also "GETTING STARTED" and "ADVANCED USAGE" sections. # # The behavior can be altered using options: # # '-R' - disable creation and submission of relics tarball. # # '-c PATH' - change to PATH (must already exist) for the duration # of the test (implies '-T'). # # '-L' - disable creation of "leaf" directory for each case id. # # '-T' - disable migration to temporary directory. xcase will still run # in a subdirectory "xcase-relics" of current directory or the # directory specified by `-c`. # # '-v' - enable variable auto-setting (see "ADVANCED USAGE" section). # local __xcase__leaves=true # enable "leaf" mode? local __xcase__tmp # results cache directory local __xcase__vars=false # enable variable auto-setting? local __xcase__runpath # directory path to run in local __xcase__start # start time local __xcase__rball=true # collect relics tarball? local __xcase__mktemp=true # make own temporary run dir? while true; do case $1 in -R) __xcase__rball=false shift ;; -c) __xcase__runpath="$2" __xcase__mktemp=false shift 2 || { jat__log_error "missing value to -c parameter" return 2 } test -d "$__xcase__runpath" || { jat__log_error "no such directory: $__xcase__runpath" return 3 } ;; -L) __xcase__leaves=false shift ;; -T) __xcase__mktemp=false shift ;; -v) __xcase__vars=true shift ;; -*) jat__log_error "bad argument: '$1'" return 2 ;; "") break ;; *) jat__log_error "bad argument: '$1'" return 2 ;; esac done __xcase__tmp=$(mktemp -d -t xcase.meta.XXXXXXXX) __xcase__start=$(python -c "import time; print time.time()") __xcase__run_all \ || jat__log_error "errors encountered during case traversal" jat__submit "$__xcase__tmp/results.yaml" "xcase-results.yaml" $__xcase__rball \ && jat__submit "$__xcase__tmp/relics.tar.gz" "xcase-relics.tar.gz" echo >&2 rm -rf "$__xcase__tmp" } # # # # TEMPLATES # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # # # _xcase__enum() { # # Skeleton for enumerator # # Body of this function must be implemented by user and it must print # out one case ID---word ([a-zA-Z_]*) per line. # # The case ID is then available inside step handlers as output of # function xcase__id() # echo "case_foo" ... } _xcase__setup() { # # Skeleton for setup handler # # Perform setup tasks for case id. # true } _xcase__test() { # # Skeleton for test handler # # Perform tests for case id. # true } _xcase__diag() { # # Skeleton for diag handler # # Perform diag tasks for case id. # true } _xcase__cleanup() { # # Skeleton for cleanup handler # # Perform cleanup tasks for case id. # true } # # risk of brick increases 20% # # INTERNAL # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # # each time you cross this line # __xcase__has() { # # Check if handler has been implemented # type -t "xcase__$1" >/dev/null } __xcase__doperm() { # # Permutation engine for xcase__per{,mute} # # This implements both permutation helpers: xcase__per() # and xcase__permute(). First parameter says which ('per' # or 'permute'), rest is processed as described in those functions' # docstrings. # # The gist is that in 'permute' mode, we just generate key=value pairs, # assumed to be "left-most" in the table. In 'per' mode, we generate # them but also combuine them with lines on stdin, assumed to be output # of one of other modes. # local mode=$1; shift # mode (permute: first column, per: rest) local name=$1; shift # variable name local src=args # value source (args: arguments, cmd: command) local value # each value local line # each line of previous content ('per' mode) local oldifs # IFS backup local iterfail=false # did iteration command fail? case $1 in -c) src=cmd; shift ;; --) shift ;; esac test -n "$name" || { __xcase__perm_usage "$mode"; return 2; } test -n "$1" || { __xcase__perm_usage "$mode"; return 2; } case $src in cmd) oldifs=$IFS IFS=$'\n' values=($("$@")) || iterfail=true IFS=$oldifs $iterfail && { jat__log_error "iteration command returned non-zero: $*" return 3 } ;; args) values=("$@") ;; esac case $mode in permute) for value in "${values[@]}"; do echo "$name=$value" done ;; per) while read -r line; do for value in "${values[@]}"; do echo "$line,$name=$value" done done ;; esac } __xcase__perm_usage() { # # Print usage message for permutation helper # local self=$1 jat__log_error \ "usage: xcase__$self NAME [--] VALUE1 [VALUE2].." \ "usage: xcase__$self NAME -c CMD [ARG].." } __xcase__plan() { # # Enumerate and validate cases # local id # each case id local lines=() # output lines __xcase__has enum || { jat__log_error "case ID enumerator handler not implemented: xcase__enum" return 3 } xcase__enum > "$__xcase__tmp/enum" __xcase__ttl=$(wc -l <"$__xcase__tmp/enum") lines+=("enumerated $__xcase__ttl cases:") lines+=("") while IFS= read -r id; do lines+=(" $id") done <"$__xcase__tmp/enum" lines+=("") jat__log_info "${lines[@]}" __xcase__validate_enum || return $? test "$__xcase__ttl" -eq 0 && { jat__log_error "no cases enumerated, nothing to do" return 1 } true } __xcase__run_all() { # # For each case, do all events # local __xcase__id # each case ID local __xcase__hstart # handler start time local __xcase__htype # handler type local __xcase__n=0 # case number (for hint) local __xcase__ttl # total cases (^^) local __xcase__varcode # variable auto-setting code local __xcase__plan_es=0 # planner (enum+validate) exit status local __xcase__sfail # did setup fail? local __xcase__tfail # did test fail? if $__xcase__mktemp; then __xcase__runpath=$(mktemp -d -t xcase.runpath.XXXXXXXX) fi jat__pstarts "xcase plan" __xcase__plan; __xcase__plan_es=$? jat__pend test $__xcase__plan_es -ne 0 && return $__xcase__plan_es if test -n "$__xcase__runpath"; then pushd "$__xcase__runpath" >/dev/null || { jat__log_error "cannot chdir to: $__xcase__runpath" return 3 } fi mkdir -p "xcase-relics" || { jat__log_error "cannot create relics directory: $PWD/xcase-relics" return 3 } pushd "xcase-relics" >/dev/null || { jat__log_error "cannot chdir to: $PWD/xcase-relics" return 3 } for __xcase__id in $(<"$__xcase__tmp/enum"); do __xcase__sfail=false __xcase__tfail=false __xcase__hstart=$(python -c "import time; print time.time()") if $__xcase__leaves; then mkdir -p "$__xcase__id" pushd "$__xcase__id" >/dev/null || { jat__log_error "cannot chdir to: $PWD/$__xcase__id" return 3 } fi ((__xcase__n++)) jat__log_info \ " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" \ " /" \ "o $__xcase__n/$__xcase__ttl: $__xcase__id" \ " \\" \ ' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' echo >&2 __xcase__varcode=$(__xcase__mkvarcode) || { jat__log_error "variable parser encountered errors" return 3 } bash -n <<<"$__xcase__varcode" || { jat__log_error "variable parser created invalid code: $__xcase__varcode" return 3 } for __xcase__htype in setup test diag cleanup; do __xcase__wrap_handler done __xcase__upload_note if $__xcase__leaves; then popd >/dev/null fi __xcase__tmpwrite C.duration "$(python -c "import time; print time.time() - $__xcase__hstart")" done popd >/dev/null # from "xcase-relics" if $__xcase__rball; then tar -czf "$__xcase__tmp/relics.tar.gz" \ "xcase-relics" fi if test -n "$__xcase__runpath"; then popd >/dev/null fi if $__xcase__mktemp; then rm -rf "$__xcase__runpath" fi __xcase__results > "$__xcase__tmp/results.yaml" __xcase__summary } __xcase__tmpread() { # # Get result metadata from key $1 # # See __xcase__tmpfile() for key syntax. # local key=$1 # key to read cat "$(__xcase__tmpfile "$key")" } __xcase__tmpwrite() { # # Save result metadata value $2 under key $1 # # See __xcase__tmpfile() for key syntax. # local key=$1 # key to write local value=$2 # value to write local tgt # target file path tgt=$(__xcase__tmpfile "$key") echo "$value" > "$tgt" } __xcase__tmpfile() { # # Dereference temp storage key $1 # # The key may be prefixed by `C.` or `H.`, meaning "for current case id" # or "for current handler", respectively. For example, following keys are # valid: # # foo # same file at any time # C.foo # same file until the end of this subtest (case id) # H.foo # same file until the end of this handler (eg. setup) # # Note: This function has a side effect within the storage structure that # directory for the data file is automatically created so that caller does # not need to. # local key=$1 # key to dereference local ns_case # case id infix local ns_htype # handler type infix local path # final path ns_case=id/$__xcase__id ns_htype=handler/$__xcase__htype path=$__xcase__tmp/data/ case $key in C.*) path+="$ns_case/${key#C.}" ;; H.*) path+="$ns_case/$ns_htype/${key#H.}" ;; *) path+="$key" ;; esac mkdir -p "${path%/*}" echo "$path" } __xcase__nonempty() { # # True if directory $1 has files # local dir=$1 local es test -n "$dir" \ || jat__log_error "usage: __xcase__nonempty DIR" test -d "$dir" \ || jat__log_error "usage: __xcase__nonempty DIR" test -e "$dir"/* 2>/dev/null; es=$? case $es in 0) return 0 ;; # one file 2) return 0 ;; # more files (`test` usage error) *) return 1 ;; # anything else esac } __xcase__upload_note() { # # Provide note about file uploads (if applicable) # local items # of filename list local item # of filename list local lines=() # output lines $__xcase__rball || return 0 __xcase__nonempty . || return 0 items=$(__xcase__tmpfile "C.relist") lines+=("Items to be auto-collected to xcase-relics.tar.gz:") find . -mindepth 1 -maxdepth 1 -printf "%P\\n" \ | sort >"$items" while read -r item; do lines+=(" * $item") done <"$items" jat__log_info "${lines[@]}" echo >&2 } __xcase__wrap_handler() { # # Handler wrapper # # Set phases, record failures, set up environment... # local __xcase__hfails="" # this handler fails num. local __xcase__hresult=none # this handler result __xcase__has "$__xcase__htype" || { __xcase__tmpwrite H.result "none" return 0 } eval "$__xcase__varcode" case $__xcase__htype in setup) jat__pstarts -c "$__xcase__id" "$__xcase__id :: setup" xcase__setup ;; test) jat__pstartt -c "$__xcase__id" "$__xcase__id :: test" if $__xcase__sfail; then jat__log_error "setup failed--skipping test" __xcase__tmpwrite H.result "abort" jat__pend return 1 else xcase__test fi ;; diag) jat__pstartd -c "$__xcase__id" "$__xcase__id :: diag" xcase__diag ;; cleanup) jat__pstartc -c "$__xcase__id" "$__xcase__id :: cleanup" xcase__cleanup ;; esac __xcase__hfails=$(jat__stat pfailc) jat__pend case $__xcase__hfails in 0) __xcase__hresult=pass ;; *) __xcase__hresult=fail ;; esac __xcase__tmpwrite H.result "$__xcase__hresult" #shellcheck disable=SC2034 case $__xcase__htype:$__xcase__hresult in setup:fail) __xcase__sfail=true ;; test:fail) __xcase__tfail=true ;; esac } __xcase__mkvarcode() { # # Parse $__xcase__id and make variable setting code # # SIDE EFFECT: write variable cache file for later reference in results # $__xcase__vars || return 0 local aname # declaration from $XCASE__ARRAYS local ciex # case id expression local vnam # variable name local vval # variable value local vcache # variable cache (for results.yaml) vcache="$(__xcase__tmpfile C.vcache)" for aname in ${XCASE__ARRAYS//,/ }; do echo "local $aname=()" echo "$aname: []" >> "$vcache" done for ciex in ${__xcase__id//,/ }; do case $ciex in *=*) vnam=${ciex%%=*} vval=${ciex#$vnam=} if grep -qw "$vnam" <<<"$XCASE__ARRAYS"; then # array # it's already assigned to empty above, so we don't have # to do anything when we see empty value if test -n "$vval"; then #shellcheck disable=SC2027,SC2086 echo "local $vnam=('"${vval//+/"' '"}"')" sed -i -e "s/^$vnam:.*/$vnam: ['$vval']/" "$vcache" sed -i -e "/^$vnam:/s/+/', '/" "$vcache" fi else echo "local $vnam=$vval" echo "$vnam: $vval" >> "$vcache" fi ;; *) jat__log_error "invalid assignment in case id: $__xcase__id" return 3 ;; "") jat__log_error "empty item in case id: $__xcase__id" ;; esac done } __xcase__summary() { # # Show summary phase # local id # case id local r_setup # handler result: setup local r_test # ^^ test local r_diag # ^^ diag local r_cleanup # ^^ cleanup local hdata # path to cached handler data local num=0 # row number local lines=() # output lines jat__pstartd "xcase summary" lines+=( "duration: $(__xcase__tstats)" "" "=============================================" "setup test diag cleanup | case id" "--------------------------------|------------" ) for id in $(<"$__xcase__tmp/enum"); do ((num++)) hdata="$__xcase__tmp/data/id/$id/handler" r_setup=$(<"$hdata/setup/result") r_test=$(<"$hdata/test/result") r_diag=$(<"$hdata/diag/result") r_cleanup=$(<"$hdata/cleanup/result") lines+=("$( __xcase__sumrow \ "$r_setup" "$r_test" "$r_diag" "$r_cleanup" \ "$num" "$(wc -l < "$__xcase__tmp/enum")" \ "$id" )") done lines+=("=============================================") jat__log_info "${lines[@]}" __xcase__stats jat__log_info "" jat__pend } __xcase__sumrow() { # # Format one row of summary table # # Rows should look something like this: # # ... # PASS FAIL none PASS | (8/20) some_case # PASS FAIL none PASS | (9/20) some_other_case # PASS FAIL none PASS | (10/20) yet_another_one # PASS FAIL none PASS | (11/20) still_not_done # ... # # Ie. cells formatted to 8 chars, except the last one, which # is free-width and prefixed by a right-aligned order hint. # local r_setup=$1 # handler result: setup local r_test=$2 # ^^ test local r_diag=$3 # ^^ diag local r_cleanup=$4 # ^^ cleanup local n=$5 # row number local ttl=$6 # total rows local id=$7 # case id local hintw # order hint width __xcase__sumcell "$r_setup" __xcase__sumcell "$r_test" __xcase__sumcell "$r_diag" __xcase__sumcell "$r_cleanup" hintw=$((${#ttl} * 2 + 3)) printf "| %${hintw}s " "($n/$ttl)" echo "$id" } __xcase__sumcell() { # # Format single value $1, from summary table, maybe yell # local value=$1 # value as stored in handler result cache case $value in none) echo -n "none " ;; pass) echo -n "PASS " ;; fail) echo -n "FAIL " ;; abort) echo -n "ABORT " ;; *) echo -n "!AXERR! " ;; esac } __xcase__tstats() { # # Time-related stats # local count # total number of cases local dur_ttl # total duration (measured here) local dur_one # average case duration local alltimes # list of all durations from result cache alltimes=$( grep -hEo "^[0-9]+\.[0-9]{1,3}" "$__xcase__tmp/data/id"/*/duration \ | sort -n ) count=$(wc -l <<<"$alltimes") dur_ttl=$(python -c "import time; print time.time() - $__xcase__start") dur_one=$(python -c "print $dur_ttl/$count") __xcase__hduration "$dur_ttl" h echo -n " (~" __xcase__hduration "$dur_one" m echo -n " x $count cases," echo -n " min=$(head -1 <<<"$alltimes")," echo -n " max=$(tail -1 <<<"$alltimes")," echo -n " SD=$(awk '{sum+=$1; sumsq+=$1*$1}END{print sqrt(sumsq/NR - (sum/NR)**2)}'<<<"$alltimes"))" } __xcase__hduration() { # # Format seconds float $1 as [[hhh:][m]mm]:[s]ss up to unit $2 # # 'h' and 'm' units round seconds to integer value ('1.9' -> '2'), # 's' rounds to miliseconds. # # If unit is 'h', format is 'H:MM:SS'. If unit is 'm', format # is 'M:SS'. If unit is 's', format is "S.SSS", i.e. number of # seconds with fractional part up to 3 digits (miliseconds). # local s=$1 # seconds (fractional) local unit=$2 # precision unit local ws # whole seconds local m # minutes local h # hours ws=$(printf "%.0f" "$s") case $unit in h) h=$((ws / 3600)); ws=$((ws % 3600)) m=$((ws / 60)); ws=$((ws % 60)) printf "%d:%02d:%02d" $h $m $ws ;; m) m=$((ws / 60)); ws=$((ws % 60)) printf "%d:%02d" $m $ws ;; s) printf "%.3f" "$s" ;; esac } __xcase__stats() { # # Print the statistics # local results # all results (as grep -H output) local result # result (not all are interesting) local setups # setups with ^^ local tests # tests ^^ local diags # diags ^^ local cleanups # cleanups ^^ local lines=() # output lines results=$(find "$__xcase__tmp/data/id/" -path "*/handler/*/result" -exec grep -H . {} +) for result in pass fail abort axerr; do setups=$(grep -c "setup/result:$result$" <<<"$results") tests=$(grep -c "test/result:$result$" <<<"$results") diags=$(grep -c "diag/result:$result$" <<<"$results") cleanups=$(grep -c "cleanup/result:$result$" <<<"$results") test "$((setups + tests + diags + cleanups))" -eq 0 && continue lines+=("$(printf '%-8s%-8s%-8s%-8s| x %s\n' "$setups" "$tests" "$diags" "$cleanups" $result)") done jat__log_info "${lines[@]}" } __xcase__results() { # # Collect case meta-data and print report in YAML # local idlist # path to cached case id list (straight from user's enum) local id # each case id idlist=$__xcase__tmp/enum echo "---" echo "cases: $(wc -l < "$idlist")" echo "results:" for id in $(<"$idlist"); do pushd "$__xcase__tmp/data/id/$id" >/dev/null echo " -" echo " id: '$id'" test -s "vcache" && { echo " variables:" sed -e "s/^/ /" "vcache" } echo " handlers:" echo " setup: $(/dev/null done } __xcase__validate_enum() { # # Make sure stuff from xcase__enum() has no banned chars # local allowed='[:alnum:]._,+%=-' # allowed chars in case id local es= # exit status of this function local lines=() # error message lines if grep "[^$allowed]" "$__xcase__tmp/enum"; then if $__xcase__leaves || $__xcase__vars; then lines+=( "Sorry, when leaf directory mode (default) or variable" "setting mode is used, range of characters that" "xcase__enum() can emit is limited to:" "" " $allowed" "" "This is to enable usage of this data as file, directory" "and variable names. To disable these modes, use flags" "-L and -V, respectively." "illegal characters in enumerator" ) es=2 else lines+=("DEPRECATED characters in enumerator, future version will only allow: $allowed") es=0 fi lines+=( "Note that in order to make best use of xcase, the case id" "should not hold any 'real' testing data but rather just" "simple generic words to hint *intent* of the test case." ) jat__log_error "${lines[@]}" fi return $es } #shellfu module-version=__MKIT_PROJ_VERSION__