12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175 |
- #!/bin/bash
-
- shellfu import jat
-
- #
- # Help perform series of homogenous tests
- #
- # This module aims to make it easier to build and maintain reasonable
- # coverage for your SUT based on various combinations od input factors.
- #
- # This is opposed to traditional "linear" style, where you repeat
- # (copy/paste) the same code over and over. There are multiple
- # benefits from this design:
- #
- # * far less code necessary,
- #
- # * more flexible (the enumeration function can employ any logic),
- #
- # * "don't run test if setup failed" logic is used, possibly saving
- # lot of resources,
- #
- # * forget about boring formalities such as `jat__pstart*`,
- #
- # * splitting code to functions, enables you to short-cut by
- # `return` keyword (=> clarity AND resource saving),
- #
- # * the test is much easier to understand (if done well).
- #
- #
- # =head1 HELP, MY TEST IS FAILING AND I DON'T UNDERSTAND THIS SORCERY!
- #
- # If you encounter xcase-based test and want to understand what's
- # happening, usually the fastest way to get the grip is:
- #
- # 1. Look at summary table near the end of test log. Alternatively
- # there's also 'xcase-results.yaml' attached to the test result.
- #
- # 2. Identify the case id that is interesing for you. Write it down!
- #
- # 3. Use full-text search to find the case within the test log. There can
- # up to 5 phases executed for the case: dummy (does not do anything),
- # 'setup', 'test', 'diag' and 'cleanup'.
- #
- # 4. Open the test code.
- #
- # For every phase (setup, test, diag and cleanup), there is one function
- # ("handler") named `xcase__test`, etc. (Actually only 'test'
- # is mandatory.)
- #
- # Most handlers will probably refer to output of function
- # xcase__id(), which is case id you have written down in
- # step 2. Sometimes, case ids can have form of variable assignments.
- # In that case handlers may refer to these variables directly.
- #
- # Remember that actual meaning of case id (or its embedded variables) is
- # entirely defined by test author. Should this information be unclear,
- # xcase can't help you, the only remaining option is to bug the test author.
- #
- # PRO TIP: Many xcase tests will be long to run, so interactive debugging
- # (e.g. using 1minutetip) could be quite painful. However, function that
- # controls *what* is executed is `xcase__enum`. You can add
- # arbitrary filter here (grep...) to speed up the loop!
- #
- #
- # =head1 GETTING STARTED
- #
- # Here's what you need to do:
- #
- # 1. Construct a list of cases. These can be simple self-explanatory
- # names or variable lists to represent combinations of various
- # input factors.
- #
- # 2. Implement function `xcase__enum()` that just
- # lists them, one per line.
- #
- # 3. Implement handlers:
- #
- # xcase__setup (optional)
- # xcase__test
- # xcase__diag (optional)
- # xcase__cleanup (optional)
- #
- # Inside these handlers, case id being currently executed can be
- # accessed by function `xcase__id()`.
- #
- # 4. Finally, run a single "magic" function, `xcase__run()`.
- # This will:
- #
- # 1. Call the enumeration function to collect case ids.
- #
- # 2. For each case id, run all implemented handlers (wrap into
- # phases as needed).
- #
- # 3. Add nice report at the end.
- #
- #
- # =head1 EXAMPLE
- #
- # xcase__enum() {
- # echo none
- # echo empty
- # echo small
- # echo normal
- # echo huge
- # }
- #
- # xcase__setup() {
- # jat__cmd mkdir /var/ftp || return 1
- # case $(xcase__id) in
- # small|normal|huge|empty) jat__cmd cp "$(xcase__id)" /var/ftp || return 1;;
- # none) true ;;
- # esac
- # jat__cmd useradd joe || return 1
- # jat__cmd rlServiceRestart hypothetical_ftp || return 1
- # }
- #
- # xcase__test() {
- # local file=$(xcase__id)
- # jat__cmd su -c 'hypo_ftp localhost 25 <<<"get $file"' - joe
- # case $(xcase__id) in
- # small|normal|huge|empty) jat__cmd diff "/var/ftp/$file" "/home/joe/$file" ;;
- # none) jat__cmd -S 1 test -e "/home/joe/$file";;
- # esac
- # }
- #
- # xcase__cleanup() {
- # jat__cmd rm -rf /var/ftp
- # jat__cmd userdel joe
- # jat__cmd rlServiceRestart hypothetical_ftp
- # }
- #
- # shellfu import xcase
- #
- # jat__pstarts
- # # some general setup
- # jat__pend
- #
- # xcase__run
- #
- # jat__pstartc
- # # some general cleanup
- # jat__pend
- #
- # Notice that the same test without xcase would probably require repeating all
- # the setup/test/cleanup code for each of the file size; that would obviously
- # make test harder to understand and extend.
- #
- # Also we were able to make the setup much more responsive to cases when
- # something is terribly wrong: the setup will now stop (causing test
- # to be skipped) if any of the commands fail.
- #
- # For more practical examples, contact author of this module.
- #
- #
- # =head1 ADVANCED USAGE
- #
- #
- # =head2 Variable auto-setting
- #
- # Since version 0.11, xcase can parse and set variables automatically from
- # the case id. For example, if case ids are:
- #
- # size=m,color=red
- # size=l,color=red
- # size=xl,color=red
- # size=m,color=blue
- # ...
- #
- # xcase will, inside each handler, set variables `$size` amnd `$color` to
- # respective value. This means you can directly do things like:
- #
- # xcase__setup() {
- # case $color in
- # red) echo "color=#f00" ;;
- # green) echo "color=#0f0" ;;
- # blue) echo "color=#00f" ;;
- # esac > /etc/foo.conf
- # }
- #
- # aiming for even better readability of code and test logs.
- #
- #
- # =head2 Permutation helpers
- #
- # The color/size example above is pretty small, but if you have
- # more variables, doing permutations like that can become tedious.
- #
- # xcase__permute() and xcase__per() are
- # here to help you generate such matrix instead of writing it
- # yourself or having couple of nested `for` cycles in each of your
- # enumerators.
- #
- # With help of these functions we could rewrite the example as:
- #
- # xcase__enum() {
- # xcase__permute size m l xl \
- # | xcase__per color red green blue
- # }
- #
- # Now, xcase__permute() creates list of name=value
- # pairs, one per line. xcase__per() takes each line
- # and prints 'red', 'green' and 'blue' version of it, appending
- # comma and its own name=value pair.
- #
-
- #
- # Variables treated as arrays
- #
- # This variable can be set to a comma-delimited list of variable
- # names. Each variable named here is then guaranteed to be declared
- # as Bash array. In case ID, this variable can then list its members
- # delimited by '+' (plus sign).
- #
- # For example, if $XCASE__ARRAYS is set to 'foo,bar,baz',
- # then following case id:
- #
- # foo=foo1+foo2,bar=bar1,baz=
- #
- # will set $foo to two-member array, $bar to a single member array, and
- # $baz to an empty array.
- #
- # Note: it's currently not possible to create array with a single item
- # of an empty string, i.e. `foo=("")`. `foo=` means empty array and
- # `foo=+` means array with two empty strings.
- #
- XCASE__ARRAYS=${XCASE__ARRAYS:-}
-
- xcase__id_error() {
- #
- # Raise test failure due to value of $1.. or whole case id
- #
- # Usage:
- # xcase__id_error [VARNAME]..
- #
- # This convenience function can be used when you detect unknown
- # value of xcase__id() or, in variable auto-setting
- # mode, a part of it.
- #
- #
- # For example, instead of:
- #
- # case $(xcase__id) in
- # foo) do_something ;;
- # bar) do_something_else ;;
- # esac
- #
- # always remember to account for unknown id:
- #
- # case $(xcase__id) in
- # foo) do_something ;;
- # bar) do_something_else ;;
- # *) xcase__id_error ;;
- # esac
- #
- # This will ensure that if your xcase__enum() emits
- # an unknown value (perhaps when you added a case but forgot to
- # account for it), the incident will not go unnoticed.
- #
- # In variable auto-setting mode, you can create more useful error
- # message by passing name of the actual variable that got unknown
- # value:
- #
- # xcase__enum() {
- # echo Foo=bar
- # echo Foo=baz
- # echo Foo=quux
- # }
- #
- # # ...later in eg. xcase__setup()
- #
- # case $Foo in
- # bar) do_something ;;
- # baz) do_something_else ;;
- # *) xcase__id_error Foo ;;
- # esac
- #
- # You can actually name more variables if you are not sure which
- # was wrong:
- #
- # case $Foo:$Bar in
- # off:off) something 0 0 ;;
- # on:off) something 0 1 ;;
- # off:on) something 1 0 ;;
- # on:on) something 1 1 ;;
- # *) xcase__id_error Foo Bar ;;
- # esac
- #
- #
- local var
- case $# in
- 0)
- jat__log_error "unhandled case id value: $(xcase__id)"
- ;;
- 1)
- var=$1
- jat__log_error "unhandled case variable: $var=${!var}"
- ;;
- *)
- jat__log_error "unhandled case variable, one of following:"
- for var in "$@";
- do
- jat__log_error " $var='${!var}'"
- done
- ;;
- esac
- jat__log_error "update xcase__enum() or ${FUNCNAME[1]}()"
- }
-
-
- xcase__id() {
- #
- # Print current case ID
- #
- # Inside handler, this function will output current case id as
- # given by enumerator.
- #
- echo "$__xcase__id"
- }
-
- xcase__per() {
- #
- # Permute each line on stdin with variable named $1 at values $2..
- #
- # Usage:
- #
- # xcase__per NAME [--] VALUE1 [VALUE2]..
- # xcase__per NAME -c CMD [ARG]..
- #
- # In first form, take line from stdin (if any), and repeat it to
- # stdout once for each VALUE, adding `,NAME=VALUE` at the end of
- # the line.
- #
- # In second form, do not take VALUEs from arguments but run CMD (witn
- # any ARGs) and use each line of its stdout as VALUE.
- #
- __xcase__doperm per "$@"
- }
-
- xcase__permute() {
- #
- # Permute variable named $1 at values $2..
- #
- # Usage:
- #
- # xcase__permute NAME [--] VALUE1 [VALUE2]..
- # xcase__permute NAME -f CMD [ARG]..
- #
- # In the first form, produce line `NAME=VALUE` for each VALUE.
- #
- # In second form, do not take VALUEs from arguments but run CMD (witn
- # any ARGs) and use each line of its stdout as VALUE.
- #
- __xcase__doperm permute "$@"
- }
-
- xcase__run() {
- #
- # Run all cases from xcase__enum()
- #
- # Usage:
- #
- # xcase__run [-v] [-R] [-T] [-L] [-c path/to/chdir]
- #
- # This function is the main launcher for tests. It will perform roughly
- # following steps:
- #
- # 1. create own temporary directory and chdir there,
- #
- # 2. for each case id emitted from xcase__enum():
- #
- # 1. create a subdirectory of that name, and chdir there,
- #
- # 2. run all available handlers (setup, test, diag, cleanup),
- # wrapped in JAT phases,
- #
- # 3. chdir back
- #
- # 4. Submit relics and reports:
- #
- # * `xcase-relics.tar.gz` with copy of the mentioned temporary
- # directory,
- #
- # * `xcase-results.yaml` with structured results and time stats,
- #
- # * and a nice table in test log
- #
- # See also "GETTING STARTED" and "ADVANCED USAGE" sections.
- #
- # The behavior can be altered using options:
- #
- # '-R' - disable creation and submission of relics tarball.
- #
- # '-c PATH' - change to PATH (must already exist) for the duration
- # of the test (implies '-T').
- #
- # '-L' - disable creation of "leaf" directory for each case id.
- #
- # '-T' - disable migration to temporary directory. xcase will still run
- # in a subdirectory "xcase-relics" of current directory or the
- # directory specified by `-c`.
- #
- # '-v' - enable variable auto-setting (see "ADVANCED USAGE" section).
- #
- local __xcase__leaves=true # enable "leaf" mode?
- local __xcase__tmp # results cache directory
- local __xcase__vars=false # enable variable auto-setting?
- local __xcase__runpath # directory path to run in
- local __xcase__start # start time
- local __xcase__rball=true # collect relics tarball?
- local __xcase__mktemp=true # make own temporary run dir?
- while true; do case $1 in
- -R)
- __xcase__rball=false
- shift
- ;;
- -c)
- __xcase__runpath="$2"
- __xcase__mktemp=false
- shift 2 || {
- jat__log_error "missing value to -c parameter"
- return 2
- }
- test -d "$__xcase__runpath" || {
- jat__log_error "no such directory: $__xcase__runpath"
- return 3
- }
- ;;
- -L)
- __xcase__leaves=false
- shift
- ;;
- -T)
- __xcase__mktemp=false
- shift
- ;;
- -v)
- __xcase__vars=true
- shift
- ;;
- -*)
- jat__log_error "bad argument: '$1'"
- return 2
- ;;
- "")
- break
- ;;
- *)
- jat__log_error "bad argument: '$1'"
- return 2
- ;;
- esac done
- __xcase__tmp=$(mktemp -d -t xcase.meta.XXXXXXXX)
- __xcase__start=$(python -c "import time; print time.time()")
- __xcase__run_all \
- || jat__log_error "errors encountered during case traversal"
- jat__submit "$__xcase__tmp/results.yaml" "xcase-results.yaml"
- $__xcase__rball \
- && jat__submit "$__xcase__tmp/relics.tar.gz" "xcase-relics.tar.gz"
- echo >&2
- rm -rf "$__xcase__tmp"
- }
-
-
- # # #
- # TEMPLATES # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
- # # #
-
- _xcase__enum() {
- #
- # Skeleton for enumerator
- #
- # Body of this function must be implemented by user and it must print
- # out one case ID---word ([a-zA-Z_]*) per line.
- #
- # The case ID is then available inside step handlers as output of
- # function xcase__id()
- #
- echo "case_foo" ...
- }
-
- _xcase__setup() {
- #
- # Skeleton for setup handler
- #
- # Perform setup tasks for case id.
- #
- true
- }
-
- _xcase__test() {
- #
- # Skeleton for test handler
- #
- # Perform tests for case id.
- #
- true
- }
-
- _xcase__diag() {
- #
- # Skeleton for diag handler
- #
- # Perform diag tasks for case id.
- #
- true
- }
-
- _xcase__cleanup() {
- #
- # Skeleton for cleanup handler
- #
- # Perform cleanup tasks for case id.
- #
- true
- }
-
-
- # # risk of brick increases 20% #
- # INTERNAL # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
- # # each time you cross this line #
-
-
- __xcase__has() {
- #
- # Check if handler has been implemented
- #
- type -t "xcase__$1" >/dev/null
- }
-
- __xcase__doperm() {
- #
- # Permutation engine for xcase__per{,mute}
- #
- # This implements both permutation helpers: xcase__per()
- # and xcase__permute(). First parameter says which ('per'
- # or 'permute'), rest is processed as described in those functions'
- # docstrings.
- #
- # The gist is that in 'permute' mode, we just generate key=value pairs,
- # assumed to be "left-most" in the table. In 'per' mode, we generate
- # them but also combuine them with lines on stdin, assumed to be output
- # of one of other modes.
- #
- local mode=$1; shift # mode (permute: first column, per: rest)
- local name=$1; shift # variable name
- local src=args # value source (args: arguments, cmd: command)
- local value # each value
- local line # each line of previous content ('per' mode)
- local oldifs # IFS backup
- local iterfail=false # did iteration command fail?
- case $1 in
- -c) src=cmd; shift ;;
- --) shift ;;
- esac
- test -n "$name" || { __xcase__perm_usage "$mode"; return 2; }
- test -n "$1" || { __xcase__perm_usage "$mode"; return 2; }
- case $src in
- cmd)
- oldifs=$IFS
- IFS=$'\n'
- values=($("$@")) || iterfail=true
- IFS=$oldifs
- $iterfail && {
- jat__log_error "iteration command returned non-zero: $*"
- return 3
- }
- ;;
- args)
- values=("$@")
- ;;
- esac
- case $mode in
- permute)
- for value in "${values[@]}";
- do
- echo "$name=$value"
- done
- ;;
- per)
- while read -r line;
- do
- for value in "${values[@]}";
- do
- echo "$line,$name=$value"
- done
- done
- ;;
- esac
- }
-
- __xcase__perm_usage() {
- #
- # Print usage message for permutation helper
- #
- local self=$1
- jat__log_error \
- "usage: xcase__$self NAME [--] VALUE1 [VALUE2].." \
- "usage: xcase__$self NAME -c CMD [ARG].."
- }
-
- __xcase__plan() {
- #
- # Enumerate and validate cases
- #
- local id # each case id
- local lines=() # output lines
- __xcase__has enum || {
- jat__log_error "case ID enumerator handler not implemented: xcase__enum"
- return 3
- }
- xcase__enum > "$__xcase__tmp/enum"
- __xcase__ttl=$(wc -l <"$__xcase__tmp/enum")
- lines+=("enumerated $__xcase__ttl cases:")
- lines+=("")
- while IFS= read -r id;
- do
- lines+=(" $id")
- done <"$__xcase__tmp/enum"
- lines+=("")
- jat__log_info "${lines[@]}"
- __xcase__validate_enum || return $?
- test "$__xcase__ttl" -eq 0 && {
- jat__log_error "no cases enumerated, nothing to do"
- return 1
- }
- true
- }
-
- __xcase__run_all() {
- #
- # For each case, do all events
- #
- local __xcase__id # each case ID
- local __xcase__hstart # handler start time
- local __xcase__htype # handler type
- local __xcase__n=0 # case number (for hint)
- local __xcase__ttl # total cases (^^)
- local __xcase__varcode # variable auto-setting code
- local __xcase__plan_es=0 # planner (enum+validate) exit status
- local __xcase__sfail # did setup fail?
- local __xcase__tfail # did test fail?
-
- if $__xcase__mktemp;
- then
- __xcase__runpath=$(mktemp -d -t xcase.runpath.XXXXXXXX)
- fi
-
- jat__pstarts "xcase plan"
- __xcase__plan; __xcase__plan_es=$?
- jat__pend
- test $__xcase__plan_es -ne 0 && return $__xcase__plan_es
-
- if test -n "$__xcase__runpath";
- then
- pushd "$__xcase__runpath" >/dev/null || {
- jat__log_error "cannot chdir to: $__xcase__runpath"
- return 3
- }
- fi
-
- mkdir -p "xcase-relics" || {
- jat__log_error "cannot create relics directory: $PWD/xcase-relics"
- return 3
- }
- pushd "xcase-relics" >/dev/null || {
- jat__log_error "cannot chdir to: $PWD/xcase-relics"
- return 3
- }
-
- for __xcase__id in $(<"$__xcase__tmp/enum");
- do
- __xcase__sfail=false
- __xcase__tfail=false
- __xcase__hstart=$(python -c "import time; print time.time()")
-
- if $__xcase__leaves;
- then
- mkdir -p "$__xcase__id"
- pushd "$__xcase__id" >/dev/null || {
- jat__log_error "cannot chdir to: $PWD/$__xcase__id"
- return 3
- }
- fi
-
- ((__xcase__n++))
-
- jat__log_info \
- " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" \
- " /" \
- "o $__xcase__n/$__xcase__ttl: $__xcase__id" \
- " \\" \
- ' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
- echo >&2
-
- __xcase__varcode=$(__xcase__mkvarcode) || {
- jat__log_error "variable parser encountered errors"
- return 3
- }
- bash -n <<<"$__xcase__varcode" || {
- jat__log_error "variable parser created invalid code: $__xcase__varcode"
- return 3
- }
-
- for __xcase__htype in setup test diag cleanup;
- do
- __xcase__wrap_handler
- done
- __xcase__upload_note
-
- if $__xcase__leaves;
- then
- popd >/dev/null
- fi
-
- __xcase__tmpwrite C.duration "$(python -c "import time; print time.time() - $__xcase__hstart")"
-
- done
-
- popd >/dev/null # from "xcase-relics"
-
- if $__xcase__rball;
- then
- tar -czf "$__xcase__tmp/relics.tar.gz" \
- "xcase-relics"
- fi
-
- if test -n "$__xcase__runpath";
- then
- popd >/dev/null
- fi
-
- if $__xcase__mktemp;
- then
- rm -rf "$__xcase__runpath"
- fi
-
- __xcase__results > "$__xcase__tmp/results.yaml"
- __xcase__summary
- }
-
- __xcase__tmpread() {
- #
- # Get result metadata from key $1
- #
- # See __xcase__tmpfile() for key syntax.
- #
- local key=$1 # key to read
- cat "$(__xcase__tmpfile "$key")"
- }
-
- __xcase__tmpwrite() {
- #
- # Save result metadata value $2 under key $1
- #
- # See __xcase__tmpfile() for key syntax.
- #
- local key=$1 # key to write
- local value=$2 # value to write
- local tgt # target file path
- tgt=$(__xcase__tmpfile "$key")
- echo "$value" > "$tgt"
- }
-
- __xcase__tmpfile() {
- #
- # Dereference temp storage key $1
- #
- # The key may be prefixed by `C.` or `H.`, meaning "for current case id"
- # or "for current handler", respectively. For example, following keys are
- # valid:
- #
- # foo # same file at any time
- # C.foo # same file until the end of this subtest (case id)
- # H.foo # same file until the end of this handler (eg. setup)
- #
- # Note: This function has a side effect within the storage structure that
- # directory for the data file is automatically created so that caller does
- # not need to.
- #
- local key=$1 # key to dereference
- local ns_case # case id infix
- local ns_htype # handler type infix
- local path # final path
- ns_case=id/$__xcase__id
- ns_htype=handler/$__xcase__htype
- path=$__xcase__tmp/data/
- case $key in
- C.*) path+="$ns_case/${key#C.}" ;;
- H.*) path+="$ns_case/$ns_htype/${key#H.}" ;;
- *) path+="$key" ;;
- esac
- mkdir -p "${path%/*}"
- echo "$path"
- }
-
- __xcase__nonempty() {
- #
- # True if directory $1 has files
- #
- local dir=$1
- local es
- test -n "$dir" \
- || jat__log_error "usage: __xcase__nonempty DIR"
- test -d "$dir" \
- || jat__log_error "usage: __xcase__nonempty DIR"
- test -e "$dir"/* 2>/dev/null; es=$?
- case $es in
- 0) return 0 ;; # one file
- 2) return 0 ;; # more files (`test` usage error)
- *) return 1 ;; # anything else
- esac
- }
-
- __xcase__upload_note() {
- #
- # Provide note about file uploads (if applicable)
- #
- local items # of filename list
- local item # of filename list
- local lines=() # output lines
- $__xcase__rball || return 0
- __xcase__nonempty . || return 0
- items=$(__xcase__tmpfile "C.relist")
- lines+=("Items to be auto-collected to xcase-relics.tar.gz:")
- find . -mindepth 1 -maxdepth 1 -printf "%P\\n" \
- | sort >"$items"
- while read -r item;
- do
- lines+=(" * $item")
- done <"$items"
- jat__log_info "${lines[@]}"
- echo >&2
- }
-
- __xcase__wrap_handler() {
- #
- # Handler wrapper
- #
- # Set phases, record failures, set up environment...
- #
- local __xcase__hfails="" # this handler fails num.
- local __xcase__hresult=none # this handler result
- __xcase__has "$__xcase__htype" || {
- __xcase__tmpwrite H.result "none"
- return 0
- }
- eval "$__xcase__varcode"
- case $__xcase__htype in
- setup)
- jat__pstarts -c "$__xcase__id" "$__xcase__id :: setup"
- xcase__setup
- ;;
- test)
- jat__pstartt -c "$__xcase__id" "$__xcase__id :: test"
- if $__xcase__sfail;
- then
- jat__log_error "setup failed--skipping test"
- __xcase__tmpwrite H.result "abort"
- jat__pend
- return 1
- else
- xcase__test
- fi
- ;;
- diag)
- jat__pstartd -c "$__xcase__id" "$__xcase__id :: diag"
- xcase__diag
- ;;
- cleanup)
- jat__pstartc -c "$__xcase__id" "$__xcase__id :: cleanup"
- xcase__cleanup
- ;;
- esac
- __xcase__hfails=$(jat__stat pfailc)
- jat__pend
- case $__xcase__hfails in
- 0) __xcase__hresult=pass ;;
- *) __xcase__hresult=fail ;;
- esac
- __xcase__tmpwrite H.result "$__xcase__hresult"
- #shellcheck disable=SC2034
- case $__xcase__htype:$__xcase__hresult in
- setup:fail) __xcase__sfail=true ;;
- test:fail) __xcase__tfail=true ;;
- esac
- }
-
- __xcase__mkvarcode() {
- #
- # Parse $__xcase__id and make variable setting code
- #
- # SIDE EFFECT: write variable cache file for later reference in results
- #
- $__xcase__vars || return 0
- local aname # declaration from $XCASE__ARRAYS
- local ciex # case id expression
- local vnam # variable name
- local vval # variable value
- local vcache # variable cache (for results.yaml)
- vcache="$(__xcase__tmpfile C.vcache)"
- for aname in ${XCASE__ARRAYS//,/ };
- do
- echo "local $aname=()"
- echo "$aname: []" >> "$vcache"
- done
- for ciex in ${__xcase__id//,/ };
- do
- case $ciex in
- *=*)
- vnam=${ciex%%=*}
- vval=${ciex#$vnam=}
- if grep -qw "$vnam" <<<"$XCASE__ARRAYS";
- then # array
- # it's already assigned to empty above, so we don't have
- # to do anything when we see empty value
- if test -n "$vval";
- then
- #shellcheck disable=SC2027,SC2086
- echo "local $vnam=('"${vval//+/"' '"}"')"
- sed -i -e "s/^$vnam:.*/$vnam: ['$vval']/" "$vcache"
- sed -i -e "/^$vnam:/s/+/', '/" "$vcache"
- fi
- else
- echo "local $vnam=$vval"
- echo "$vnam: $vval" >> "$vcache"
- fi
- ;;
- *)
- jat__log_error "invalid assignment in case id: $__xcase__id"
- return 3
- ;;
-
- "")
- jat__log_error "empty item in case id: $__xcase__id"
- ;;
- esac
- done
- }
-
- __xcase__summary() {
- #
- # Show summary phase
- #
- local id # case id
- local r_setup # handler result: setup
- local r_test # ^^ test
- local r_diag # ^^ diag
- local r_cleanup # ^^ cleanup
- local hdata # path to cached handler data
- local num=0 # row number
- local lines=() # output lines
- jat__pstartd "xcase summary"
- lines+=(
- "duration: $(__xcase__tstats)"
- ""
- "============================================="
- "setup test diag cleanup | case id"
- "--------------------------------|------------"
- )
- for id in $(<"$__xcase__tmp/enum");
- do
- ((num++))
- hdata="$__xcase__tmp/data/id/$id/handler"
- r_setup=$(<"$hdata/setup/result")
- r_test=$(<"$hdata/test/result")
- r_diag=$(<"$hdata/diag/result")
- r_cleanup=$(<"$hdata/cleanup/result")
- lines+=("$(
- __xcase__sumrow \
- "$r_setup" "$r_test" "$r_diag" "$r_cleanup" \
- "$num" "$(wc -l < "$__xcase__tmp/enum")" \
- "$id"
- )")
- done
- lines+=("=============================================")
- jat__log_info "${lines[@]}"
- __xcase__stats
- jat__log_info ""
- jat__pend
- }
-
- __xcase__sumrow() {
- #
- # Format one row of summary table
- #
- # Rows should look something like this:
- #
- # ...
- # PASS FAIL none PASS | (8/20) some_case
- # PASS FAIL none PASS | (9/20) some_other_case
- # PASS FAIL none PASS | (10/20) yet_another_one
- # PASS FAIL none PASS | (11/20) still_not_done
- # ...
- #
- # Ie. cells formatted to 8 chars, except the last one, which
- # is free-width and prefixed by a right-aligned order hint.
- #
- local r_setup=$1 # handler result: setup
- local r_test=$2 # ^^ test
- local r_diag=$3 # ^^ diag
- local r_cleanup=$4 # ^^ cleanup
- local n=$5 # row number
- local ttl=$6 # total rows
- local id=$7 # case id
- local hintw # order hint width
- __xcase__sumcell "$r_setup"
- __xcase__sumcell "$r_test"
- __xcase__sumcell "$r_diag"
- __xcase__sumcell "$r_cleanup"
- hintw=$((${#ttl} * 2 + 3))
- printf "| %${hintw}s " "($n/$ttl)"
- echo "$id"
- }
-
- __xcase__sumcell() {
- #
- # Format single value $1, from summary table, maybe yell
- #
- local value=$1 # value as stored in handler result cache
- case $value in
- none) echo -n "none " ;;
- pass) echo -n "PASS " ;;
- fail) echo -n "FAIL " ;;
- abort) echo -n "ABORT " ;;
- *) echo -n "!AXERR! " ;;
- esac
- }
-
- __xcase__tstats() {
- #
- # Time-related stats
- #
- local count # total number of cases
- local dur_ttl # total duration (measured here)
- local dur_one # average case duration
- local alltimes # list of all durations from result cache
- alltimes=$(
- grep -hEo "^[0-9]+\.[0-9]{1,3}" "$__xcase__tmp/data/id"/*/duration \
- | sort -n
- )
- count=$(wc -l <<<"$alltimes")
- dur_ttl=$(python -c "import time; print time.time() - $__xcase__start")
- dur_one=$(python -c "print $dur_ttl/$count")
- __xcase__hduration "$dur_ttl" h
- echo -n " (~"
- __xcase__hduration "$dur_one" m
- echo -n " x $count cases,"
- echo -n " min=$(head -1 <<<"$alltimes"),"
- echo -n " max=$(tail -1 <<<"$alltimes"),"
- echo -n " SD=$(awk '{sum+=$1; sumsq+=$1*$1}END{print sqrt(sumsq/NR - (sum/NR)**2)}'<<<"$alltimes"))"
- }
-
- __xcase__hduration() {
- #
- # Format seconds float $1 as [[hhh:][m]mm]:[s]ss up to unit $2
- #
- # 'h' and 'm' units round seconds to integer value ('1.9' -> '2'),
- # 's' rounds to miliseconds.
- #
- # If unit is 'h', format is 'H:MM:SS'. If unit is 'm', format
- # is 'M:SS'. If unit is 's', format is "S.SSS", i.e. number of
- # seconds with fractional part up to 3 digits (miliseconds).
- #
- local s=$1 # seconds (fractional)
- local unit=$2 # precision unit
- local ws # whole seconds
- local m # minutes
- local h # hours
- ws=$(printf "%.0f" "$s")
- case $unit in
- h) h=$((ws / 3600)); ws=$((ws % 3600))
- m=$((ws / 60)); ws=$((ws % 60))
- printf "%d:%02d:%02d" $h $m $ws
- ;;
- m) m=$((ws / 60)); ws=$((ws % 60))
- printf "%d:%02d" $m $ws
- ;;
- s) printf "%.3f" "$s"
- ;;
- esac
- }
-
- __xcase__stats() {
- #
- # Print the statistics
- #
- local results # all results (as grep -H output)
- local result # result (not all are interesting)
- local setups # setups with ^^
- local tests # tests ^^
- local diags # diags ^^
- local cleanups # cleanups ^^
- local lines=() # output lines
- results=$(find "$__xcase__tmp/data/id/" -path "*/handler/*/result" -exec grep -H . {} +)
- for result in pass fail abort axerr;
- do
- setups=$(grep -c "setup/result:$result$" <<<"$results")
- tests=$(grep -c "test/result:$result$" <<<"$results")
- diags=$(grep -c "diag/result:$result$" <<<"$results")
- cleanups=$(grep -c "cleanup/result:$result$" <<<"$results")
- test "$((setups + tests + diags + cleanups))" -eq 0 && continue
- lines+=("$(printf '%-8s%-8s%-8s%-8s| x %s\n' "$setups" "$tests" "$diags" "$cleanups" $result)")
- done
- jat__log_info "${lines[@]}"
- }
-
- __xcase__results() {
- #
- # Collect case meta-data and print report in YAML
- #
- local idlist # path to cached case id list (straight from user's enum)
- local id # each case id
- idlist=$__xcase__tmp/enum
- echo "---"
- echo "cases: $(wc -l < "$idlist")"
- echo "results:"
- for id in $(<"$idlist");
- do
- pushd "$__xcase__tmp/data/id/$id" >/dev/null
- echo " -"
- echo " id: '$id'"
- test -s "vcache" && {
- echo " variables:"
- sed -e "s/^/ /" "vcache"
- }
- echo " handlers:"
- echo " setup: $(<handler/setup/result)"
- echo " test: $(<handler/test/result)"
- echo " diag: $(<handler/diag/result)"
- echo " cleanup: $(<handler/cleanup/result)"
- echo " duration: $(<duration)"
- echo " result: $(<handler/test/result)"
- popd >/dev/null
- done
- }
-
- __xcase__validate_enum() {
- #
- # Make sure stuff from xcase__enum() has no banned chars
- #
- local allowed='[:alnum:]._,+%=-' # allowed chars in case id
- local es= # exit status of this function
- local lines=() # error message lines
- if grep "[^$allowed]" "$__xcase__tmp/enum";
- then
- if $__xcase__leaves || $__xcase__vars;
- then
- lines+=(
- "Sorry, when leaf directory mode (default) or variable"
- "setting mode is used, range of characters that"
- "xcase__enum() can emit is limited to:"
- ""
- " $allowed"
- ""
- "This is to enable usage of this data as file, directory"
- "and variable names. To disable these modes, use flags"
- "-L and -V, respectively."
- "illegal characters in enumerator"
- )
- es=2
- else
- lines+=("DEPRECATED characters in enumerator, future version will only allow: $allowed")
- es=0
- fi
- lines+=(
- "Note that in order to make best use of xcase, the case id"
- "should not hold any 'real' testing data but rather just"
- "simple generic words to hint *intent* of the test case."
- )
- jat__log_error "${lines[@]}"
- fi
- return $es
- }
-
- #shellfu module-version=__MKIT_PROJ_VERSION__
|