123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164 |
- #!/bin/bash
-
- shellfu import jat
-
- #
- # Help perform series of homogenous tests
- #
- # This module aims to make it easier to build and maintain reasonable
- # coverage for your SUT based on various combinations od input factors.
- #
- # This is opposed to traditional "linear" style, where you repeat
- # (copy/paste) the same code over and over. There are multiple
- # benefits from this design:
- #
- # * far less code necessary,
- #
- # * more flexible (the enumeration function can employ any logic),
- #
- # * "don't run test if setup failed" logic is used, possibly saving
- # lot of resources,
- #
- # * forget about boring formalities such as `jat__pstart*`,
- #
- # * splitting code to functions, enables you to short-cut by
- # `return` keyword (=> clarity AND resource saving),
- #
- # * the test is much easier to understand (if done well).
- #
- #
- # =head1 HELP, MY TEST IS FAILING AND I DON'T UNDERSTAND THIS SORCERY!
- #
- # If you encounter xcase-based test and want to understand what's
- # happening, usually the fastest way to get the grip is:
- #
- # 1. Look at summary table near the end of test log. Alternatively
- # there's also 'xcase-results.yaml' attached to the test result.
- #
- # 2. Identify the case id that is interesing for you. Write it down!
- #
- # 3. Use full-text search to find the case within the test log. There can
- # up to 5 phases executed for the case: dummy (does not do anything),
- # 'setup', 'test', 'diag' and 'cleanup'.
- #
- # 4. Open the test code.
- #
- # For every phase (setup, test, diag and cleanup), there is one function
- # ("handler") named `xcase__test`, etc. (Actually only 'test'
- # is mandatory.)
- #
- # Most handlers will probably refer to output of function
- # xcase__id(), which is case id you have written down in
- # step 2. Sometimes, case ids can have form of variable assignments.
- # In that case handlers may refer to these variables directly.
- #
- # Remember that actual meaning of case id (or its embedded variables) is
- # entirely defined by test author. Should this information be unclear,
- # xcase can't help you, the only remaining option is to bug the test author.
- #
- # PRO TIP: Many xcase tests will be long to run, so interactive debugging
- # (e.g. using 1minutetip) could be quite painful. However, function that
- # controls *what* is executed is `xcase__enum`. You can add
- # arbitrary filter here (grep...) to speed up the loop!
- #
- #
- # =head1 GETTING STARTED
- #
- # Here's what you need to do:
- #
- # 1. Construct a list of cases. These can be simple self-explanatory
- # names or variable lists to represent combinations of various
- # input factors.
- #
- # 2. Implement function `xcase__enum()` that just
- # lists them, one per line.
- #
- # 3. Implement handlers:
- #
- # xcase__setup (optional)
- # xcase__test
- # xcase__diag (optional)
- # xcase__cleanup (optional)
- #
- # Inside these handlers, case id being currently executed can be
- # accessed by function `xcase__id()`.
- #
- # 4. Finally, run a single "magic" function, `xcase__run()`.
- # This will:
- #
- # 1. Call the enumeration function to collect case ids.
- #
- # 2. For each case id, run all implemented handlers (wrap into
- # phases as needed).
- #
- # 3. Add nice report at the end.
- #
- #
- # =head1 EXAMPLE
- #
- # xcase__enum() {
- # echo none
- # echo empty
- # echo small
- # echo normal
- # echo huge
- # }
- #
- # xcase__setup() {
- # jat__cmd mkdir /var/ftp || return 1
- # case $(xcase__id) in
- # small|normal|huge|empty) jat__cmd cp "$(xcase__id)" /var/ftp || return 1;;
- # none) true ;;
- # esac
- # jat__cmd useradd joe || return 1
- # jat__cmd rlServiceRestart hypothetical_ftp || return 1
- # }
- #
- # xcase__test() {
- # local file=$(xcase__id)
- # jat__cmd su -c 'hypo_ftp localhost 25 <<<"get $file"' - joe
- # case $(xcase__id) in
- # small|normal|huge|empty) jat__cmd diff "/var/ftp/$file" "/home/joe/$file" ;;
- # none) jat__cmd -S 1 test -e "/home/joe/$file";;
- # esac
- # }
- #
- # xcase__cleanup() {
- # jat__cmd rm -rf /var/ftp
- # jat__cmd userdel joe
- # jat__cmd rlServiceRestart hypothetical_ftp
- # }
- #
- # shellfu import xcase
- #
- # jat__pstarts
- # # some general setup
- # jat__pend
- #
- # xcase__run
- #
- # jat__pstartc
- # # some general cleanup
- # jat__pend
- #
- # Notice that the same test without xcase would probably require repeating all
- # the setup/test/cleanup code for each of the file size; that would obviously
- # make test harder to understand and extend.
- #
- # Also we were able to make the setup much more responsive to cases when
- # something is terribly wrong: the setup will now stop (causing test
- # to be skipped) if any of the commands fail.
- #
- # For more practical examples, contact author of this module.
- #
- #
- # =head1 ADVANCED USAGE
- #
- #
- # =head2 Variable auto-setting
- #
- # Since version 0.11, xcase can parse and set variables automatically from
- # the case id. For example, if case ids are:
- #
- # size=m,color=red
- # size=l,color=red
- # size=xl,color=red
- # size=m,color=blue
- # ...
- #
- # xcase will, inside each handler, set variables `$size` amnd `$color` to
- # respective value. This means you can directly do things like:
- #
- # xcase__setup() {
- # case $color in
- # red) echo "color=#f00" ;;
- # green) echo "color=#0f0" ;;
- # blue) echo "color=#00f" ;;
- # esac > /etc/foo.conf
- # }
- #
- # aiming for even better readability of code and test logs.
- #
- #
- # =head2 Permutation helpers
- #
- # The color/size example above is pretty small, but if you have
- # more variables, doing permutations like that can become tedious.
- #
- # xcase__permute() and xcase__per() are
- # here to help you generate such matrix instead of writing it
- # yourself or having couple of nested `for` cycles in each of your
- # enumerators.
- #
- # With help of these functions we could rewrite the example as:
- #
- # xcase__enum() {
- # xcase__permute size m l xl \
- # | xcase__per color red green blue
- # }
- #
- # Now, xcase__permute() creates list of name=value
- # pairs, one per line. xcase__per() takes each line
- # and prints 'red', 'green' and 'blue' version of it, appending
- # comma and its own name=value pair.
- #
-
- #
- # Variables treated as arrays
- #
- # This variable can be set to a comma-delimited list of variable
- # names. Each variable named here is then guaranteed to be declared
- # as Bash array. In case ID, this variable can then list its members
- # delimited by '+' (plus sign).
- #
- # For example, if $XCASE__ARRAYS is set to 'foo,bar,baz',
- # then following case id:
- #
- # foo=foo1+foo2,bar=bar1,baz=
- #
- # will set $foo to two-member array, $bar to a single member array, and
- # $baz to an empty array.
- #
- # Note: it's currently not possible to create array with a single item
- # of an empty string, i.e. `foo=("")`. `foo=` means empty array and
- # `foo=+` means array with two empty strings.
- #
- XCASE__ARRAYS=${XCASE__ARRAYS:-}
-
- xcase__id_error() {
- #
- # Raise test failure due to value of $1.. or whole case id
- #
- # Usage:
- # xcase__id_error [VARNAME]..
- #
- # This convenience function can be used when you detect unknown
- # value of xcase__id() or, in variable auto-setting
- # mode, a part of it.
- #
- #
- # For example, instead of:
- #
- # case $(xcase__id) in
- # foo) do_something ;;
- # bar) do_something_else ;;
- # esac
- #
- # always remember to account for unknown id:
- #
- # case $(xcase__id) in
- # foo) do_something ;;
- # bar) do_something_else ;;
- # *) xcase__id_error ;;
- # esac
- #
- # This will ensure that if your xcase__enum() emits
- # an unknown value (perhaps when you added a case but forgot to
- # account for it), the incident will not go unnoticed.
- #
- # In variable auto-setting mode, you can create more useful error
- # message by passing name of the actual variable that got unknown
- # value:
- #
- # xcase__enum() {
- # echo Foo=bar
- # echo Foo=baz
- # echo Foo=quux
- # }
- #
- # # ...later in eg. xcase__setup()
- #
- # case $Foo in
- # bar) do_something ;;
- # baz) do_something_else ;;
- # *) xcase__id_error Foo ;;
- # esac
- #
- # You can actually name more variables if you are not sure which
- # was wrong:
- #
- # case $Foo:$Bar in
- # off:off) something 0 0 ;;
- # on:off) something 0 1 ;;
- # off:on) something 1 0 ;;
- # on:on) something 1 1 ;;
- # *) xcase__id_error Foo Bar ;;
- # esac
- #
- #
- local var
- case $# in
- 0)
- jat__log_error "unhandled case id value: $(xcase__id)"
- ;;
- 1)
- var=$1
- jat__log_error "unhandled case variable: $var=${!var}"
- ;;
- *)
- jat__log_error "unhandled case variable, one of following:"
- for var in "$@";
- do
- jat__log_error " $var='${!var}'"
- done
- ;;
- esac
- jat__log_error "update xcase__enum() or ${FUNCNAME[1]}()"
- }
-
-
- xcase__id() {
- #
- # Print current case ID
- #
- # Inside handler, this function will output current case id as
- # given by enumerator.
- #
- echo "$__xcase__id"
- }
-
- xcase__per() {
- #
- # Permute each line on stdin with variable named $1 at values $2..
- #
- # Usage:
- #
- # xcase__per NAME [--] VALUE1 [VALUE2]..
- # xcase__per NAME -c CMD [ARG]..
- #
- # In first form, take line from stdin (if any), and repeat it to
- # stdout once for each VALUE, adding `,NAME=VALUE` at the end of
- # the line.
- #
- # In second form, do not take VALUEs from arguments but run CMD (witn
- # any ARGs) and use each line of its stdout as VALUE.
- #
- __xcase__doperm per "$@"
- }
-
- xcase__permute() {
- #
- # Permute variable named $1 at values $2..
- #
- # Usage:
- #
- # xcase__permute NAME [--] VALUE1 [VALUE2]..
- # xcase__permute NAME -f CMD [ARG]..
- #
- # In the first form, produce line `NAME=VALUE` for each VALUE.
- #
- # In second form, do not take VALUEs from arguments but run CMD (witn
- # any ARGs) and use each line of its stdout as VALUE.
- #
- __xcase__doperm permute "$@"
- }
-
- xcase__run() {
- #
- # Run all cases from xcase__enum()
- #
- # Usage:
- #
- # xcase__run [-v] [-R] [-T] [-L] [-c path/to/chdir]
- #
- # This function is the main launcher for tests. It will perform roughly
- # following steps:
- #
- # 1. create own temporary directory and chdir there,
- #
- # 2. for each case id emitted from xcase__enum():
- #
- # 1. create a subdirectory of that name, and chdir there,
- #
- # 2. run all available handlers (setup, test, diag, cleanup),
- # wrapped in JAT phases,
- #
- # 3. chdir back
- #
- # 4. Submit relics and reports:
- #
- # * `xcase-relics.tar.gz` with copy of the mentioned temporary
- # directory,
- #
- # * `xcase-results.yaml` with structured results and time stats,
- #
- # * and a nice table in test log
- #
- # See also "GETTING STARTED" and "ADVANCED USAGE" sections.
- #
- # The behavior can be altered using options:
- #
- # '-R' - disable creation and submission of relics tarball.
- #
- # '-c PATH' - change to PATH (must already exist) for the duration
- # of the test (implies '-T').
- #
- # '-L' - disable creation of "leaf" directory for each case id.
- #
- # '-T' - disable migration to temporary directory. xcase will still run
- # in a subdirectory "xcase-relics" of current directory or the
- # directory specified by `-c`.
- #
- # '-v' - enable variable auto-setting (see "ADVANCED USAGE" section).
- #
- local __xcase__leaves=true # enable "leaf" mode?
- local __xcase__tmp # results cache directory
- local __xcase__vars=false # enable variable auto-setting?
- local __xcase__runpath # directory path to run in
- local __xcase__start # start time
- local __xcase__rball=true # collect relics tarball?
- local __xcase__mktemp=true # make own temporary run dir?
- while true; do case $1 in
- -R)
- __xcase__rball=false
- shift
- ;;
- -c)
- __xcase__runpath="$2"
- __xcase__mktemp=false
- shift 2 || {
- jat__log_error "missing value to -c parameter"
- return 2
- }
- test -d "$__xcase__runpath" || {
- jat__log_error "no such directory: $__xcase__runpath"
- return 3
- }
- ;;
- -L)
- __xcase__leaves=false
- shift
- ;;
- -T)
- __xcase__mktemp=false
- shift
- ;;
- -v)
- __xcase__vars=true
- shift
- ;;
- -*)
- jat__log_error "bad argument: '$1'"
- return 2
- ;;
- "")
- break
- ;;
- *)
- jat__log_error "bad argument: '$1'"
- return 2
- ;;
- esac done
- __xcase__tmp=$(mktemp -d -t xcase.meta.XXXXXXXX)
- __xcase__start=$(python -c "import time; print time.time()")
- __xcase__run_all \
- || jat__log_error "errors encountered during case traversal"
- jat__submit "$__xcase__tmp/results.yaml" "xcase-results.yaml"
- $__xcase__rball \
- && jat__submit "$__xcase__tmp/relics.tar.gz" "xcase-relics.tar.gz"
- echo >&2
- rm -rf "$__xcase__tmp"
- }
-
-
- # # #
- # TEMPLATES # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
- # # #
-
- _xcase__enum() {
- #
- # Skeleton for enumerator
- #
- # Body of this function must be implemented by user and it must print
- # out one case ID---word ([a-zA-Z_]*) per line.
- #
- # The case ID is then available inside step handlers as output of
- # function xcase__id()
- #
- echo "case_foo" ...
- }
-
- _xcase__setup() {
- #
- # Skeleton for setup handler
- #
- # Perform setup tasks for case id.
- #
- true
- }
-
- _xcase__test() {
- #
- # Skeleton for test handler
- #
- # Perform tests for case id.
- #
- true
- }
-
- _xcase__diag() {
- #
- # Skeleton for diag handler
- #
- # Perform diag tasks for case id.
- #
- true
- }
-
- _xcase__cleanup() {
- #
- # Skeleton for cleanup handler
- #
- # Perform cleanup tasks for case id.
- #
- true
- }
-
-
- # # risk of brick increases 20% #
- # INTERNAL # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
- # # each time you cross this line #
-
-
- __xcase__has() {
- #
- # Check if handler has been implemented
- #
- type -t "xcase__$1" >/dev/null
- }
-
- __xcase__doperm() {
- #
- # Permutation engine for xcase__per{,mute}
- #
- # This implements both permutation helpers: xcase__per()
- # and xcase__permute(). First parameter says which ('per'
- # or 'permute'), rest is processed as described in those functions'
- # docstrings.
- #
- # The gist is that in 'permute' mode, we just generate key=value pairs,
- # assumed to be "left-most" in the table. In 'per' mode, we generate
- # them but also combuine them with lines on stdin, assumed to be output
- # of one of other modes.
- #
- local mode=$1; shift # mode (permute: first column, per: rest)
- local name=$1; shift # variable name
- local src=args # value source (args: arguments, cmd: command)
- local value # each value
- local line # each line of previous content ('per' mode)
- local oldifs # IFS backup
- local iterfail=false # did iteration command fail?
- case $1 in
- -c) src=cmd; shift ;;
- --) shift ;;
- esac
- test -n "$name" || { __xcase__perm_usage "$mode"; return 2; }
- test -n "$1" || { __xcase__perm_usage "$mode"; return 2; }
- case $src in
- cmd)
- oldifs=$IFS
- IFS=$'\n'
- values=($("$@")) || iterfail=true
- IFS=$oldifs
- $iterfail && {
- jat__log_error "iteration command returned non-zero: $*"
- return 3
- }
- ;;
- args)
- values=("$@")
- ;;
- esac
- case $mode in
- permute)
- for value in "${values[@]}";
- do
- echo "$name=$value"
- done
- ;;
- per)
- while read -r line;
- do
- for value in "${values[@]}";
- do
- echo "$line,$name=$value"
- done
- done
- ;;
- esac
- }
-
- __xcase__perm_usage() {
- #
- # Print usage message for permutation helper
- #
- local self=$1
- jat__log_error \
- "usage: xcase__$self NAME [--] VALUE1 [VALUE2].." \
- "usage: xcase__$self NAME -c CMD [ARG].."
- }
-
- __xcase__plan() {
- #
- # Enumerate and validate cases
- #
- local id # each case id
- __xcase__has enum || {
- jat__log_error "case ID enumerator handler not implemented: xcase__enum"
- return 3
- }
- xcase__enum > "$__xcase__tmp/enum"
- __xcase__ttl=$(wc -l <"$__xcase__tmp/enum")
- jat__log_info "enumerated $__xcase__ttl cases:"
- jat__log_info ""
- while IFS= read -r id;
- do
- jat__log_info " $id"
- done <"$__xcase__tmp/enum"
- jat__log_info ""
- __xcase__validate_enum || return $?
- test "$__xcase__ttl" -eq 0 && {
- jat__log_error "no cases enumerated, nothing to do"
- return 1
- }
- true
- }
-
- __xcase__run_all() {
- #
- # For each case, do all events
- #
- local __xcase__id # each case ID
- local __xcase__hstart # handler start time
- local __xcase__htype # handler type
- local __xcase__n=0 # case number (for hint)
- local __xcase__ttl # total cases (^^)
- local __xcase__varcode # variable auto-setting code
- local __xcase__plan_es=0 # planner (enum+validate) exit status
- local __xcase__sfail # did setup fail?
- local __xcase__tfail # did test fail?
-
- if $__xcase__mktemp;
- then
- __xcase__runpath=$(mktemp -d -t xcase.runpath.XXXXXXXX)
- fi
-
- jat__pstarts "xcase plan"
- __xcase__plan; __xcase__plan_es=$?
- jat__pend
- test $__xcase__plan_es -ne 0 && return $__xcase__plan_es
-
- if test -n "$__xcase__runpath";
- then
- pushd "$__xcase__runpath" >/dev/null || {
- jat__log_error "cannot chdir to: $__xcase__runpath"
- return 3
- }
- fi
-
- mkdir -p "xcase-relics" || {
- jat__log_error "cannot create relics directory: $PWD/xcase-relics"
- return 3
- }
- pushd "xcase-relics" >/dev/null || {
- jat__log_error "cannot chdir to: $PWD/xcase-relics"
- return 3
- }
-
- for __xcase__id in $(<"$__xcase__tmp/enum");
- do
- __xcase__sfail=false
- __xcase__tfail=false
- __xcase__hstart=$(python -c "import time; print time.time()")
-
- if $__xcase__leaves;
- then
- mkdir -p "$__xcase__id"
- pushd "$__xcase__id" >/dev/null || {
- jat__log_error "cannot chdir to: $PWD/$__xcase__id"
- return 3
- }
- fi
-
- ((__xcase__n++))
-
- jat__log_info " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
- jat__log_info " /"
- jat__log_info "o $__xcase__n/$__xcase__ttl: $__xcase__id"
- jat__log_info " \\"
- jat__log_info ' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
- echo >&2
-
- __xcase__varcode=$(__xcase__mkvarcode) || {
- jat__log_error "variable parser encountered errors"
- return 3
- }
- bash -n <<<"$__xcase__varcode" || {
- jat__log_error "variable parser created invalid code: $__xcase__varcode"
- return 3
- }
-
- for __xcase__htype in setup test diag cleanup;
- do
- __xcase__wrap_handler
- done
- __xcase__upload_note
-
- if $__xcase__leaves;
- then
- popd >/dev/null
- fi
-
- __xcase__tmpwrite C.duration "$(python -c "import time; print time.time() - $__xcase__hstart")"
-
- done
-
- popd >/dev/null # from "xcase-relics"
-
- if $__xcase__rball;
- then
- tar -czf "$__xcase__tmp/relics.tar.gz" \
- "xcase-relics"
- fi
-
- if test -n "$__xcase__runpath";
- then
- popd >/dev/null
- fi
-
- if $__xcase__mktemp;
- then
- rm -rf "$__xcase__runpath"
- fi
-
- __xcase__results > "$__xcase__tmp/results.yaml"
- __xcase__summary
- }
-
- __xcase__tmpread() {
- #
- # Get result metadata from key $1
- #
- # See __xcase__tmpfile() for key syntax.
- #
- local key=$1 # key to read
- cat "$(__xcase__tmpfile "$key")"
- }
-
- __xcase__tmpwrite() {
- #
- # Save result metadata value $2 under key $1
- #
- # See __xcase__tmpfile() for key syntax.
- #
- local key=$1 # key to write
- local value=$2 # value to write
- local tgt # target file path
- tgt=$(__xcase__tmpfile "$key")
- echo "$value" > "$tgt"
- }
-
- __xcase__tmpfile() {
- #
- # Dereference temp storage key $1
- #
- # The key may be prefixed by `C.` or `H.`, meaning "for current case id"
- # or "for current handler", respectively. For example, following keys are
- # valid:
- #
- # foo # same file at any time
- # C.foo # same file until the end of this subtest (case id)
- # H.foo # same file until the end of this handler (eg. setup)
- #
- # Note: This function has a side effect within the storage structure that
- # directory for the data file is automatically created so that caller does
- # not need to.
- #
- local key=$1 # key to dereference
- local ns_case # case id infix
- local ns_htype # handler type infix
- local path # final path
- ns_case=id/$__xcase__id
- ns_htype=handler/$__xcase__htype
- path=$__xcase__tmp/data/
- case $key in
- C.*) path+="$ns_case/${key#C.}" ;;
- H.*) path+="$ns_case/$ns_htype/${key#H.}" ;;
- *) path+="$key" ;;
- esac
- mkdir -p "${path%/*}"
- echo "$path"
- }
-
- __xcase__nonempty() {
- #
- # True if directory $1 has files
- #
- local dir=$1
- local es
- test -n "$dir" \
- || jat__log_error "usage: __xcase__nonempty DIR"
- test -d "$dir" \
- || jat__log_error "usage: __xcase__nonempty DIR"
- test -e "$dir"/* 2>/dev/null; es=$?
- case $es in
- 0) return 0 ;; # one file
- 2) return 0 ;; # more files (`test` usage error)
- *) return 1 ;; # anything else
- esac
- }
-
- __xcase__upload_note() {
- #
- # Provide note about file uploads (if applicable)
- #
- local item # of filename list
- $__xcase__rball || return 0
- __xcase__nonempty . || return 0
- jat__log_info "Items to be auto-collected to xcase-relics.tar.gz:"
- find . -mindepth 1 -maxdepth 1 -printf "%P\\n" \
- | sort \
- | while read -r item;
- do
- jat__log_info " * $item"
- done
- echo >&2
- }
-
- __xcase__wrap_handler() {
- #
- # Handler wrapper
- #
- # Set phases, record failures, set up environment...
- #
- local __xcase__hfails="" # this handler fails num.
- local __xcase__hresult=none # this handler result
- __xcase__has "$__xcase__htype" || {
- __xcase__tmpwrite H.result "none"
- return 0
- }
- eval "$__xcase__varcode"
- case $__xcase__htype in
- setup)
- jat__pstarts -c "$__xcase__id" "$__xcase__id :: setup"
- xcase__setup
- ;;
- test)
- jat__pstartt -c "$__xcase__id" "$__xcase__id :: test"
- if $__xcase__sfail;
- then
- jat__log_error "setup failed--skipping test"
- __xcase__tmpwrite H.result "abort"
- jat__pend
- return 1
- else
- xcase__test
- fi
- ;;
- diag)
- jat__pstartd -c "$__xcase__id" "$__xcase__id :: diag"
- xcase__diag
- ;;
- cleanup)
- jat__pstartc -c "$__xcase__id" "$__xcase__id :: cleanup"
- xcase__cleanup
- ;;
- esac
- __xcase__hfails=$(jat__stat pfailc)
- jat__pend
- case $__xcase__hfails in
- 0) __xcase__hresult=pass ;;
- *) __xcase__hresult=fail ;;
- esac
- __xcase__tmpwrite H.result "$__xcase__hresult"
- #shellcheck disable=SC2034
- case $__xcase__htype:$__xcase__hresult in
- setup:fail) __xcase__sfail=true ;;
- test:fail) __xcase__tfail=true ;;
- esac
- }
-
- __xcase__mkvarcode() {
- #
- # Parse $__xcase__id and make variable setting code
- #
- # SIDE EFFECT: write variable cache file for later reference in results
- #
- $__xcase__vars || return 0
- local aname # declaration from $XCASE__ARRAYS
- local ciex # case id expression
- local vnam # variable name
- local vval # variable value
- local vcache # variable cache (for results.yaml)
- vcache="$(__xcase__tmpfile C.vcache)"
- for aname in ${XCASE__ARRAYS//,/ };
- do
- echo "local $aname=()"
- echo "$aname: []" >> "$vcache"
- done
- for ciex in ${__xcase__id//,/ };
- do
- case $ciex in
- *=*)
- vnam=${ciex%%=*}
- vval=${ciex#$vnam=}
- if grep -qw "$vnam" <<<"$XCASE__ARRAYS";
- then # array
- # it's already assigned to empty above, so we don't have
- # to do anything when we see empty value
- if test -n "$vval";
- then
- #shellcheck disable=SC2027,SC2086
- echo "local $vnam=('"${vval//+/"' '"}"')"
- sed -i -e "s/^$vnam:.*/$vnam: ['$vval']/" "$vcache"
- sed -i -e "/^$vnam:/s/+/', '/" "$vcache"
- fi
- else
- echo "local $vnam=$vval"
- echo "$vnam: $vval" >> "$vcache"
- fi
- ;;
- *)
- jat__log_error "invalid assignment in case id: $__xcase__id"
- return 3
- ;;
-
- "")
- jat__log_error "empty item in case id: $__xcase__id"
- ;;
- esac
- done
- }
-
- __xcase__summary() {
- #
- # Show summary phase
- #
- local logfunc # log function for a row
- local id # case id
- local r_setup # handler result: setup
- local r_test # ^^ test
- local r_diag # ^^ diag
- local r_cleanup # ^^ cleanup
- local hdata # path to cached handler data
- local num=0 # row number
- jat__pstartd "xcase summary"
- jat__log_info "duration: $(__xcase__tstats)"
- jat__log_info ""
- jat__log_info "============================================="
- jat__log_info "setup test diag cleanup | case id";
- jat__log_info "--------------------------------|------------"
- for id in $(<"$__xcase__tmp/enum");
- do
- ((num++))
- hdata="$__xcase__tmp/data/id/$id/handler"
- r_setup=$(<"$hdata/setup/result")
- r_test=$(<"$hdata/test/result")
- r_diag=$(<"$hdata/diag/result")
- r_cleanup=$(<"$hdata/cleanup/result")
- case $r_setup:$r_test:$r_diag:$r_cleanup in
- # internal error or cleanup fail: severe issue
- *axerr*) logfunc=jat__log_error ;;
- *:fail) logfunc=jat__log_error ;;
- # else: ok
- *) logfunc=jat__log_info ;;
- esac
- $logfunc "$(
- __xcase__sumrow \
- "$r_setup" "$r_test" "$r_diag" "$r_cleanup" \
- "$num" "$(wc -l < "$__xcase__tmp/enum")" \
- "$id"
- )"
- done
- jat__log_info "============================================="
- __xcase__stats
- jat__log_info ""
- jat__pend
- }
-
- __xcase__sumrow() {
- #
- # Format one row of summary table
- #
- # Rows should look something like this:
- #
- # ...
- # PASS FAIL none PASS | (8/20) some_case
- # PASS FAIL none PASS | (9/20) some_other_case
- # PASS FAIL none PASS | (10/20) yet_another_one
- # PASS FAIL none PASS | (11/20) still_not_done
- # ...
- #
- # Ie. cells formatted to 8 chars, except the last one, which
- # is free-width and prefixed by a right-aligned order hint.
- #
- local r_setup=$1 # handler result: setup
- local r_test=$2 # ^^ test
- local r_diag=$3 # ^^ diag
- local r_cleanup=$4 # ^^ cleanup
- local n=$5 # row number
- local ttl=$6 # total rows
- local id=$7 # case id
- local hintw # order hint width
- __xcase__sumcell "$r_setup"
- __xcase__sumcell "$r_test"
- __xcase__sumcell "$r_diag"
- __xcase__sumcell "$r_cleanup"
- hintw=$((${#ttl} * 2 + 3))
- printf "| %${hintw}s " "($n/$ttl)"
- echo "$id"
- }
-
- __xcase__sumcell() {
- #
- # Format single value $1, from summary table, maybe yell
- #
- local value=$1 # value as stored in handler result cache
- case $value in
- none) echo -n "none " ;;
- pass) echo -n "PASS " ;;
- fail) echo -n "FAIL " ;;
- abort) echo -n "ABORT " ;;
- *) echo -n "!AXERR! " ;;
- esac
- }
-
- __xcase__tstats() {
- #
- # Time-related stats
- #
- local count # total number of cases
- local dur_ttl # total duration (measured here)
- local dur_one # average case duration
- local alltimes # list of all durations from result cache
- alltimes=$(
- grep -hEo "^[0-9]+\.[0-9]{1,3}" "$__xcase__tmp/data/id"/*/duration \
- | sort -n
- )
- count=$(wc -l <<<"$alltimes")
- dur_ttl=$(python -c "import time; print time.time() - $__xcase__start")
- dur_one=$(python -c "print $dur_ttl/$count")
- __xcase__hduration "$dur_ttl" h
- echo -n " (~"
- __xcase__hduration "$dur_one" m
- echo -n " x $count cases,"
- echo -n " min=$(head -1 <<<"$alltimes"),"
- echo -n " max=$(tail -1 <<<"$alltimes"),"
- echo -n " SD=$(awk '{sum+=$1; sumsq+=$1*$1}END{print sqrt(sumsq/NR - (sum/NR)**2)}'<<<"$alltimes"))"
- }
-
- __xcase__hduration() {
- #
- # Format seconds float $1 as [[hhh:][m]mm]:[s]ss up to unit $2
- #
- # 'h' and 'm' units round seconds to integer value ('1.9' -> '2'),
- # 's' rounds to miliseconds.
- #
- # If unit is 'h', format is 'H:MM:SS'. If unit is 'm', format
- # is 'M:SS'. If unit is 's', format is "S.SSS", i.e. number of
- # seconds with fractional part up to 3 digits (miliseconds).
- #
- local s=$1 # seconds (fractional)
- local unit=$2 # precision unit
- local ws # whole seconds
- local m # minutes
- local h # hours
- ws=$(printf "%.0f" "$s")
- case $unit in
- h) h=$((ws / 3600)); ws=$((ws % 3600))
- m=$((ws / 60)); ws=$((ws % 60))
- printf "%d:%02d:%02d" $h $m $ws
- ;;
- m) m=$((ws / 60)); ws=$((ws % 60))
- printf "%d:%02d" $m $ws
- ;;
- s) printf "%.3f" "$s"
- ;;
- esac
- }
-
- __xcase__stats() {
- #
- # Print the statistics
- #
- local results # all results (as grep -H output)
- local result # result (not all are interesting)
- local setups # setups with ^^
- local tests # tests ^^
- local diags # diags ^^
- local cleanups # cleanups ^^
- results=$(find "$__xcase__tmp/data/id/" -path "*/handler/*/result" -exec grep -H . {} +)
- for result in pass fail abort axerr;
- do
- setups=$(grep -c "setup/result:$result$" <<<"$results")
- tests=$(grep -c "test/result:$result$" <<<"$results")
- diags=$(grep -c "diag/result:$result$" <<<"$results")
- cleanups=$(grep -c "cleanup/result:$result$" <<<"$results")
- test "$((setups + tests + diags + cleanups))" -eq 0 && continue
- jat__log_info "$(printf '%-8s%-8s%-8s%-8s| x %s\n' "$setups" "$tests" "$diags" "$cleanups" $result)"
- done
- }
-
- __xcase__results() {
- #
- # Collect case meta-data and print report in YAML
- #
- local idlist # path to cached case id list (straight from user's enum)
- local id # each case id
- idlist=$__xcase__tmp/enum
- echo "---"
- echo "cases: $(wc -l < "$idlist")"
- echo "results:"
- for id in $(<"$idlist");
- do
- pushd "$__xcase__tmp/data/id/$id" >/dev/null
- echo " -"
- echo " id: '$id'"
- test -s "vcache" && {
- echo " variables:"
- sed -e "s/^/ /" "vcache"
- }
- echo " handlers:"
- echo " setup: $(<handler/setup/result)"
- echo " test: $(<handler/test/result)"
- echo " diag: $(<handler/diag/result)"
- echo " cleanup: $(<handler/cleanup/result)"
- echo " duration: $(<duration)"
- echo " result: $(<handler/test/result)"
- popd >/dev/null
- done
- }
-
- __xcase__validate_enum() {
- #
- # Make sure stuff from xcase__enum() has no banned chars
- #
- local allowed='[:alnum:]._,+%=-' # allowed chars in case id
- local es= # exit status of this function
- if grep "[^$allowed]" "$__xcase__tmp/enum";
- then
- if $__xcase__leaves || $__xcase__vars;
- then
- jat__log_error "Sorry, when leaf directory mode (default) or variable"
- jat__log_error "setting mode is used, range of characters that"
- jat__log_error "xcase__enum() can emit is limited to:"
- jat__log_error ""
- jat__log_error " $allowed"
- jat__log_error ""
- jat__log_error "This is to enable usage of this data as file, directory"
- jat__log_error "and variable names. To disable these modes, use flags"
- jat__log_error "-L and -V, respectively."
- jat__log_error "illegal characters in enumerator"
- es=2
- else
- jat__log_error "DEPRECATED characters in enumerator, future version will only allow: $allowed"
- es=0
- fi
- jat__log_error "Note that in order to make best use of xcase, the case id"
- jat__log_error "should not hold any 'real' testing data but rather just"
- jat__log_error "simple generic words to hint *intent* of the test case."
- fi
- return $es
- }
-
- #shellfu module-version=__MKIT_PROJ_VERSION__
|