shellfu-bash-xcase/src/xcase.sh.skel

#!/bin/bash

shellfu import jat

#
# Help perform series of homogenous tests
#
# This module aims to make it easier to build and maintain reasonable
# coverage for your SUT based on various combinations od input factors.
#
# This is opposed to traditional "linear" style, where you repeat
# (copy/paste) the same code over and over.  There are multiple
# benefits from this design:
#
#  *  far less code necessary,
#
#  *  more flexible (the enumeration function can employ any logic),
#
#  *  "don't run test if setup failed" logic is used, possibly saving
#     lot of resources,
#
#  *  forget about boring formalities such as `jat__pstart*`,
#
#  *  splitting code to functions, enables you to short-cut by
#     `return` keyword (=> clarity AND resource saving),
#
#  *  the test is much easier to understand (if done well).
#
#
# =head1 HELP, MY TEST IS FAILING AND I DON'T UNDERSTAND THIS SORCERY!
#
# If you encounter xcase-based test and want to understand what's
# happening, usually the fastest way to get the grip is:
#
#  1. Look at summary table near the end of test log.  Alternatively
#     there's also 'xcase-results.yaml' attached to the test result.
#
#  2. Identify the case id that is interesing for you.  Write it down!
#
#  3. Use full-text search to find the case within the test log.  There can
#     up to 5 phases executed for the case: dummy (does not do anything),
#     'setup', 'test', 'diag' and 'cleanup'.
#
#  4. Open the test code.
#
#     For every phase (setup, test, diag and cleanup), there is one function
#     ("handler") named `xcase__test`, etc. (Actually only 'test'
#     is mandatory.)
#
#     Most handlers will probably refer to output of function
#     xcase__id(), which is case id you have written down in
#     step 2.  Sometimes, case ids can have form of variable assignments.
#     In that case handlers may refer to these variables directly.
#
# Remember that actual meaning of case id (or its embedded variables) is
# entirely defined by test author.  Should this information be unclear,
# xcase can't help you, the only remaining option is to bug the test author.
#
# PRO TIP: Many xcase tests will be long to run, so interactive debugging
# (e.g. using 1minutetip) could be quite painful.  However, function that
# controls *what* is executed is `xcase__enum`.  You can add
# arbitrary filter here (grep...) to speed up the loop!
#
#
# =head1 GETTING STARTED
#
# Here's what you need to do:
#
#  1. Construct a list of cases.  These can be simple self-explanatory
#     names or variable lists to represent combinations of various
#     input factors.
#
#  2. Implement function `xcase__enum()` that just
#     lists them, one per line.
#
#  3. Implement handlers:
#
#         xcase__setup (optional)
#         xcase__test
#         xcase__diag (optional)
#         xcase__cleanup (optional)
#
#     Inside these handlers, case id being currently executed can be
#     accessed by function `xcase__id()`.
#
#  4. Finally, run a single "magic" function, `xcase__run()`.
#     This will:
#
#      1. Call the enumeration function to collect case ids.
#
#      2. For each case id, run all implemented handlers (wrap into
#         phases as needed).
#
#      3. Add nice report at the end.
#
#
# =head1 EXAMPLE
#
#     xcase__enum() {
#         echo none
#         echo empty
#         echo small
#         echo normal
#         echo huge
#     }
#
#     xcase__setup() {
#         jat__cmd mkdir /var/ftp || return 1
#         case $(xcase__id) in
#             small|normal|huge|empty)    jat__cmd cp "$(xcase__id)" /var/ftp || return 1;;
#             none)                       true ;;
#         esac
#         jat__cmd useradd joe || return 1
#         jat__cmd rlServiceRestart hypothetical_ftp || return 1
#     }
#
#     xcase__test() {
#         local file=$(xcase__id)
#         jat__cmd su -c 'hypo_ftp localhost 25 <<<"get $file"' - joe
#         case $(xcase__id) in
#             small|normal|huge|empty) jat__cmd diff "/var/ftp/$file" "/home/joe/$file" ;;
#             none)                    jat__cmd -S 1 test -e "/home/joe/$file";;
#         esac
#     }
#
#     xcase__cleanup() {
#         jat__cmd rm -rf /var/ftp
#         jat__cmd userdel joe
#         jat__cmd rlServiceRestart hypothetical_ftp
#     }
#
#     shellfu import xcase
#
#     jat__pstarts
#         # some general setup
#     jat__pend
#
#     xcase__run
#
#     jat__pstartc
#         # some general cleanup
#     jat__pend
#
# Notice that the same test without xcase would probably require repeating all
# the setup/test/cleanup code for each of the file size; that would obviously
# make test harder to understand and extend.
#
# Also we were able to make the setup much more responsive to cases when
# something is terribly wrong: the setup will now stop (causing test
# to be skipped) if any of the commands fail.
#
# For more practical examples, contact author of this module.
#
#
# =head1 ADVANCED USAGE
#
#
# =head2 Variable auto-setting
#
# Since version 0.11, xcase can parse and set variables automatically from
# the case id.  For example, if case ids are:
#
#     size=m,color=red
#     size=l,color=red
#     size=xl,color=red
#     size=m,color=blue
#     ...
#
# xcase will, inside each handler, set variables `$size` amnd `$color` to
# respective value.  This means you can directly do things like:
#
#     xcase__setup() {
#         case $color in
#             red)   echo "color=#f00" ;;
#             green) echo "color=#0f0" ;;
#             blue)  echo "color=#00f" ;;
#         esac > /etc/foo.conf
#     }
#
# aiming for even better readability of code and test logs.
#
#
# =head2 Permutation helpers
#
# The color/size example above is pretty small, but if you have
# more variables, doing permutations like that can become tedious.
#
# xcase__permute() and xcase__per() are
# here to help you generate such matrix instead of writing it
# yourself or having couple of nested `for` cycles in each of your
# enumerators.
#
# With help of these functions we could rewrite the example as:
#
#     xcase__enum() {
#         xcase__permute size  m l xl \
#           | xcase__per color red green blue
#     }
#
# Now, xcase__permute() creates list of name=value
# pairs, one per line.  xcase__per() takes each line
# and prints 'red', 'green' and 'blue' version of it, appending
# comma and its own name=value pair.
#

#
# Variables treated as arrays
#
# This variable can be set to a comma-delimited list of variable
# names.  Each variable named here is then guaranteed to be declared
# as Bash array.  In case ID, this variable can then list its members
# delimited by '+' (plus sign).
#
# For example, if $XCASE__ARRAYS is set to 'foo,bar,baz',
# then following case id:
#
#     foo=foo1+foo2,bar=bar1,baz=
#
# will set $foo to two-member array, $bar to a single member array, and
# $baz to an empty array.
#
# Note: it's currently not possible to create array with a single item
# of an empty string, i.e. `foo=("")`.  `foo=` means empty array and
# `foo=+` means array with two empty strings.
#
XCASE__ARRAYS=${XCASE__ARRAYS:-}

xcase__id_error() {
    #
    # Raise test failure due to value of $1.. or whole case id
    #
    # Usage:
    #     xcase__id_error [VARNAME]..
    #
    # This convenience function can be used when you detect unknown
    # value of xcase__id() or, in variable auto-setting
    # mode, a part of it.
    #
    #
    # For example, instead of:
    #
    #     case $(xcase__id) in
    #         foo)  do_something ;;
    #         bar)  do_something_else ;;
    #     esac
    #
    # always remember to account for unknown id:
    #
    #     case $(xcase__id) in
    #         foo)  do_something ;;
    #         bar)  do_something_else ;;
    #         *)    xcase__id_error ;;
    #     esac
    #
    # This will ensure that if your xcase__enum() emits
    # an unknown value (perhaps when you added a case but forgot to
    # account for it), the incident will not go unnoticed.
    #
    # In variable auto-setting mode, you can create more useful error
    # message by passing name of the actual variable that got unknown
    # value:
    #
    #     xcase__enum() {
    #         echo Foo=bar
    #         echo Foo=baz
    #         echo Foo=quux
    #     }
    #
    #     # ...later in eg. xcase__setup()
    #
    #     case $Foo in
    #         bar)  do_something ;;
    #         baz)  do_something_else ;;
    #         *)    xcase__id_error Foo ;;
    #     esac
    #
    # You can actually name more variables if you are not sure which
    # was wrong:
    #
    #     case $Foo:$Bar in
    #         off:off)  something 0 0 ;;
    #         on:off)   something 0 1 ;;
    #         off:on)   something 1 0 ;;
    #         on:on)    something 1 1 ;;
    #         *) xcase__id_error Foo Bar ;;
    #     esac
    #
    #
    local var
    case $# in
        0)
            jat__log_error "unhandled case id value: $(xcase__id)"
            ;;
        1)
            var=$1
            jat__log_error "unhandled case variable: $var=${!var}"
            ;;
        *)
            jat__log_error "unhandled case variable, one of following:"
            for var in "$@";
            do
                jat__log_error "    $var='${!var}'"
            done
            ;;
    esac
    jat__log_error "update xcase__enum() or ${FUNCNAME[1]}()"
}


xcase__id() {
    #
    # Print current case ID
    #
    # Inside handler, this function will output current case id as
    # given by enumerator.
    #
    echo "$__xcase__id"
}

xcase__per() {
    #
    # Permute each line on stdin with variable named $1 at values $2..
    #
    # Usage:
    #
    #     xcase__per NAME [--] VALUE1 [VALUE2]..
    #     xcase__per NAME -c CMD [ARG]..
    #
    # In first form, take line from stdin (if any), and repeat it to
    # stdout once for each VALUE, adding `,NAME=VALUE` at the end of
    # the line.
    #
    # In second form, do not take VALUEs from arguments but run CMD (witn
    # any ARGs) and use each line of its stdout as VALUE.
    #
    __xcase__doperm per "$@"
}

xcase__permute() {
    #
    # Permute variable named $1 at values $2..
    #
    # Usage:
    #
    #     xcase__permute NAME [--] VALUE1 [VALUE2]..
    #     xcase__permute NAME -f CMD [ARG]..
    #
    # In the first form, produce line `NAME=VALUE` for each VALUE.
    #
    # In second form, do not take VALUEs from arguments but run CMD (witn
    # any ARGs) and use each line of its stdout as VALUE.
    #
    __xcase__doperm permute "$@"
}

xcase__run() {
    #
    # Run all cases from xcase__enum()
    #
    # Usage:
    #
    #     xcase__run [-v] [-R] [-T] [-L] [-c path/to/chdir]
    #
    # This function is the main launcher for tests.  It will perform roughly
    # following steps:
    #
    #  1. create own temporary directory and chdir there,
    #
    #  2. for each case id emitted from xcase__enum():
    #
    #      1. create a subdirectory of that name, and chdir there,
    #
    #      2. run all available handlers (setup, test, diag, cleanup),
    #         wrapped in JAT phases,
    #
    #  3. chdir back
    #
    #  4. Submit relics and reports:
    #
    #      *  `xcase-relics.tar.gz` with copy of the mentioned temporary
    #         directory,
    #
    #      *  `xcase-results.yaml` with structured results and time stats,
    #
    #      *  and a nice table in test log
    #
    # See also "GETTING STARTED" and "ADVANCED USAGE" sections.
    #
    # The behavior can be altered using options:
    #
    #   '-R' - disable creation and submission of relics tarball.
    #
    #   '-c PATH' - change to PATH (must already exist) for the duration
    #       of the test (implies '-T').
    #
    #   '-L' - disable creation of "leaf" directory for each case id.
    #
    #   '-T' - disable migration to temporary directory.  xcase will still run
    #       in a subdirectory "xcase-relics" of current directory or the
    #       directory specified by `-c`.
    #
    #   '-v' - enable variable auto-setting (see "ADVANCED USAGE" section).
    #
    local __xcase__leaves=true     # enable "leaf" mode?
    local __xcase__tmp             # results cache directory
    local __xcase__vars=false      # enable variable auto-setting?
    local __xcase__runpath         # directory path to run in
    local __xcase__start           # start time
    local __xcase__rball=true      # collect relics tarball?
    local __xcase__mktemp=true     # make own temporary run dir?
    while true; do case $1 in
        -R)
            __xcase__rball=false
            shift
            ;;
        -c)
            __xcase__runpath="$2"
            __xcase__mktemp=false
            shift 2 || {
                jat__log_error "missing value to -c parameter"
                return 2
            }
            test -d "$__xcase__runpath" || {
                jat__log_error "no such directory: $__xcase__runpath"
                return 3
            }
            ;;
        -L)
            __xcase__leaves=false
            shift
            ;;
        -T)
            __xcase__mktemp=false
            shift
            ;;
        -v)
            __xcase__vars=true
            shift
            ;;
        -*)
            jat__log_error "bad argument: '$1'"
            return 2
            ;;
        "")
            break
            ;;
        *)
            jat__log_error "bad argument: '$1'"
            return 2
            ;;
    esac done
    __xcase__tmp=$(mktemp -d -t xcase.meta.XXXXXXXX)
    __xcase__start=$(python -c "import time; print time.time()")
    __xcase__run_all \
     || jat__log_error "errors encountered during case traversal"
    jat__submit "$__xcase__tmp/results.yaml" "xcase-results.yaml"
    $__xcase__rball \
     && jat__submit "$__xcase__tmp/relics.tar.gz" "xcase-relics.tar.gz"
    echo >&2
    rm -rf "$__xcase__tmp"
}


#           #                                                            #
# TEMPLATES # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
#           #                                                            #

_xcase__enum() {
    #
    # Skeleton for enumerator
    #
    # Body of this function must be implemented by user and it must print
    # out one case ID---word ([a-zA-Z_]*) per line.
    #
    # The case ID is then available inside step handlers as output of
    # function xcase__id()
    #
    echo "case_foo" ...
}

_xcase__setup() {
    #
    # Skeleton for setup handler
    #
    # Perform setup tasks for case id.
    #
    true
}

_xcase__test() {
    #
    # Skeleton for test handler
    #
    # Perform tests for case id.
    #
    true
}

_xcase__diag() {
    #
    # Skeleton for diag handler
    #
    # Perform diag tasks for case id.
    #
    true
}

_xcase__cleanup() {
    #
    # Skeleton for cleanup handler
    #
    # Perform cleanup tasks for case id.
    #
    true
}


#          #                                 risk of brick increases 20% #
# INTERNAL # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
#          #                               each time you cross this line #


__xcase__has() {
    #
    # Check if handler has been implemented
    #
    type -t "xcase__$1" >/dev/null
}

__xcase__doperm() {
    #
    # Permutation engine for xcase__per{,mute}
    #
    # This implements both permutation helpers: xcase__per()
    # and xcase__permute().  First parameter says which ('per'
    # or 'permute'), rest is processed as described in those functions'
    # docstrings.
    #
    # The gist is that in 'permute' mode, we just generate key=value pairs,
    # assumed to be "left-most" in the table.  In 'per' mode, we generate
    # them but also combuine them with lines on stdin, assumed to be output
    # of one of other modes.
    #
    local mode=$1; shift    # mode (permute: first column, per: rest)
    local name=$1; shift    # variable name
    local src=args          # value source (args: arguments, cmd: command)
    local value             # each value
    local line              # each line of previous content ('per' mode)
    local oldifs            # IFS backup
    local iterfail=false    # did iteration command fail?
    case $1 in
        -c) src=cmd; shift ;;
        --) shift ;;
    esac
    test -n "$name" || { __xcase__perm_usage "$mode"; return 2; }
    test -n "$1"    || { __xcase__perm_usage "$mode"; return 2; }
    case $src in
        cmd)
            oldifs=$IFS
            IFS=$'\n'
            values=($("$@")) || iterfail=true
            IFS=$oldifs
            $iterfail && {
                jat__log_error "iteration command returned non-zero: $*"
                return 3
            }
            ;;
        args)
            values=("$@")
            ;;
    esac
    case $mode in
        permute)
            for value in "${values[@]}";
            do
                echo "$name=$value"
            done
            ;;
        per)
            while read -r line;
            do
                for value in "${values[@]}";
                do
                    echo "$line,$name=$value"
                done
            done
            ;;
    esac
}

__xcase__perm_usage() {
    #
    # Print usage message for permutation helper
    #
    local self=$1
    jat__log_error \
        "usage: xcase__$self NAME [--] VALUE1 [VALUE2].." \
        "usage: xcase__$self NAME -c CMD [ARG].."
}

__xcase__plan() {
    #
    # Enumerate and validate cases
    #
    local id    # each case id
    __xcase__has enum || {
        jat__log_error "case ID enumerator handler not implemented: xcase__enum"
        return 3
    }
    xcase__enum > "$__xcase__tmp/enum"
    __xcase__ttl=$(wc -l <"$__xcase__tmp/enum")
    jat__log_info "enumerated $__xcase__ttl cases:"
    jat__log_info ""
    while IFS= read -r id;
    do
        jat__log_info "    $id"
    done <"$__xcase__tmp/enum"
    jat__log_info ""
    __xcase__validate_enum || return $?
    test "$__xcase__ttl" -eq 0 && {
        jat__log_error "no cases enumerated, nothing to do"
        return 1
    }
    true
}

__xcase__run_all() {
    #
    # For each case, do all events
    #
    local __xcase__id          # each case ID
    local __xcase__hstart      # handler start time
    local __xcase__htype       # handler type
    local __xcase__n=0         # case number (for hint)
    local __xcase__ttl         # total cases (^^)
    local __xcase__varcode     # variable auto-setting code
    local __xcase__plan_es=0   # planner (enum+validate) exit status
    local __xcase__sfail       # did setup fail?
    local __xcase__tfail       # did test fail?

    if $__xcase__mktemp;
    then
        __xcase__runpath=$(mktemp -d -t xcase.runpath.XXXXXXXX)
    fi

    jat__pstarts "xcase plan"
        __xcase__plan; __xcase__plan_es=$?
    jat__pend
    test $__xcase__plan_es -ne 0 && return $__xcase__plan_es

    if test -n "$__xcase__runpath";
    then
        pushd "$__xcase__runpath" >/dev/null || {
            jat__log_error "cannot chdir to: $__xcase__runpath"
            return 3
        }
    fi

    mkdir -p "xcase-relics" || {
        jat__log_error "cannot create relics directory: $PWD/xcase-relics"
        return 3
    }
    pushd "xcase-relics" >/dev/null || {
        jat__log_error "cannot chdir to: $PWD/xcase-relics"
        return 3
    }

    for __xcase__id in $(<"$__xcase__tmp/enum");
    do
        __xcase__sfail=false
        __xcase__tfail=false
        __xcase__hstart=$(python -c "import time; print time.time()")

        if $__xcase__leaves;
        then
            mkdir -p "$__xcase__id"
            pushd "$__xcase__id" >/dev/null || {
                jat__log_error "cannot chdir to: $PWD/$__xcase__id"
                return 3
            }
        fi

        ((__xcase__n++))

        jat__log_info "  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        jat__log_info " /"
        jat__log_info "o   $__xcase__n/$__xcase__ttl: $__xcase__id"
        jat__log_info " \\"
        jat__log_info '  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
        echo >&2

        __xcase__varcode=$(__xcase__mkvarcode) || {
            jat__log_error "variable parser encountered errors"
            return 3
        }
        bash -n <<<"$__xcase__varcode" || {
            jat__log_error "variable parser created invalid code: $__xcase__varcode"
            return 3
        }

        for __xcase__htype in setup test diag cleanup;
        do
            __xcase__wrap_handler
        done
        __xcase__upload_note

        if $__xcase__leaves;
        then
            popd >/dev/null
        fi

        __xcase__tmpwrite C.duration "$(python -c "import time; print time.time() - $__xcase__hstart")"

    done

    popd >/dev/null     # from "xcase-relics"

    if $__xcase__rball;
    then
        tar -czf "$__xcase__tmp/relics.tar.gz" \
            "xcase-relics"
    fi

    if test -n "$__xcase__runpath";
    then
        popd >/dev/null
    fi

    if $__xcase__mktemp;
    then
        rm -rf "$__xcase__runpath"
    fi

    __xcase__results > "$__xcase__tmp/results.yaml"
    __xcase__summary
}

__xcase__tmpread() {
    #
    # Get result metadata from key $1
    #
    # See __xcase__tmpfile() for key syntax.
    #
    local key=$1    # key to read
    cat "$(__xcase__tmpfile "$key")"
}

__xcase__tmpwrite() {
    #
    # Save result metadata value $2 under key $1
    #
    # See __xcase__tmpfile() for key syntax.
    #
    local key=$1    # key to write
    local value=$2  # value to write
    local tgt       # target file path
    tgt=$(__xcase__tmpfile "$key")
    echo "$value" > "$tgt"
}

__xcase__tmpfile() {
    #
    # Dereference temp storage key $1
    #
    # The key may be prefixed by `C.` or `H.`, meaning "for current case id"
    # or "for current handler", respectively.  For example, following keys are
    # valid:
    #
    #     foo       # same file at any time
    #     C.foo     # same file until the end of this subtest (case id)
    #     H.foo     # same file until the end of this handler (eg. setup)
    #
    # Note: This function has a side effect within the storage structure that
    # directory for the data file is automatically created so that caller does
    # not need to.
    #
    local key=$1    # key to dereference
    local ns_case   # case id infix
    local ns_htype  # handler type infix
    local path      # final path
    ns_case=id/$__xcase__id
    ns_htype=handler/$__xcase__htype
    path=$__xcase__tmp/data/
    case $key in
        C.*)    path+="$ns_case/${key#C.}"           ;;
        H.*)    path+="$ns_case/$ns_htype/${key#H.}" ;;
        *)      path+="$key"                         ;;
    esac
    mkdir -p "${path%/*}"
    echo "$path"
}

__xcase__nonempty() {
    #
    # True if directory $1 has files
    #
    local dir=$1
    local es
    test -n "$dir" \
     || jat__log_error "usage: __xcase__nonempty DIR"
    test -d "$dir" \
     || jat__log_error "usage: __xcase__nonempty DIR"
    test -e "$dir"/* 2>/dev/null; es=$?
    case $es in
        0)  return 0 ;;     # one file
        2)  return 0 ;;     # more files (`test` usage error)
        *)  return 1 ;;     # anything else
    esac
}

__xcase__upload_note() {
    #
    # Provide note about file uploads (if applicable)
    #
    local item      # of filename list
    $__xcase__rball     || return 0
    __xcase__nonempty . || return 0
    jat__log_info "Items to be auto-collected to xcase-relics.tar.gz:"
    find . -mindepth 1 -maxdepth 1 -printf "%P\\n" \
      | sort \
      | while read -r item;
        do
            jat__log_info " *  $item"
        done
    echo >&2
}

__xcase__wrap_handler() {
    #
    # Handler wrapper
    #
    # Set phases, record failures, set up environment...
    #
    local __xcase__hfails=""       # this handler fails num.
    local __xcase__hresult=none    # this handler result
    __xcase__has "$__xcase__htype" || {
        __xcase__tmpwrite H.result "none"
        return 0
    }
    eval "$__xcase__varcode"
    case $__xcase__htype in
        setup)
            jat__pstarts "$__xcase__id :: setup"
            xcase__setup
            ;;
        test)
            jat__pstartt "$__xcase__id :: test"
            if $__xcase__sfail;
            then
                jat__log_error "setup failed--skipping test"
                __xcase__tmpwrite H.result "abort"
                jat__pend
                return 1
            else
                xcase__test
            fi
            ;;
        diag)
            jat__pstartd "$__xcase__id :: diag"
            xcase__diag
            ;;
        cleanup)
            jat__pstartc "$__xcase__id :: cleanup"
            xcase__cleanup
            ;;
    esac
    __xcase__hfails=$(jat__stat pfailc)
    jat__pend
    case $__xcase__hfails in
        0)  __xcase__hresult=pass ;;
        *)  __xcase__hresult=fail ;;
    esac
    __xcase__tmpwrite H.result "$__xcase__hresult"
    #shellcheck disable=SC2034
    case $__xcase__htype:$__xcase__hresult in
        setup:fail) __xcase__sfail=true ;;
        test:fail)  __xcase__tfail=true  ;;
    esac
}

__xcase__mkvarcode() {
    #
    # Parse $__xcase__id and make variable setting code
    #
    # SIDE EFFECT: write variable cache file for later reference in results
    #
    $__xcase__vars || return 0
    local aname     # declaration from $XCASE__ARRAYS
    local ciex      # case id expression
    local vnam      # variable name
    local vval      # variable value
    local vcache    # variable cache (for results.yaml)
    vcache="$(__xcase__tmpfile C.vcache)"
    for aname in ${XCASE__ARRAYS//,/ };
    do
        echo "local $aname=()"
        echo "$aname: []" >> "$vcache"
    done
    for ciex in ${__xcase__id//,/ };
    do
        case $ciex in
            *=*)
                vnam=${ciex%%=*}
                vval=${ciex#$vnam=}
                if grep -qw "$vnam" <<<"$XCASE__ARRAYS";
                then    # array
                    # it's already assigned to empty above, so we don't have
                    # to do anything when we see empty value
                    if test -n "$vval";
                    then
                        #shellcheck disable=SC2027,SC2086
                        echo "local $vnam=('"${vval//+/"' '"}"')"
                        sed -i -e "s/^$vnam:.*/$vnam: ['$vval']/" "$vcache"
                        sed -i -e "/^$vnam:/s/+/', '/" "$vcache"
                    fi
                else
                    echo "local $vnam=$vval"
                    echo "$vnam: $vval" >> "$vcache"
                fi
                ;;
            *)
                jat__log_error "invalid assignment in case id: $__xcase__id"
                return 3
                ;;

            "")
                jat__log_error "empty item in case id: $__xcase__id"
                ;;
        esac
    done
}

__xcase__summary() {
    #
    # Show summary phase
    #
    local logfunc       # log function for a row
    local id            # case id
    local r_setup       # handler result: setup
    local r_test        #              ^^ test
    local r_diag        #              ^^ diag
    local r_cleanup     #              ^^ cleanup
    local hdata         # path to cached handler data
    local num=0         # row number
    jat__pstartd "xcase summary"
        test -n "$PACKAGE" \
         && jat__log_info "PACKAGE version: $(rpm -q "$PACKAGE")"
        jat__log_info "duration: $(__xcase__tstats)"
        jat__log_info ""
        jat__log_info "============================================="
        jat__log_info "setup   test    diag    cleanup | case id";
        jat__log_info "--------------------------------|------------"
        for id in $(<"$__xcase__tmp/enum");
        do
            ((num++))
            hdata="$__xcase__tmp/data/id/$id/handler"
            r_setup=$(<"$hdata/setup/result")
            r_test=$(<"$hdata/test/result")
            r_diag=$(<"$hdata/diag/result")
            r_cleanup=$(<"$hdata/cleanup/result")
            case $r_setup:$r_test:$r_diag:$r_cleanup in
                # internal error or cleanup fail: severe issue
                *axerr*)    logfunc=jat__log_error ;;
                *:fail)     logfunc=jat__log_error ;;
                # else: ok
                *)          logfunc=jat__log_info ;;
            esac
            $logfunc "$(
                __xcase__sumrow \
                    "$r_setup" "$r_test" "$r_diag" "$r_cleanup" \
                    "$num" "$(wc -l < "$__xcase__tmp/enum")" \
                    "$id"
            )"
        done
        jat__log_info "============================================="
        __xcase__stats
        jat__log_info ""
    jat__pend
}

__xcase__sumrow() {
    #
    # Format one row of summary table
    #
    # Rows should look something like this:
    #
    #   ...
    #   PASS    FAIL    none    PASS    |  (8/20) some_case
    #   PASS    FAIL    none    PASS    |  (9/20) some_other_case
    #   PASS    FAIL    none    PASS    | (10/20) yet_another_one
    #   PASS    FAIL    none    PASS    | (11/20) still_not_done
    #   ...
    #
    # Ie. cells formatted to 8 chars, except the last one, which
    # is free-width and prefixed by a right-aligned order hint.
    #
    local r_setup=$1    # handler result: setup
    local r_test=$2     #              ^^ test
    local r_diag=$3     #              ^^ diag
    local r_cleanup=$4  #              ^^ cleanup
    local n=$5          # row number
    local ttl=$6        # total rows
    local id=$7         # case id
    local hintw         # order hint width
    __xcase__sumcell "$r_setup"
    __xcase__sumcell "$r_test"
    __xcase__sumcell "$r_diag"
    __xcase__sumcell "$r_cleanup"
    hintw=$((${#ttl} * 2 + 3))
    printf "| %${hintw}s " "($n/$ttl)"
    echo "$id"
}

__xcase__sumcell() {
    #
    # Format single value $1, from summary table, maybe yell
    #
    local value=$1  # value as stored in handler result cache
    case $value in
        none)   echo -n "none    " ;;
        pass)   echo -n "PASS    " ;;
        fail)   echo -n "FAIL    " ;;
        abort)  echo -n "ABORT   " ;;
        *)      echo -n "!AXERR! " ;;
    esac
}

__xcase__tstats() {
    #
    # Time-related stats
    #
    local count     # total number of cases
    local dur_ttl   # total duration (measured here)
    local dur_one   # average case duration
    local alltimes  # list of all durations from result cache
    alltimes=$(
        grep -hEo "^[0-9]+\.[0-9]{1,3}" "$__xcase__tmp/data/id"/*/duration \
          | sort -n
    )
    count=$(wc -l <<<"$alltimes")
    dur_ttl=$(python -c "import time; print time.time() - $__xcase__start")
    dur_one=$(python -c "print $dur_ttl/$count")
    __xcase__hduration "$dur_ttl" h
    echo -n " (~"
    __xcase__hduration "$dur_one" m
    echo -n " x $count cases,"
    echo -n " min=$(head -1 <<<"$alltimes"),"
    echo -n " max=$(tail -1 <<<"$alltimes"),"
    echo -n " SD=$(awk '{sum+=$1; sumsq+=$1*$1}END{print sqrt(sumsq/NR - (sum/NR)**2)}'<<<"$alltimes"))"
}

__xcase__hduration() {
    #
    # Format seconds float $1 as [[hhh:][m]mm]:[s]ss up to unit $2
    #
    # 'h' and 'm' units round seconds to integer value ('1.9' -> '2'),
    # 's' rounds to miliseconds.
    #
    # If unit is 'h', format is 'H:MM:SS'.  If unit is 'm', format
    # is 'M:SS'.  If unit is 's', format is "S.SSS", i.e. number of
    # seconds with fractional part up to 3 digits (miliseconds).
    #
    local s=$1      # seconds (fractional)
    local unit=$2   # precision unit
    local ws        # whole seconds
    local m         # minutes
    local h         # hours
    ws=$(printf "%.0f" "$s")
    case $unit in
        h)  h=$((ws / 3600)); ws=$((ws % 3600))
            m=$((ws / 60));   ws=$((ws % 60))
            printf "%d:%02d:%02d" $h $m $ws
            ;;
        m)  m=$((ws / 60));   ws=$((ws % 60))
            printf "%d:%02d" $m $ws
            ;;
        s)  printf "%.3f" "$s"
            ;;
    esac
}

__xcase__stats() {
    #
    # Print the statistics
    #
    local results   # all results (as grep -H output)
    local result    # result (not all are interesting)
    local setups    # setups with ^^
    local tests     # tests       ^^
    local diags     # diags       ^^
    local cleanups  # cleanups    ^^
    results=$(find "$__xcase__tmp/data/id/" -path "*/handler/*/result" -exec grep -H . {} +)
    for result in pass fail abort axerr;
    do
        setups=$(grep -c "setup/result:$result$" <<<"$results")
        tests=$(grep -c "test/result:$result$" <<<"$results")
        diags=$(grep -c "diag/result:$result$" <<<"$results")
        cleanups=$(grep -c "cleanup/result:$result$" <<<"$results")
        test "$((setups + tests + diags + cleanups))" -eq 0 && continue
        jat__log_info "$(printf '%-8s%-8s%-8s%-8s| x %s\n' "$setups" "$tests" "$diags" "$cleanups" $result)"
    done
}

__xcase__results() {
    #
    # Collect case meta-data and print report in YAML
    #
    local idlist    # path to cached case id list (straight from user's enum)
    local id        # each case id
    idlist=$__xcase__tmp/enum
    echo "---"
    echo "cases: $(wc -l < "$idlist")"
    echo "results:"
    for id in $(<"$idlist");
    do
        pushd "$__xcase__tmp/data/id/$id" >/dev/null
            echo "    -"
            echo "        id: '$id'"
            test -s "vcache" && {
                echo "        variables:"
                sed -e "s/^/            /" "vcache"
            }
            echo "        handlers:"
            echo "            setup: $(<handler/setup/result)"
            echo "            test: $(<handler/test/result)"
            echo "            diag: $(<handler/diag/result)"
            echo "            cleanup: $(<handler/cleanup/result)"
            echo "        duration: $(<duration)"
            echo "        result: $(<handler/test/result)"
        popd >/dev/null
    done
}

__xcase__validate_enum() {
    #
    # Make sure stuff from xcase__enum() has no banned chars
    #
    local allowed='[:alnum:]._,+%@=-'   # allowed chars in case id
    local es=                           # exit status of this function
    if grep "[^$allowed]" "$__xcase__tmp/enum";
    then
        if $__xcase__leaves || $__xcase__vars;
        then
            jat__log_error "Sorry, when leaf directory mode (default) or variable"
            jat__log_error "setting mode is used, range of characters that"
            jat__log_error "xcase__enum() can emit is limited to:"
            jat__log_error ""
            jat__log_error "    $allowed"
            jat__log_error ""
            jat__log_error "This is to enable usage of this data as file, directory"
            jat__log_error "and variable names.  To disable these modes, use flags"
            jat__log_error "-L and -V, respectively."
            jat__log_error "illegal characters in enumerator"
            es=2
        else
            jat__log_error "DEPRECATED characters in enumerator, future version will only allow: $allowed"
            es=0
        fi
        jat__log_error "Note that in order to make best use of xcase, the case id"
        jat__log_error "should not hold any 'real' testing data but rather just"
        jat__log_error "simple generic words to hint *intent* of the test case."
    fi
    return $es
}

#shellfu module-version=__MKIT_PROJ_VERSION__