Overview of changes:

 *  Enabled Vornet Gladness pipeline

 *  Minor cleanups and improvements

 *  Updated MKit to v0.0.40

 *  Updated TFKit to v0.0.18

.gitlab-ci.yml

@@ -0,0 +1,5 @@
+---
+include:
+  - project: 'vornet/infra/gladness'
+    ref: master
+    file: '/pipelines/shellfu.yaml'

README.md

@@ -87,7 +87,7 @@ And that's also *all* this command does.
 Next, if you have also installed some modules, you can look around
 using `sfdoc`:
 
-    sfdoc --lsmods
+    sfdoc --lsmod
     sfdoc inigrep
 
 This is how you access documentation of modules available on the system.

mkit.ini

@@ -1,5 +1,5 @@
 [project]
-    version     = 0.10.11
+    version     = 0.10.27
     codename    = FloatingPointError
     name        = Shellfu
     tagline     = Shell dot on steroids
@@ -7,7 +7,7 @@
     maintainer  = Alois Mahdal <netvor@vornet.cz>
     vcs_browser = https://pagure.io/shellfu
     relsrc      = master
-    reldst      = last
+    reldst      = latest
 
 [dist]
     tarball  = LICENSE.md

notes/style.md

@@ -1,4 +1,4 @@
-Coding style - draft
+Coding style
 ============
 
 This is the official coding style for Shellfu project.
@@ -84,6 +84,7 @@ Imports should be grouped in this order:
 
  1. standard library modules,
  2. related third party modules,
+ 3. local application/library specific imports,
 
 with empty line between the groups, and each group being sorted
 alphabetically.

packaging/debian/control

@@ -48,6 +48,23 @@ Description: Bash-specific base
  This sub-package contains Bash-specific parts of library infrastructure.
  Shellfu modules written for Bash should depend on this.
 
+Package: shellfu-bash-arr
+Architecture: all
+Depends: shellfu-bash, shellfu-bash-pretty, shellfu-sh-isa
+Description: Array utilities
+ Shellfu is an attempt to add modularity to your shell scripts.
+ .
+ With Shellfu you can develop your shell modules separately from your
+ scripts, and test, use, explore or study them without need to be aware
+ of details such as where the actual files are placed.
+ .
+ Shellfu is mostly intended for cases when there's need for non-trivial
+ amount of reliable body of shell scripts, and access to advanced modular
+ languages such as Python or Perl is limited.
+ .
+ This sub-package contains 'arr', Shellfu/Bash module with few utilities
+ for manipulating Bash arrays
+
 Package: shellfu-bash-inigrep
 Architecture: all
 Depends: shellfu-bash, shellfu-bash-pretty
@@ -172,6 +189,26 @@ Description: Shellfu/sh module for building interactive terminal menus
  This sub-package contains 'charmenu', a Shellfu/sh module for building
  interactive terminal menus
 
+Package: shellfu-sh-coerce
+Architecture: all
+Depends:
+ shellfu-sh,
+ perl,
+ perl-modules,
+Description: Shellfu/sh module with data coercion helpers
+ Shellfu is an attempt to add modularity to your shell scripts.
+ .
+ With Shellfu you can develop your shell modules separately from your
+ scripts, and test, use, explore or study them without need to be aware
+ of details such as where the actual files are placed.
+ .
+ Shellfu is mostly intended for cases when there's need for non-trivial
+ amount of reliable body of shell scripts, and access to advanced modular
+ languages such as Python or Perl is limited.
+ .
+ This sub-package contains 'coerce', Shellfu module containing few
+ data coercion helpers.
+
 Package: shellfu-sh-exit
 Architecture: all
 Depends: shellfu-sh
@@ -245,4 +282,4 @@ Description: Shellfu/sh module for terminal color names
  common ANSI color names.
 
 
-# control file built with MKit __MKIT_SELF_VERSION__
+# control file built with MKit __MKIT_MKIT_VERSION__

packaging/debian/shellfu-bash-arr.install

@@ -0,0 +1 @@
+/usr/share/shellfu/include-bash/arr.sh

packaging/debian/shellfu-bash-pretty.install

@@ -1,4 +1,5 @@
 /usr/share/shellfu/include-bash/_pretty_color.sh
+/usr/share/shellfu/include-bash/_pretty_forcecolor.sh
 /usr/share/shellfu/include-bash/_pretty_html.sh
 /usr/share/shellfu/include-bash/_pretty_journald.sh
 /usr/share/shellfu/include-bash/_pretty_notify.sh

packaging/debian/shellfu-sh-coerce.install

@@ -0,0 +1 @@
+/usr/share/shellfu/include-sh/coerce.sh

packaging/template.spec

@@ -4,6 +4,12 @@
 %global pspkg procps-ng
 %endif
 
+%if ( 0%{?rhel} && 0%{?rhel} < 8 ) || ( 0%{?centos} && 0%{?centos} < 8 )
+%global legacy_coerce true
+%else
+%global legacy_coerce false
+%endif
+
 Name:           __MKIT_PROJ_PKGNAME__
 Version:        __MKIT_PROJ_VERSION__
 Release:        1%{?dist}
@@ -38,6 +44,15 @@ Requires: shellfu
 This sub-package contains Bash-specific parts of library infrastructure.
 Shellfu modules written for Bash should depend on this.
 
+%package bash-arr
+Summary: Array utilities
+Requires: shellfu-bash
+Requires: shellfu-bash-pretty
+Requires: shellfu-sh-isa
+%description bash-arr
+This sub-package contains 'arr', Shellfu/Bash module with few utilities
+for manipulating Bash arrays
+
 %package bash-inigrep
 Summary: INI grepping Shellfu/Bash module
 Requires: perl
@@ -97,6 +112,17 @@ Obsoletes: shellfu-bash-extras < 0.10
 This sub-package contains 'charmenu', a Shellfu/sh module for building
 interactive terminal menus
 
+%package sh-coerce
+Summary: Shellfu/sh module with data coercion helpers
+Requires: perl
+%if %{legacy_coerce} == "false"
+Requires: perl-Term-ANSIColor
+%endif
+Requires: shellfu-sh
+%description sh-coerce
+This sub-package contains 'coerce', Shellfu module containing few
+data coercion helpers.
+
 %package sh-exit
 Summary: Shellfu/sh module for standard exit statuses
 Requires: shellfu-sh
@@ -129,6 +155,8 @@ common ANSI color names.
 
 %prep
 %setup -q
+/bin/sed -i '/^__COERCE__LEGACY=/s/:-false/:-%{legacy_coerce}/' ./src/include-sh/coerce.sh
+
 
 %build
 make %{?_smp_mflags} PREFIX=/usr
@@ -155,11 +183,15 @@ make test \
 %files bash
 %dir %{_datadir}/%{name}/include-bash
 
+%files bash-arr
+%{_datadir}/%{name}/include-bash/arr.sh
+
 %files bash-inigrep
 %{_datadir}/%{name}/include-bash/inigrep.sh
 
 %files bash-pretty
 %{_datadir}/%{name}/include-bash/_pretty_color.sh
+%{_datadir}/%{name}/include-bash/_pretty_forcecolor.sh
 %{_datadir}/%{name}/include-bash/_pretty_html.sh
 %{_datadir}/%{name}/include-bash/_pretty_journald.sh
 %{_datadir}/%{name}/include-bash/_pretty_notify.sh
@@ -181,6 +213,9 @@ make test \
 %files sh-charmenu
 %{_datadir}/%{name}/include-sh/charmenu.sh
 
+%files sh-coerce
+%{_datadir}/%{name}/include-sh/coerce.sh
+
 %files sh-exit
 %{_datadir}/%{name}/include-sh/exit.sh
 
@@ -195,4 +230,4 @@ make test \
 
 %changelog
 
-# specfile built with MKit __MKIT_SELF_VERSION__
+# specfile built with MKit __MKIT_MKIT_VERSION__

src/bin/sfdoc

@@ -12,7 +12,7 @@ usage() {
         "[options] MODULE"                                              \
         "[options] --ls [MODULE]"                                       \
         "[options] --ls{var|fun} MODULE"                                \
-        "[options] --which"                                             \
+        "[options] --which MODULE"                                      \
         "[options] --lsmod"                                             \
         "[options] -s|--src MODULE"                                     \
         "[options] --export FMT MODULE"                                 \
@@ -31,6 +31,8 @@ usage() {
             "-O, --object       Treat MODULE as function or variable"   \
             "                   name and find respective module first." \
             "-d, --debug        Turn on debugging"                      \
+            "-h, --hide REGEX   Use REGEX to determine what is hidden." \
+            "                   (See also -a and 'sfdoc sfdoc'.)"       \
             "-a, --all          Don't ignore hidden (starting with"     \
             "                   underscore) modules or objects"         \
             "--encoding ENC     Override encoding of the source text"   \
@@ -93,6 +95,7 @@ main() {
     local RealModuleName    # name to override eg. if accessing via filename
     local encoding          # encoding
     local mpath             # path to module file
+    local jump              # jump to first occurence of this
     action=man
     format=markdown
     encoding=utf8
@@ -102,6 +105,7 @@ main() {
         -O|--object)    spectype=obj;               shift ;;
         -d|--debug)     PRETTY_DEBUG=true;          shift ;;
         -a|--all)       SFDOC_SHOW_HIDDEN=true;     shift ;;
+        -h|--hide)      SFDOC_SHOW_HIDDEN=false; SFDOC_HIDE_REGEX="$2"; shift 2 || usage -w "no REGEX?" ;;
         -I|--include)   SHELLFU_PATH="$2:$SHELLFU_PATH"; shift 2 || usage -w "no PTH?" ;;
         -s|--src)       action=src;                 shift; break ;;
         -l|--ls)        action=lsx;                 shift; break ;;
@@ -120,12 +124,14 @@ main() {
     debug -v SHELLFU_INCLUDE SHELLFU_PATH SFDOC_SHOW_HIDDEN
     debug -v action format module RealModuleName
     case $action:$mspec in
-        lsx:*|lsm:*|wch:*) true ;;
+        lsx:*|lsm:*)    true ;;
         *:)             usage -w "no MODULE?" ;;
     esac
     case $spectype in
         obj) module=$(find_by "$mspec") \
-              || die "no module found with: $mspec" ;;
+              || die "no module found with: $mspec"
+             jump="$mspec"
+             ;;
         *)   module=$mspec ;;
     esac
     case $action in
@@ -134,7 +140,7 @@ main() {
     esac
     case $action in
         exp)    # --export
-            grep -qw "$format" <<<manpage,markdown,pod \
+            grep -qwe "$format" <<<manpage,markdown,pod \
              || die "unknown format: $format"
             sfdoc__export "$format" "${RealModuleName:-$module}" "$mpath" "$encoding"
             ;;
@@ -164,7 +170,9 @@ main() {
              || die "pod2man is missing; cannot use man page mode"
             manfile=$(mktemp -t "sfdoc.manfile.XXXXXXXX")
             sfdoc__export manpage "${RealModuleName:-$module}" "$mpath" "$encoding" >"$manfile"
-            man "$manfile"
+            manless="$LESS"
+            test -n "$jump" && manless+=" +/$jump"
+            LESS="$manless" man "$manfile"
             rm "$manfile"
             ;;
         src)

src/complete.bash

@@ -1,25 +1,88 @@
 
 __sfpath() {
-    local cur prev opts
+    local cur       # current word
+    local prev      # previous word
+    local lopts     # long options
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
-    opts="--version --version-semver"
+    lopts="--version --version-semver"
     COMPREPLY=(
-        $(compgen -W "$opts" -- ${cur})
+        $(compgen -W "$lopts" -- "$cur")
     )
 }
 
 __sfdoc() {
-    local cur prev opts
+    local cur       # current word
+    local prev      # previous word
+    local opts      # options
+    local rest      # rest of options
+    local cmds      # commands
+    local crtext    # text for COMPREPLY
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
-    args="--help --debug --include --ls --lsmod --lsfun --lsvar --export --name"
+    opts="-d -O -a -I -o --debug --object --all --include --only-from --encoding --name"
+    cmds="-l -L -s -e --ls --lsvar --lsfun --which --lsmod --src --export"
+    rest="${opts/$prev/}"
+    crtext=$(
+        case $prev in
+            sfdoc)                  echo "$opts $cmds"; __sfdoc_compgen_M2 ;;
+            -d|--debug)             echo "$rest $cmds"; __sfdoc_compgen_M3 ;;
+            -O|--object)            echo "$rest $cmds"; __sfdoc_compgen_M3 ;;
+            -a|--all)               echo "$rest $cmds"; __sfdoc_compgen_M3 ;;
+            -I|--include)           compgen -o nospace -d -- "$cur" ;;
+            -o|--only-from)         compgen -o nospace -d -- "$cur" ;;
+            --encoding)             iconv -l | tr '[:upper:]' '[:lower:]' | sed s://$:: ;;
+            --name)                 echo "" ;;
+            -e|--export)            echo "markdown manpage pod" ;;
+            -s|--src)               __sfdoc_compgen_M3 ;;
+            -l|--ls)                __sfdoc_compgen_M2 ;;
+            --lsfun|--lsvar)        __sfdoc_compgen_M2 ;;
+            --which)                sfdoc --lsmod ;;
+            markdown|manpage|pod)   __sfdoc_cwhas "-e" "--export" \
+                                      && __sfdoc_compgen_M3  ;;
+        esac
+    )
     COMPREPLY=(
-        $(compgen -W "$args" -- ${cur})
+        $(compgen -W "$crtext" -- "$cur")
     )
 }
 
+__sfdoc_compgen_M2() {
+    #
+    # Produce list of possible MODULE values
+    #
+    case $cur in
+        */*)    compgen -f -o nospace -- "$cur" ;;
+        *)      sfdoc --lsmod ;;
+    esac
+}
+
+__sfdoc_cwhas() {
+    #
+    # True if COMP_WORDS already has word $1 or $2...
+    #
+    local word
+    local want
+    for want in "$@"; do
+        for word in "${COMP_WORDS[@]}"; do
+            test "$word" == "$want" && return 0
+        done
+    done
+    return 1
+}
+
+__sfdoc_compgen_M3() {
+    #
+    # Produce list of possible MODULE values
+    #
+    if __sfdoc_cwhas "-O" "--object"; then
+        sfdoc --ls | cut -d: -f2
+    else
+        __sfdoc_compgen_M2
+    fi
+}
+
 complete -F __sfpath sfpath
 complete -F __sfdoc sfdoc

src/include-bash/_pretty_forcecolor.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+shellfu import termcolors
+
+
+_PRETTY_FORCECOLOR_DEBUG="${TERMCOLORS_LBLUE}"
+_PRETTY_FORCECOLOR_DIE="${TERMCOLORS_LRED}"
+_PRETTY_FORCECOLOR_USAGE_IS="${TERMCOLORS_YELLOW}"
+_PRETTY_FORCECOLOR_THINK="${TERMCOLORS_LBLACK}"
+_PRETTY_FORCECOLOR_WARN="${TERMCOLORS_LRED}"
+_PRETTY_FORCECOLOR_OFF="${TERMCOLORS_NONE}"
+
+
+_pretty__debug() {
+    local decor="()"
+    local caller_is_main=${caller_is_main:-false}
+    local caller=${caller:-UNKNOWN}
+    $caller_is_main && decor=
+    while IFS= read -r line;
+    do echo -ne "${_PRETTY_FORCECOLOR_DEBUG}debug:$caller$decor:$_PRETTY_FORCECOLOR_OFF"
+       echo "$line"; done
+}
+
+
+_pretty__die() {
+    echo -ne "$_PRETTY_FORCECOLOR_DIE"
+    while IFS= read -r line;
+    do echo "$line"; done
+    echo -ne "$_PRETTY_FORCECOLOR_OFF"
+}
+
+
+_pretty__mkhelp() {
+    echo -ne "$_PRETTY_FORCECOLOR_USAGE_IS"
+    while IFS= read -r line;
+    do echo -e "$line"; done
+    echo -ne "$_PRETTY_FORCECOLOR_OFF"
+}
+
+
+_pretty__mkusage() {
+    echo -ne "$_PRETTY_FORCECOLOR_USAGE_IS"
+    while IFS= read -r line;
+    do echo -e "$line"; done
+    echo -ne "$_PRETTY_FORCECOLOR_OFF"
+}
+
+
+_pretty__think() {
+    echo -ne "$_PRETTY_FORCECOLOR_THINK"
+    while IFS= read -r line;
+    do echo "$line"; done
+    echo -ne "$_PRETTY_FORCECOLOR_OFF"
+}
+
+
+_pretty__warn() {
+    echo -ne "$_PRETTY_FORCECOLOR_WARN"
+    while IFS= read -r line;
+    do echo "$line"; done
+    echo -ne "$_PRETTY_FORCECOLOR_OFF"
+}

src/include-bash/arr.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+shellfu import isa
+shellfu import pretty
+
+#
+# Array utilities
+#
+# Several utilities for loading and printing Bash arrays.
+#
+
+arr__fromcmd() {
+    #
+    # Save output lines from command $2.. to array named $1
+    #
+    # Usage:
+    #     arr__fromcmd ARRAY CMD [ARG]..
+    #
+    # Unlike `mapfile -t <<<"$(command)", this produces zero
+    # items if command output was empty.
+    #
+    local __arr__fromcmd__oarr=$1; shift
+    local __arr__fromcmd__cmd=("$@")
+    local __arr__fromcmd__tmp
+    local es
+    __arr__val_oarr "$__arr__fromcmd__oarr" || return 2
+    __arr__fromcmd__tmp=$(mktemp arr__fromcmd.__arr__fromcmd__tmp.XXXXX)
+    "${__arr__fromcmd__cmd[@]}" > "$__arr__fromcmd__tmp"; es=$?
+    mapfile -t "$__arr__fromcmd__oarr" <"$__arr__fromcmd__tmp"
+    rm "$__arr__fromcmd__tmp"
+    return $es
+}
+
+arr__has() {
+    #
+    # True if value $1 is in array $2
+    #
+    # Usage:
+    #     arr__has VALUE ARRAY
+    #
+    local want=$1
+    local iarr=$2
+    local copy
+    local value
+    local code
+    __arr__val_iarr "$iarr" || return 2
+    code=$(declare -p "$iarr"); code=${code/ $iarr=/ copy=}
+    eval "$code"
+    for value in "${copy[@]}"; do
+        test "$value" == "$want" && return 0
+    done
+    return 1
+}
+
+arr__join() {
+    #
+    # Join items in array $2 by string $1
+    #
+    # Usage:
+    #     arr__join DELIM ARRAY
+    #
+    # Example:
+    #
+    #     names=( Alice Bob )
+    #     arr__join , names
+    #     # prints 'Alice,Bob'
+    #
+    local delim=$1
+    local iarr=$2
+    local first=true
+    local item
+    local copy
+    local code
+    __arr__val_iarr "$iarr" || return 2
+    code=$(declare -p "$iarr"); code=${code/ $iarr=/ copy=}
+    eval "$code"
+    for item in "${copy[@]}"; do
+        $first || echo -n "$delim"
+        first=false
+        echo -n "$item"
+    done
+}
+
+__arr__val_iarr() {
+    #
+    # Validate input array named $1
+    #
+    local name=$1
+    set | grep -q "^$name=" || { warn "no such array: $name"; return 2; }
+}
+
+__arr__val_oarr() {
+    #
+    # Validate output array named $1
+    #
+    local name=$1
+    isa__name "$name" || { warn "invalid array name: $name"; return 2; }
+}

src/include-bash/pretty.sh

@@ -5,10 +5,10 @@ shellfu import exit
 #
 # Application debug mode
 #
-PRETTY_DEBUG=${PRETTY_DEBUG:-false}
+PRETTY_DEBUG=${PRETTY_DEBUG:-}
 
 #
-# List of module/function names to exclude from debuging
+# Comma-separated list of module/function names to exclude from debuging
 #
 # For the sake of readability of your debug dumps, you can set this
 # variable to comma-separated list of module or function names that
@@ -24,7 +24,7 @@ PRETTY_DEBUG_EXCLUDE=${PRETTY_DEBUG_EXCLUDE:-}
 #
 # Application verbosity mode
 #
-PRETTY_VERBOSE=${PRETTY_VERBOSE:-false}
+PRETTY_VERBOSE=${PRETTY_VERBOSE:-}
 
 #
 # Name of pretty-printer module
@@ -76,7 +76,7 @@ debug() {
     #     debug "var1=$var1" "var2=$var2" "result=$result"
     #     debug -v var1 var2 result
     #
-    $PRETTY_DEBUG || return 0
+    test "$PRETTY_DEBUG" == true || return 0
     _pretty__echo "$@"
 }
 
@@ -135,7 +135,14 @@ mkusage() {
     #
     # Optionally, you can add -w MSG to add clarifying message
     # (presented as warning) to help user understand what is
-    # wrong with arguments they have passed.
+    # wrong with arguments they have passed.  Alternatively,
+    # -m SUBJ, -M SUBJ, -u SUBJ, and -U SUBJ are shorthands for
+    # error messages describing missing positional argument (-m),
+    # missing value to a parametrized argument (-M), unknown
+    # argument (-u) or unknown command (-U).  Using these
+    # shorthands you don't need  to provide whole message, but
+    # only subject in question; rest of the message is provided
+    # by pretty.sh.
     #
     # Use "--" to delimit end of arguments processed by mkusage.
     #
@@ -154,6 +161,10 @@ mkusage() {
         -E) doexit=false;   shift ;;
         -k) es=$EXIT_OK;    shift ;;
         -w) cmsg="$2";      shift 2 || return 2 ;;
+        -m) cmsg="no $2?";  shift 2 || return 2 ;;
+        -M) cmsg="missing value for: $2";    shift 2 || return 2 ;;
+        -u) cmsg="unknown argument: $2";     shift 2 || return 2 ;;
+        -U) cmsg="unknown command: $2";      shift 2 || return 2 ;;
         --)                 shift; break ;;
         *)                  break ;;
     esac done
@@ -203,7 +214,7 @@ think() {
     # Use "-l" to split every parameter to separate line, (useful
     # or longer warnings)
     #
-    $PRETTY_VERBOSE || return 0
+    test "$PRETTY_VERBOSE" == true || return 0
     _pretty__echo "$@"
 }
 
@@ -378,11 +389,10 @@ _pretty__echo_cmd() {
     #
     # Print command line, launch it and report exit status
     #
-    local c=\$
-    test "$(id -u)" -eq 0 && c="#"
-    echo "$c $*"
-    "$@"
-    echo "^ exit status: $?"
+    local es
+    echo "-- begin command $* --"
+    "$@"; es=$?
+    echo "-- end command ($es) $* --"
 }
 
 
@@ -393,10 +403,13 @@ _pretty__echo_files() {
     local fp
     for fp in "$@"; do
         if test "$fp" = "-"; then
+            echo "-- begin pipe --"
             cat
+            echo "-- end pipe --"
         elif test -s "$fp" || test "$fp" = "/dev/stdin"; then
-            echo "-- $fp --"
+            echo "-- begin file $fp --"
             cat "$fp"
+            echo "-- end file $fp --"
         fi
     done
 }
@@ -415,7 +428,7 @@ _pretty__echo_trace() {
     #
     # Print "decorated" call trace (only in debug mode)
     #
-    $PRETTY_DEBUG || return 0
+    test "$PRETTY_DEBUG" == true || return 0
     local depth
     echo "== trace =="
     for depth in $(seq 0 ${#FUNCNAME}); do

src/include-bash/sfdoc.sh

@@ -23,8 +23,22 @@ shellfu import pretty
 # with `_` (underscore).  These are ignored by default.  Set this to 'true' to show
 # them.
 #
+# See also $SFDOC_HIDE_REGEX.
+#
 SFDOC_SHOW_HIDDEN=${SFDOC_SHOW_HIDDEN:-false}
 
+#
+# Regex to override what is hidden
+#
+# Override definition of which objects are considered "private", thus hidden
+# by default.  Regular expression is used as Extended Regular Expressions,
+# is anchored on both sides and case-sensitive, e setting 'foo|bAr' matches
+# only objects whose entire name is either 'foo' or 'bAr'.
+#
+# See also $SFDOC_SHOW_HIDDEN.
+#
+SFDOC_HIDE_REGEX=${SFDOC_HIDE_REGEX:-_.*}
+
 sfdoc__export() {
     #
     # Export docs of module $2 from file $3 as format $3
@@ -345,10 +359,11 @@ __sfdoc__filter_hidden() {
     #
     local what=$1
     $SFDOC_SHOW_HIDDEN && cat && return
+    debug -v SFDOC_HIDE_REGEX
     case $what in
-        m)  grep -v ^_ ;;
-        v)  grep -v :_ ;;
-        f)  grep -v :_ ;;
+        m)  grep -vE "^$SFDOC_HIDE_REGEX$" ;;
+        v)  grep -vE ":$SFDOC_HIDE_REGEX$" ;;
+        f)  grep -vE ":$SFDOC_HIDE_REGEX$" ;;
         *)  warn "bug: invalid object type: $otype" ;;
     esac
 }

src/include-bash/sfpi.sh

@@ -226,14 +226,17 @@ sfpi__key() {
     local key=$2    # meta-data key
     local value     # ^^ value
     local fun="${SFPI__PREFIX}_${plg}__sfpimeta" # plg getter function name
-    if type -t "$fun" >/dev/null; then
-        value=$("$fun" "$key")
-        debug -v fun key value
-        test -n "$value" || value="(none)"
-    else
-        value='(undefined)'
-    fi
-    echo "$value"
+    (
+        sfpi__import "$plg"
+        if type -t "$fun" >/dev/null; then
+            value=$("$fun" "$key")
+            debug -v fun key value
+            test -n "$value" || value="(none)"
+        else
+            value='(undefined)'
+        fi
+        echo "$value"
+    )
 }
 
 sfpi__ls_all() {

src/include-sh/coerce.sh

@@ -0,0 +1,95 @@
+#!/bin/sh
+
+#
+# Data coercion helpers
+#
+# This module provides several simple functions to help transform
+# data to fit certain constraints.
+#
+
+#
+# Replacement character
+#
+COERCE__REPCHAR=${COERCE__REPCHAR:-�}
+
+coerce__nocolor() {
+    #
+    # Remove ANSI color codes
+    #
+    if $__COERCE__LEGACY; then
+        sed 's/\x1b\[[0-9;]*m//g'
+    else
+        perl -CS -Mutf8 -MTerm::ANSIColor=colorstrip -ne 'print colorstrip $_;'
+    fi
+}
+
+coerce__noctl() {
+    #
+    # Replace non-printable characters with $COERCE__REPCHAR
+    #
+    # Keep only characters that have grapgical representation ([:graph:] POSIX
+    # class), newline, tab and space.  Replace rest with $COERCE__REPCHAR.
+    #
+    perl -CS -Mutf8 -pe "s|[^[:graph:] \t\n]|$COERCE__REPCHAR|g"
+}
+
+coerce__nofdraw() {
+    #
+    # Replace frame-drawing characters with ASCII
+    #
+    # Replace frame-drawing characters according
+    # to following mapping:
+    #
+    #     ┌ ┬ ┐ └ ┴ ┘ ├ ┤ │ ┼ ─
+    #
+    #     ' ' ' . . . | | | | -
+    #
+    # This converts frame-drawing to ASCII, making it
+    # safer when fonts on terminals are not rendering
+    # properly.
+    #
+    perl -CS -Mutf8 -pe "
+        tr{┌┬┐}{.};
+        tr{└┴┘}{'};
+        tr{├┼┤│}{|};
+        tr{─}{-};
+    "
+}
+
+coerce__for_yaml() {
+    #
+    # Replace yaml-invalid characters
+    #
+    # Yaml won't allow all characters:
+    #
+    # > [...] The allowed character range explicitly excludes the C0 control
+    # > block #x0-#x1F (except for TAB #x9, LF #xA, and CR #xD which are
+    # > allowed), DEL #x7F, the C1 control block #x80-#x9F (except for NEL
+    # > #x85 which is allowed), the surrogate block #xD800-#xDFFF, #xFFFE,
+    # > and #xFFFF.
+    #
+    # so take stdin and replace all unacceptable characters with '�'.
+    #
+    perl -CS -Mutf8 -pe "tr/$(__coerce__for_yaml_bad)/$COERCE__REPCHAR/"
+}
+
+#
+# Legacy mode
+#
+# If 'true', avoids using Term::ANSIColor in favor of a local hacky
+# sed expression yanked from stackoverflow.  Use if Term::ANSIColor
+# is not available (as is case of eg. RHEL-6).
+#
+__COERCE__LEGACY=${__COERCE__LEGACY:-false}
+
+__coerce__for_yaml_bad() {
+    #
+    # Print all YAML-bad chars
+    #
+    printf '\N{U+0}-\N{U+8}\N{U+B}\N{U+C}\N{U+E}-\N{U+1F}'    # C0 with some gaps
+    printf '\N{U+7F}'                                         # DEL alone
+    printf '\N{U+80}-\N{U+84}\N{U+86}-\N{U+9F}'               # C1 with NEL gap
+    # printf -n '\N{U+D800}-\N{U+DFFF}'                        # surrogates
+    # printf -n '\N{U+FFFE}-\N{U+FFFF}'                        # 0xFFFE and 0xFFFF
+    #FIXME: for some reasons perl complains about these ^^
+}

src/include-sh/mdfmt.sh

@@ -78,6 +78,7 @@ _mdfmt__indent() {
     test "$Elem" == h2 && { cat; return; }
     test "$Elem" == h3 && { cat; return; }
     pfx=$(
+        #shellcheck disable=SC2034
         for n in $(seq 1 "$Nest"); do
             printf '    '
         done

src/shellfu.sh.skel

@@ -173,6 +173,19 @@ shellfu() {
                 shellfu __die "cannot import module: $mname"
             fi
             ;;
+        reload)
+            #
+            # Reload module named $1 (unsafe!)
+            #
+            local mname=$1
+            shellfu _is_imported "$mname" || {
+                shellfu __warn "cannot reload; not imported yet: $mname"
+                return 2
+            }
+            if ! shellfu _do_import "$mname"; then
+                shellfu __die "cannot reload module: $mname"
+            fi
+            ;;
         try_import)
             #
             # Try if module $1 could be imported
@@ -185,6 +198,7 @@ shellfu() {
             # Show version of module named $1
             #
             local mname=$1
+            local mpath
             mpath=$(shellfu _select_mfile "$mname") || {
                 shellfu __warn "cannot find module: $mname"
                 return 3
@@ -230,7 +244,7 @@ shellfu() {
             # True if module $1 is already imported
             #
             local mname=$1
-            echo "$__SHELLFU_IMPORTED" | tr : \\n | grep -qx "$mname"
+            echo "$__SHELLFU_IMPORTED" | tr : \\n | grep -qxe "$mname"
             ;;
 
         _list_mfiles)

tests/arr/TF_RUN

@@ -0,0 +1,229 @@
+#!/bin/bash
+#shellcheck disable=SC1090
+
+. "$TF_DIR/include/subtest.sh"
+. "$TF_DIR/include/tools.sh"
+
+. "$(sfpath)" || tf_exit_error "failed to init shellfu"
+export PRETTY=plain
+shellfu import arr    || tf_exit_error "failed to import arr"
+
+
+tf_enum_subtests() {
+    echo arr__fromcmd,inval
+    echo arr__fromcmd,c_null
+    echo arr__fromcmd,c_zlz
+    echo arr__fromcmd,c_lzl
+    echo arr__fromcmd,c_lz
+    echo arr__fromcmd,c_zl
+    echo arr__fromcmd,c_zz
+    echo arr__fromcmd,c_zzz
+    echo arr__fromcmd,c_l
+    echo arr__fromcmd,c_ll
+    echo arr__fromcmd,c_lll
+    echo arr__has,nsuch
+    echo arr__has,c_null
+    echo arr__has,w_null_yes
+    echo arr__has,w_null_not
+    echo arr__has,m_not
+    echo arr__has,m_mid
+    echo arr__has,m_a
+    echo arr__has,m_z
+    echo arr__join,nsuch
+    echo arr__join,d_null
+    echo arr__join,c_null
+    echo arr__join,d_mult
+    echo arr__join,c_zz
+    echo arr__join,c_lll
+    echo arr__join,c_z
+    echo arr__join,c_l
+}
+
+bashfix() {
+    local x=${BASH_VERSION%%.*}
+    local yz=${BASH_VERSION#$x.}
+    local y=${yz%%.**}
+    if test "$x" -eq 4 && test "$y" -lt 4; then
+        tr -d "'"
+    else
+        cat
+    fi
+}
+
+cmd_printing() {
+    local what=$1
+    case $1 in
+        c_null)   : ;;
+        c_z)      echo ;;
+        c_zz)     echo; echo ;;
+        c_zzz)    echo; echo; echo ;;
+        c_l)      echo all ;;
+        c_ll)     echo first; echo last ;;
+        c_lll)    echo first; echo mid; echo last ;;
+        c_zlz)    echo; echo mid; echo ;;
+        c_lzl)    echo first; echo; echo last ;;
+        c_lz)     echo left; echo  ;;
+        c_zl)     echo; echo right  ;;
+    esac
+}
+
+mkadecl() {
+    local name=$1
+    local shape=$2
+    echo -n "declare -a $name="
+    case $shape in
+        inval)    echo '()' ;;
+        c_null)   echo '()' ;;
+        c_z)      echo '([0]="")' ;;
+        c_zz)     echo '([0]="" [1]="")' ;;
+        c_zzz)    echo '([0]="" [1]="" [2]="")' ;;
+        c_l)      echo '([0]="all")' ;;
+        c_ll)     echo '([0]="first" [1]="last")' ;;
+        c_lll)    echo '([0]="first" [1]="mid" [2]="last")' ;;
+        c_zlz)    echo '([0]="" [1]="mid" [2]="")' ;;
+        c_lzl)    echo '([0]="first" [1]="" [2]="last")' ;;
+        c_lz)     echo '([0]="left" [1]="")' ;;
+        c_zl)     echo '([0]="" [1]="right")' ;;
+    esac
+}
+
+getprop() {
+    #
+    # Make artifact or oracle $what for $TestName
+    #
+    local what=$1
+    case $TestName:$what in
+
+        # arr__fromcmd
+
+        arr__fromcmd,inval:args)    echo bad=array=name; echo echo; echo hello ;;
+        arr__fromcmd,c_*:args)      echo cout; echo cmd_printing; echo "${TestName#*,}" ;;
+
+        arr__fromcmd,*:o_decl)      mkadecl cout "${TestName#*,}" ;;
+
+        arr__fromcmd,inval:o_es)    echo 2 ;;
+        arr__fromcmd,*:o_es)        echo 0 ;;
+
+        # arr__has
+
+        arr__has,w_null_yes:a_decl) mkadecl cin c_lzl ;;
+        arr__has,w_null_not:a_decl) mkadecl cin c_lll ;;
+        arr__has,c_null:a_decl)     mkadecl cin c_null ;;
+        arr__has,m_not:a_decl)      mkadecl cin c_lll ;;
+        arr__has,m_mid:a_decl)      mkadecl cin c_lll ;;
+        arr__has,m_a:a_decl)        mkadecl cin c_lll ;;
+        arr__has,m_z:a_decl)        mkadecl cin c_lll ;;
+
+        arr__has,nsuch:args)        echo foo; echo nonesuch ;;
+        arr__has,c_null:args)       echo foo;   echo cin ;;
+        arr__has,w_null*:args)      echo "";    echo cin ;;
+        arr__has,m_not:args)        echo bar;   echo cin ;;
+        arr__has,m_mid:args)        echo mid;   echo cin ;;
+        arr__has,m_a:args)          echo first; echo cin ;;
+        arr__has,m_z:args)          echo last;  echo cin ;;
+
+        arr__has,nsuch:o_es)        echo 2 ;;
+        arr__has,m_not:o_es)        echo 1 ;;
+        arr__has,c_null:o_es)       echo 1 ;;
+        arr__has,w_null_not:o_es)   echo 1 ;;
+        arr__has,*:o_es)            echo 0 ;;
+
+        # arr__join
+
+        arr__join,nsuch:a_decl)     mkadecl cin c_lll ;;
+        arr__join,d_null:a_decl)    mkadecl cin c_lll ;;
+        arr__join,d_mult:a_decl)    mkadecl cin c_lll ;;
+        arr__join,c_null:a_decl)    mkadecl cin c_null ;;
+        arr__join,c_zz:a_decl)      mkadecl cin c_zz ;;
+        arr__join,c_zzz:a_decl)     mkadecl cin c_zzz ;;
+        arr__join,c_lll:a_decl)     mkadecl cin c_lll ;;
+        arr__join,c_z:a_decl)       mkadecl cin c_z ;;
+        arr__join,c_l:a_decl)       mkadecl cin c_l ;;
+
+        arr__join,nsuch:args)       echo oxo;  echo nonesuch ;;
+        arr__join,d_null:args)      echo "";   echo cin ;;
+        arr__join,d_mult:args)      echo oxo;  echo cin ;;
+        arr__join,c_null:args)      echo .;    echo cin ;;
+        arr__join,c_zz:args)        echo .;    echo cin ;;
+        arr__join,c_zzz:args)       echo .;    echo cin ;;
+        arr__join,c_lll:args)       echo .;    echo cin ;;
+        arr__join,c_z:args)         echo .;    echo cin ;;
+        arr__join,c_l:args)         echo .;    echo cin ;;
+
+        arr__join,nsuch:o_out)      echo -n ;;
+        arr__join,d_null:o_out)     echo -n firstmidlast ;;
+        arr__join,d_mult:o_out)     echo -n firstoxomidoxolast ;;
+        arr__join,c_null:o_out)     echo -n '' ;;
+        arr__join,c_zz:o_out)       echo -n . ;;
+        arr__join,c_zzz:o_out)      echo -n .. ;;
+        arr__join,c_lll:o_out)      echo -n first.mid.last ;;
+        arr__join,c_z:o_out)        echo -n ;;
+        arr__join,c_l:o_out)        echo -n all ;;
+
+        arr__join,nsuch:o_es)       echo 2 ;;
+        arr__join,*:o_es)           echo 0 ;;
+
+    esac
+}
+
+mkresult() {
+    local sut
+    local ttype
+    local args=()
+    local cout=()
+    local cin=()
+    local r_es
+    ttype=${TestName##*,}
+    sut=${TestName%,$ttype}
+    type -t "$sut" >/dev/null || {
+        tf_exit_error "no such function: $sut"
+    }
+    readarray -t args <<<"$(getprop args)"
+    tf_debugv args
+    case $sut in
+        arr__fromcmd)
+            cout=()
+            "$sut" "${args[@]}"; r_es=$?
+            declare -p cout | bashfix
+            ;;
+        *)
+            eval "$(getprop a_decl)"
+            "$sut" "${args[@]}"; r_es=$?
+            ;;
+    esac
+    return "$r_es"
+}
+
+mkoracle() {
+    case $TestName in
+        *,nsuch)
+            echo 'no such array: nonesuch' >&2
+            ;;
+        arr__fromcmd,inval)
+            getprop o_decl
+            echo 'invalid array name: bad=array=name' >&2
+            ;;
+        arr__fromcmd,*)
+            getprop o_decl
+            ;;
+        arr__join,*)
+            getprop o_out
+            ;;
+    esac >"oracle/$TestName.out" 2>"oracle/$TestName.err"
+}
+
+tf_do_subtest() {
+    local TestName=$1
+    local o_es
+    mkdir -p oracle result
+    o_es=$(getprop o_es)
+    mkoracle
+    tf_testflt \
+        -n "$TestName" \
+        -O "oracle/$TestName.out" \
+        -E "oracle/$TestName.err" \
+        -S "$o_es" \
+        "mkresult"
+}
+
+tf_do_subtests

tests/shellfu_api/TF_RUN

@@ -12,6 +12,7 @@ flt_ours() {
         -e _pretty_journald \
         -e _pretty_html \
         -e sync \
+        -e _pretty_forcecolor \
         -e _pretty_color \
         -e inigrep \
         -e _pretty_notify \
@@ -22,6 +23,7 @@ flt_ours() {
         -e sfdoc \
         -e sfpi \
         -e mdfmt \
+        -e arr \
     #FIXME: remove the filter when TFKit learns to test in sandbox
 }
 

tests/shellfu_api/oracle/functions.stdout

@@ -5,6 +5,12 @@ _pretty_color:_pretty__mkhelp
 _pretty_color:_pretty__mkusage
 _pretty_color:_pretty__think
 _pretty_color:_pretty__warn
+_pretty_forcecolor:_pretty__debug
+_pretty_forcecolor:_pretty__die
+_pretty_forcecolor:_pretty__mkhelp
+_pretty_forcecolor:_pretty__mkusage
+_pretty_forcecolor:_pretty__think
+_pretty_forcecolor:_pretty__warn
 _pretty_html:_pretty__debug
 _pretty_html:_pretty__die
 _pretty_html:_pretty__mkhelp
@@ -29,6 +35,11 @@ _pretty_plain:_pretty__mkhelp
 _pretty_plain:_pretty__mkusage
 _pretty_plain:_pretty__think
 _pretty_plain:_pretty__warn
+arr:__arr__val_iarr
+arr:__arr__val_oarr
+arr:arr__fromcmd
+arr:arr__has
+arr:arr__join
 charmenu:charmenu
 exit:exit_error
 exit:exit_no

tests/shellfu_api/oracle/modules.stdout

@@ -1,8 +1,10 @@
 _pretty_color
+_pretty_forcecolor
 _pretty_html
 _pretty_journald
 _pretty_notify
 _pretty_plain
+arr
 charmenu
 exit
 inigrep

tests/shellfu_api/oracle/variables.stdout

@@ -4,6 +4,12 @@ _pretty_color:_PRETTY_COLOR_OFF
 _pretty_color:_PRETTY_COLOR_THINK
 _pretty_color:_PRETTY_COLOR_USAGE_IS
 _pretty_color:_PRETTY_COLOR_WARN
+_pretty_forcecolor:_PRETTY_FORCECOLOR_DEBUG
+_pretty_forcecolor:_PRETTY_FORCECOLOR_DIE
+_pretty_forcecolor:_PRETTY_FORCECOLOR_OFF
+_pretty_forcecolor:_PRETTY_FORCECOLOR_THINK
+_pretty_forcecolor:_PRETTY_FORCECOLOR_USAGE_IS
+_pretty_forcecolor:_PRETTY_FORCECOLOR_WARN
 charmenu:CHARMENU_FILE
 exit:EXIT_ERROR
 exit:EXIT_NO
@@ -16,6 +22,7 @@ pretty:PRETTY_DEBUG
 pretty:PRETTY_DEBUG_EXCLUDE
 pretty:PRETTY_USAGE
 pretty:PRETTY_VERBOSE
+sfdoc:SFDOC_HIDE_REGEX
 sfdoc:SFDOC_SHOW_HIDDEN
 sfpi:SFPI__PREFIX
 termcolors:TERMCOLORS_BLACK

utils/mkit/include/build.sh

@@ -71,6 +71,7 @@ __expand_macros() {
     local line          # each line on stdin
     local mname         # each macro name
     local -A MacroMap   # macro value map
+    MacroMap[__MKIT_MKIT_VERSION__]=$MKIT_VERSION
     MacroMap[__MKIT_PROJ_NAME__]=$(ini 1value project:name)
     MacroMap[__MKIT_PROJ_CODENAME__]=$(ini 1value project:codename)
     MacroMap[__MKIT_PROJ_LICENSE__]=$(ini 1value project:license)
@@ -79,8 +80,8 @@ __expand_macros() {
     MacroMap[__MKIT_PROJ_MAINTAINER__]=$(ini 1value project:maintainer)
     MacroMap[__MKIT_PROJ_VCS_BROWSER__]=$(ini 1value project:vcs_browser)
     MacroMap[__MKIT_PROJ_GIT_LASTHASH__]=$(__cached git_lasthash)
+    MacroMap[__MKIT_PROJ_GIT_LASTSUMMARY__]=$(__cached git_lastsummary)
     MacroMap[__MKIT_PROJ_VERSION__]=$(__cached semver)
-    MacroMap[__MKIT_SELF_VERSION__]=$MKIT_VERSION
     for section in "$@"; do
         for mname in $(ini lskeys "$section"); do
             MacroMap[$mname]=$(ini values "$section:$mname")
@@ -161,7 +162,7 @@ __rec_built() {
     #
     local file=$1
     mkdir -p "$MKIT_LOCAL"
-    echo "$file" >> "$MKIT_LOCAL/built.lst"
+    echo "1:$file" >> "$MKIT_LOCAL/built.lst"
 }
 
 _mkit_data() {
@@ -176,6 +177,7 @@ _mkit_data() {
     sections+=( $(ini lssect | grep ':macros$') )
     {
         echo "(builtin):"
+        echo "  x_MKIT_MKIT_VERSION__ => '__MKIT_MKIT_VERSION__'"
         echo "  x_MKIT_PROJ_NAME__ => '__MKIT_PROJ_NAME__'"
         echo "  x_MKIT_PROJ_CODENAME__ => '__MKIT_PROJ_CODENAME__'"
         echo "  x_MKIT_PROJ_LICENSE__ => '__MKIT_PROJ_LICENSE__'"
@@ -184,8 +186,8 @@ _mkit_data() {
         echo "  x_MKIT_PROJ_MAINTAINER__ => '__MKIT_PROJ_MAINTAINER__'"
         echo "  x_MKIT_PROJ_VCS_BROWSER__ => '__MKIT_PROJ_VCS_BROWSER__'"
         echo "  x_MKIT_PROJ_GIT_LASTHASH__ => '__MKIT_PROJ_GIT_LASTHASH__'"
+        echo "  x_MKIT_PROJ_GIT_LASTSUMMARY__ => '__MKIT_PROJ_GIT_LASTSUMMARY__'"
         echo "  x_MKIT_PROJ_VERSION__ => '__MKIT_PROJ_VERSION__'"
-        echo "  x_MKIT_SELF_VERSION__ => '__MKIT_SELF_VERSION__'"
         for section in "${sections[@]}"; do
             echo "$section:"
             for macro in $(ini lskeys "$section"); do
@@ -212,12 +214,31 @@ clean() {
     #
     # Clean up tree after building
     #
-    test -f "$MKIT_LOCAL/built.lst" && {
-        <"$MKIT_LOCAL/built.lst" grep -v -e '\.\.' -e ^/ \
-          | xargs -r rm -rf
-        rm -f "$MKIT_LOCAL/built.lst"
-        rmdir --ignore-fail-on-non-empty "$MKIT_LOCAL"
-    }
+    local path
+    local line
+    local depth
+    test -f "$MKIT_LOCAL/built.lst" || return 0
+    {
+        cat "$MKIT_LOCAL/built.lst"
+        echo "1:$MKIT_LOCAL/built.lst"
+        echo "1:$MKIT_LOCAL"
+    } \
+      | grep -v -e '\.\.' -e ^/ -e '^~' \
+      | while IFS=: read -r depth path; do
+            test -e "$path" || continue
+            case $depth in
+                1)  warn "removing: $path"
+                    test -d "$path" \
+                     && rmdir -p --ignore-fail-on-non-empty "$path"
+                    test -f "$path" && rm "$path"
+                    ;;
+                r)  warn "removing recursively: $path"
+                    rm -r "$path"
+                    ;;
+                *)  warn "invalid built.lst format!"
+                    ;;
+            esac
+        done
     true
 }
 

utils/mkit/include/deploy.sh

@@ -35,7 +35,7 @@ __deploy_item() {
         find "$src" -type f \
           | while read -r item; do
                 [[ $item =~ .skel$ ]] \
-                 && grep -q "${item%.skel}" "$MKIT_LOCAL/built.lst" \
+                 && grep -qe "${item%.skel}" "$MKIT_LOCAL/built.lst" \
                  && continue
                 __deploy_item "$item" "$dst${item#$src}" "$mode"
             done

utils/mkit/include/facts.sh

@@ -103,6 +103,27 @@ git_lasthash() {
     git_bool dirty && echo -n ".dirty"
 }
 
+git_lastsummary() {
+    #
+    # Show last commit summary
+    #
+    # We can't do it outside git repo (or without git) but we should
+    # not be asked to; targets that don't require git should make use
+    # of cache built by dist target.
+    #
+    git_present || {
+        echo UNKNOWN_SUMMARY
+        warn "no git present; could not determine last summary"
+        return 3
+    }
+    git_bool dirty && {
+        echo "(index is dirty)"
+        return
+    }
+    git log -1 --format=oneline HEAD \
+      | cut -d' ' -f2-
+}
+
 semver() {
     #
     # Build proper SemVer version string
@@ -141,15 +162,17 @@ semver() {
     local xyz           # base version string
     local prerl         # pre-release keyword (from mkit.ini, eg. 'beta')
     local latest_tag    # latest git tag
-    local commit        # commit indicator (CURRENT_BRANCH.gHASH)
-    local dirty=F       # F if dirty, T if clean
-    local btime         # timestamp or nothing (see $MKIT_TTAG)
+    local brname        # current branch name
+    local ghash         # current commit short hash
+    local is_tagged=T   # T if tagged (clean, no metadata), F if devel
+    local is_dirty=F    # F if dirty, T if clean
+    local stamp         # timestamp or nothing (see $MKIT_TSTAMP)
     local suffix        # version suffix
     prerl=$(ini 1value project:prerl)
-    case $MKIT_TTAG in
-        none)   btime= ;;
-        btime)  btime=$(date -u +%Y%m%d%H%M%S) ;;
-        ctime)  btime=$(date -d @"$(git_fact latest_cdate)" -u +%Y%m%d%H%M%S) ;;
+    case $MKIT_TSTAMP in
+        none)   stamp= ;;
+        btime)  stamp=$(date -u +%Y%m%d%H%M%S) ;;
+        ctime)  stamp=$(date -d @"$(git_fact latest_cdate)" -u +%Y%m%d%H%M%S) ;;
     esac
     grep ":" <<<"$prerl" \
      && warn "colon in project:prerl may corrupt version data: $prerl"
@@ -168,17 +191,19 @@ semver() {
     esac
     if ! git describe --tags --exact-match HEAD >&/dev/null;
     then    # we are at a later commit than the last tag
-        commit="$(git_fact current_branch).g$(git_fact latest_sha)"
+        is_tagged=F
+        brname=$(git_fact current_branch | sed 's/[^[:alnum:]]/_/g')
+        ghash=$(git_fact latest_sha)
     fi
-    git_bool dirty && dirty=T
-    case "$dirty:$btime:$commit" in
-        F:*:)   suffix=""                       ;;
-        T::)    suffix="+dirty"                 ;;
-        T:*:)   suffix="+t$btime.dirty"           ;;
-        F::*)   suffix="+$commit"               ;;
-        F:*:*)  suffix="+t$btime.$commit"         ;;
-        T::*)   suffix="+$commit.dirty"         ;;
-        T:*:*)  suffix="+t$btime.$commit.dirty"   ;;
+    git_bool dirty && is_dirty=T
+    case "$is_dirty:$is_tagged:$stamp" in
+        F:T:*)  suffix=""                                 ;;
+        T:T:)   suffix="+dirty"                           ;;
+        T:T:*)  suffix="+t$stamp.dirty"                   ;;
+        F:F:)   suffix="+$brname.g$ghash"                 ;;
+        F:F:*)  suffix="+t$stamp.$brname.g$ghash"         ;;
+        T:F:)   suffix="+$brname.g$ghash.dirty"           ;;
+        T:F:*)  suffix="+t$stamp.$brname.g$ghash.dirty"   ;;
         *)      suffix=MKIT_BUG
                 warn "MKIT_BUG: bad dirt/commit detection" ;;
     esac

utils/mkit/include/ini.sh

@@ -8,7 +8,7 @@ __ini_cat() {
     #
     local line      # each line
     while read -r line; do
-        printf -- "%s\n" "$line"
+        printf -- '%s\n' "$line"
     done
 }
 
@@ -17,19 +17,58 @@ __ini_expand() {
     # Expand reference value (prefix only)
     #
     local line      # each input line
-    local suffix    # tail of the line
-    local ref       # reference
-    local value     # value if reference
     while read -r line; do                  # [foo:bar]/path
-        suffix="${line#\[*\]}"              # /path
-        ref="${line%$suffix}"               # [foo:bar]
-        ref="${ref%\]}"                     # [foo:bar
-        ref="${ref#\[}"                     # foo:bar
-        value="$(ini 1value "$ref")"        # foo_bar_value
-        printf -- "%s\n" "$value$suffix"    # foo_bar_value/path
+        __ini_expandln "$line"
     done
 }
 
+__ini_expandln() {
+    #
+    # Fully expand references in line $1
+    #
+    local line_orig=$1          # original line
+    local line_todo=$line_orig  # current state
+    local line_done             # next state
+    local Depth=0               # current depth
+    local MaxDepth=10           # maximum depth
+    while true; do
+        ((Depth++))
+        debug_var line_todo
+        test "$Depth" -le "$MaxDepth" || {
+            warn "expansion error: reached maximum depth: $Depth > $MaxDepth"
+            warn "    original line: $line_orig"
+            warn "    expanded line: $line_todo"
+            return 3
+        }
+        line_done=$(__ini_expandln_once "$line_todo")
+        debug_var line_done
+        test "$line_done" == "$line_todo" && break
+        line_todo=$line_done
+    done
+    echo "$line_done"
+}
+
+__ini_expandln_once() {
+    #
+    # Run through line $1 once and replace all references
+    #
+    local line=$1   # line to expand
+    local ref       # full reference (incl. brackets)
+    local ipath     # just ini path from ^^ (stripped brackets)
+    local value     # value of reference
+    local refs=()   # all references found in line
+    mapfile -t refs <<<"$(grep -Eo '[[][^]]+[]]' <<< "$line_todo")"
+    debug_var refs
+    for ref in "${refs[@]}"; do
+        test -n "$ref" || continue
+        ipath=${ref#[}; ipath=${ipath%]}
+        value=$(ini 1value "$ipath")
+        debug_var line ref ipath value
+        line=$(sed "s|\\[$ipath\\]|$value|" <<<"$line")
+    done
+    echo "$line"
+}
+
 __ini_grepcmt() {
     #
     # Remove comments from INI file on stdin
@@ -85,7 +124,7 @@ __ini_grepsec() {
                 \[*\])    ok=false; continue ;;
             esac
             $ok || continue
-            printf -- "%s\n" "$line"
+            printf -- '%s\n' "$line"
         done \
       | sed -e 's/ *= */=/; s/ +$//; s/^//;'
 }
@@ -102,7 +141,6 @@ __ini_lssect() {
     #
     # List all section names
     #
-    local arg=$1    # unused argument
     grep -x '\[.*\]' | sed 's/^.//; s/.$//'
 }
 
@@ -126,10 +164,10 @@ __ini_body() {
     local inc                       # file to include
     local incre='\[INCLUDE:.*\]'    # include directive regex
     local iline                     # include directive line
-    if iline=$(grep -m1 -x "$incre" "$MKIT_INI"); then
+    if iline=$(grep -m1 -xe "$incre" "$MKIT_INI"); then
         inc=${iline#*:}; inc=${inc%]}
-        grep -vx "$incre" "$inc"
-        grep -vx "$incre" "$MKIT_INI"
+        grep -vxe "$incre" "$inc"
+        grep -vxe "$incre" "$MKIT_INI"
     else
         cat "$MKIT_INI"
     fi | __ini_grepcmt

utils/mkit/include/release.sh

@@ -44,7 +44,7 @@ __relck() {
         at_relsrc)
             oracle=$(ini 1value project:relsrc)
             git_fact current_branch \
-              | grep -qFx "$oracle" \
+              | grep -qFxe "$oracle" \
              || die "you are not on release source branch: $oracle"
             ;;
         not_dirty)
@@ -70,7 +70,7 @@ __relck() {
         ini_version)
             oracle=$(git_fact latest_version | __bump_version "$rlevel")
             ini 1value project:version  \
-              | grep -qFx "$oracle" \
+              | grep -qFxe "$oracle" \
              || die "new version not in mkit.ini: $oracle"
             ;;
         *)
@@ -154,7 +154,7 @@ __vbump() {
     git add mkit.ini \
       || die "failed to add mkit.ini to the index"
     cache=$(mktemp -t "mkit._vbump_gitmsg.XXXXXXXX")
-    _vbump_gitmsg > "$cache"
+    _vbump_gitmsg "$nextver" > "$cache"
     git commit -e -F "$cache"   # note: reading from stdin will break vim
     rm "$cache"
 }
@@ -163,7 +163,8 @@ _vbump_gitmsg() {
     #
     # Compose git message template for 'Bump version' commit
     #
-    echo "Bump version"
+    local nextver=$1
+    echo "Bump version to $nextver"
     echo ""
     echo "Overview of changes:"
     echo ""

utils/mkit/include/vars.sh

@@ -71,7 +71,7 @@ MKIT_PROJ_PKGNAME=""
 # For example, old yum version (as of RHEL-6) will not let you install
 # version that it deems older than is installed, making it hard to
 # continually upgrade during active development.  While packaging
-# systems have their own rukes (and SemVer says both versions should be
+# systems have their own rules (and SemVer says both versions should be
 # considered same) this tag will make it more likely to "win" the build
 # you made later.
 #
@@ -81,9 +81,9 @@ MKIT_PROJ_PKGNAME=""
 # Also note that 'btime' makes the version non-deterministic: merely
 # initiating the build a second later will result in different version.
 #
-MKIT_TTAG=${MKIT_TTAG:-ctime}
+MKIT_TSTAMP=${MKIT_TSTAMP:-ctime}
 
 #
 # This MKit version
 #
-MKIT_VERSION=0.0.35
+MKIT_VERSION=0.0.40

utils/mkit/stub

@@ -162,7 +162,7 @@ deploy() {
             echo ''
             echo '%changelog'
             echo ''
-            echo '# specfile built with MKit __MKIT_SELF_VERSION__'
+            echo '# specfile built with MKit __MKIT_MKIT_VERSION__'
             ;;
 
         packaging/debian/copyright)
@@ -185,7 +185,7 @@ deploy() {
             echo 'Description: __MKIT_PROJ_NAME__ - __MKIT_PROJ_TAGLINE__'
             echo ' MKIT_STUB_DESCRIPTION'
             echo ''
-            echo '# control file built with MKit __MKIT_SELF_VERSION__'
+            echo '# control file built with MKit __MKIT_MKIT_VERSION__'
             ;;
 
         packaging/debian/changelog)
@@ -686,7 +686,7 @@ main() {
     esac
     updating && init_from_existing
     if test -n "$License"; then
-        known_licenses | grep -qxF "$License" \
+        known_licenses | grep -qxFe "$License" \
          || die "unknown license (use -L to get list): $License"
         MkLicense=true
     fi

utils/tfkit/include/common.sh

@@ -1,3 +1,6 @@
+#!/bin/bash
+#shellcheck disable=SC2034,SC1117
+
 #
 # Variables to hold exit status semantic
 #
@@ -123,7 +126,7 @@ tf_think() {
     for msg in "$@";
     do
         $TF_COLOR && echo -ne "$TF_COLOR_LBLACK" >&2
-        echo "$pfx$msg$sfx" >&2;
+        echo "$msg" >&2;
         $TF_COLOR && echo -ne "$TF_COLOR_NONE" >&2
     done
 }

utils/tfkit/include/harness.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
+#shellcheck disable=SC1090
 # tfkit - Shellfu's movable test framework
 # See LICENSE file for copyright and license details.
 
+#shellcheck disable=SC2153
 . "$TF_DIR/include/common.sh"
 
 
@@ -104,7 +106,7 @@ tf_run_tests() {
         tmpdir=$(mktemp -d)
         stamp=$(date "+relics-$tname-%Y%m%d-%H%M%S")
         cp -r "$TF_SUITE/$tname/"* "$tmpdir"
-        pushd "$tmpdir" >/dev/null
+        pushd "$tmpdir" >/dev/null || tf_exit_panic "could not chdir to: $tmpdir"
             TF_DEBUG=$TF_DEBUG TF_VERBOSE=$TF_VERBOSE \
             TF_DIR="$tf_dir" TF_TEST="$tname" \
             TF_FILTER_SUBTEST=$TF_FILTER_SUBTEST \
@@ -112,7 +114,7 @@ tf_run_tests() {
             tes=$?
             __tf_collect_if_needed $tes
             test $tes -gt $es && es=$tes
-        popd >/dev/null
+        popd >/dev/null || tf_exit_panic "could not chdir from: $tmpdir to $OLDPWD"
         rm -rf "$tmpdir"
         if test $tes -eq 0;
         then

utils/tfkit/include/subtest.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+#shellcheck disable=SC1090
 
 . "$TF_DIR/include/common.sh"
 

utils/tfkit/include/tools.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+#shellcheck disable=SC1090
 
 . "$TF_DIR/include/common.sh"
 
@@ -10,10 +11,10 @@ tf_testflt() {
     # Run a simple test for a unix filter
     #
     #     tf_testflt -n foo [-i foo.stdin] \
-    #                [-O foo.stdout] [-E foo.stderr] [-S 3] \
+    #                [-O foo.out] [-E foo.err] [-S 3] \
     #                cmd arg...
     #
-    # Will drop *result/NAME.stdout* and *result/NAME.stderr* (intentionally
+    # Will drop *result/NAME.out* and *result/NAME.err* (intentionally
     # not cleaning up).
     #
 
@@ -44,10 +45,10 @@ tf_testflt() {
         -S) o_es="$2";          shift 2 || { arg_err=true; break; } ;;
         --)                     shift; break ;;
         "")                            break ;;
-        -*) tf_warn "wrong testcli arg: $1"; return $TF_ES_BAILOUT ;;
+        -*) tf_warn "wrong testcli arg: $1"; return "$TF_ES_BAILOUT" ;;
         *)                             break ;;
     esac done
-    $arg_err && { tf_warn "error parsing arguments: $orig_args"; return $TF_ES_BAILOUT; }
+    $arg_err && { tf_warn "error parsing arguments: $orig_args"; return "$TF_ES_BAILOUT"; }
     tf_debug "t_in='$t_in'"
     tf_debug "t_name='$t_name'"
     tf_debug "o_out='$o_out'"
@@ -55,22 +56,22 @@ tf_testflt() {
     tf_debug "o_es='$o_es'"
     tf_debug "test command: $*"
     test "$t_in" = "-" && t_in=/dev/stdin   # works better for check below
-    test -z "$t_name"  && { tf_warn "missing test name"             ; return $TF_ES_BAILOUT; }
-    test -z "$1"       && { tf_warn "missing test command"          ; return $TF_ES_BAILOUT; }
-    test -r "$t_in"    || { tf_warn "missing input file: $t_in"     ; return $TF_ES_BAILOUT; }
-    test -e "$o_out"   || { tf_warn "missing oracle stdout: $o_out" ; return $TF_ES_BAILOUT; }
-    test -e "$o_err"   || { tf_warn "missing oracle stderr: $o_err" ; return $TF_ES_BAILOUT; }
-    test "$o_es" -ge 0 || { tf_warn "invalid oracle status: $o_es"  ; return $TF_ES_BAILOUT; }
+    test -z "$t_name"  && { tf_warn "missing test name"             ; return "$TF_ES_BAILOUT"; }
+    test -z "$1"       && { tf_warn "missing test command"          ; return "$TF_ES_BAILOUT"; }
+    test -r "$t_in"    || { tf_warn "missing input file: $t_in"     ; return "$TF_ES_BAILOUT"; }
+    test -e "$o_out"   || { tf_warn "missing oracle stdout: $o_out" ; return "$TF_ES_BAILOUT"; }
+    test -e "$o_err"   || { tf_warn "missing oracle stderr: $o_err" ; return "$TF_ES_BAILOUT"; }
+    test "$o_es" -ge 0 || { tf_warn "invalid oracle status: $o_es"  ; return "$TF_ES_BAILOUT"; }
 
     # prepare
     #
     mkdir -p result
-    r_out="result/$t_name.stdout"
-    r_err="result/$t_name.stderr"
+    r_out="result/$t_name.out"
+    r_err="result/$t_name.err"
     tf_debug "r_out='$r_out'"
     tf_debug "r_err='$r_err'"
-    touch "$r_out" || { tf_warn "cannot create tmp file: $r_out" ; return $TF_ES_BAILOUT; }
-    touch "$r_err" || { tf_warn "cannot create tmp file: $r_err" ; return $TF_ES_PANIC; }
+    touch "$r_out" || { tf_warn "cannot create tmp file: $r_out" ; return "$TF_ES_BAILOUT"; }
+    touch "$r_err" || { tf_warn "cannot create tmp file: $r_err" ; return "$TF_ES_PANIC"; }
 
     # run
     #
@@ -79,8 +80,8 @@ tf_testflt() {
 
     # eval/report/exit
     #
-    test $r_es = $o_es || { tf_warn "bad exit status: $r_es (need $o_es)" ; t_es=$TF_ES_FAIL; }
-    $diff -u "$o_err" "$r_err" || t_es=$TF_ES_FAIL
-    $diff -u "$o_out" "$r_out" || t_es=$TF_ES_FAIL
-    return $t_es
+    test "$r_es" = "$o_es" || { tf_warn "bad exit status: $r_es (need $o_es)" ; t_es=$TF_ES_FAIL; }
+    $diff -u "$r_err" "$o_err" || t_es=$TF_ES_FAIL
+    $diff -u "$r_out" "$o_out" || t_es=$TF_ES_FAIL
+    return "$t_es"
 }

utils/tfkit/runtests

@@ -1,8 +1,9 @@
 #!/bin/bash
+#shellcheck disable=SC1090
 # tfkit - Shellfu's movable test framework
 # See LICENSE file for copyright and license details.
 
-TF_VERSION="0.0.16"
+TF_VERSION="0.0.18"
 
 die() {
     echo "$@" && exit 9
@@ -75,8 +76,8 @@ main() {
         -c|--collect)           TF_COLLECT=always;          shift ;;
         -C|--no-collect)        TF_COLLECT=never;           shift ;;
         -d|--debug)             TF_DEBUG=true;              shift ;;
-        -p|--prefix)            export PATH="$(readlink -f "$2")/bin:$PATH"
-                                                            shift 2 || usage ;;
+        -p|--prefix)            PATH="$(readlink -f "$2")/bin:$PATH"
+                                export PATH;                shift 2 || usage ;;
         -s|--filter-subtest)    TF_FILTER_SUBTEST="$2";     shift 2 || usage ;;
         -t|--filter-test)       TF_FILTER_TEST="$2";        shift 2 || usage ;;
         -v|--verbose)           TF_VERBOSE=true;            shift ;;