Move from fastfoo version 0.7.0+master.g7fde2b3

Makefile

@@ -0,0 +1,9 @@
+# ffoo - Fastfoo - Bash dot on steroids
+# See LICENSE file for copyright and license details.
+
+include config.mk
+
+MKIT_DIR=utils/mkit
+include $(MKIT_DIR)/mkit.mk
+
+.PHONY: test

config.mk

@@ -0,0 +1,3 @@
+VERSION = 0.0.0
+RELSRC = master
+RELDST = last

mkit.ini

@@ -0,0 +1,33 @@
+
+[ENV]
+
+    PKGNAME = mkit
+    PROJNAME = Mkit
+    PREFIX = utils
+
+[lists]
+
+    group = bin
+    group = rest
+
+    dist = config.mk
+    dist = LICENSE
+    dist = Makefile
+    dist = mkit.ini
+    dist = README.md
+    dist = src
+    dist = utils
+
+[modes]
+    bin = 755
+
+[roots]
+    bin  = [ENV:PREFIX]/mkit
+    rest = [ENV:PREFIX]/mkit
+
+[files:bin]
+    src/make     = make
+
+[files:rest]
+    src/include  = include
+    src/mkit.mk  = mkit.mk

src/include/build.sh

@@ -0,0 +1,194 @@
+#!/bin/bash
+
+. "$MKIT_DIR/include/ini.sh" || die "cannot import ini.sh"
+
+
+build() {
+    #
+    # Add meat to all skeletons
+    #
+    local srcpath
+    find src -type f -name '*.skel' \
+     | while read srcpath;
+       do
+           build1 "$srcpath"
+       done
+}
+
+build1() {
+    #
+    # Process one skeleton
+    #
+    local srcpath dstpath
+    srcpath=$1
+    dstpath=${srcpath%.skel}
+    case $dstpath in
+        *.md) <"$srcpath" expand_includes | expand_variables >"$dstpath" ;;
+        *)    <"$srcpath"                   expand_variables >"$dstpath" ;;
+    esac
+    mkdir -p "$MKIT_LOCAL"
+    echo "$dstpath" >> "$MKIT_LOCAL/built.lst"
+}
+
+build_manpages() {
+    local manfile mdfile
+    if command -v ronn >/dev/null;
+    then
+        ini lskeys "files:man" \
+          | while read manfile;
+            do
+                mdfile="$manfile.md"
+                ronn -r "$mdfile"
+                mkdir -p "$MKIT_LOCAL"
+                echo "$manfile" >> "$MKIT_LOCAL/built.lst"
+            done
+    else
+        echo "ronn is not installed"
+        return 1
+    fi
+}
+
+clean() {
+    #
+    # Clean up tree after building
+    #
+    test -f "$MKIT_LOCAL/built.lst" && {
+        <"$MKIT_LOCAL/built.lst" xargs -r rm -f
+        rm -f "$MKIT_LOCAL/built.lst"
+        rmdir --ignore-fail-on-non-empty "$MKIT_LOCAL"
+    }
+    true
+}
+
+dist() {
+    #
+    # Create distributable tarball
+    #
+    local version=$(get_version)
+    local dirname=$MKIT_PKGNAME-$version
+    mkdir -p "$dirname"
+    local item
+    cp -R "$(ini values "lists:dist")" "$dirname"
+    sed -i -e "s/^VERSION = .*/VERSION = $version/" "$dirname/config.mk"
+    tar -cf "$dirname.tar" "$dirname"
+    gzip "$dirname.tar"
+    rm -rf "$dirname"
+}
+
+expand_includes() {
+    #
+    # Expand include directives
+    #
+    # Expand e.g. `<!-- include4: foo.sh -->` to include code of foo.sh
+    #
+    perl -we '
+        use strict;
+        my $text;
+        while (<>) {
+            chomp;
+            if (m/<!-- include4: (\S+) -->/) {
+                open my $fh, $1 or warn "cannot find: $1";
+                my $text = do { local($/); <$fh> };
+                close $fh;
+                $text =~ s/^(.)/    $1/gm;
+                chomp $text;
+                print "$text\n";
+            } else {
+                print "$_\n";
+            }
+        }
+    '
+}
+
+expand_variables() {
+    #
+    # Expand variable values
+    #
+    local script=$(mktemp --tmpdir mkit-tmp.XXXXXXXXXX)
+    local varname varvalue
+    ini lskeys "vars" \
+      | while read varname;
+        do
+            varvalue="$(ini 1value "vars:$varname" | sed -e 's/\$/\\$/' )"
+            echo "s|$varname|$varvalue|;" >> "$script"
+        done
+    echo "s|__CODENAME__|$CODENAME|;"     >> "$script"
+    echo "s|__VERSION__|$(get_version)|;" >> "$script"
+    perl -wp "$script"
+    rm "$script"
+}
+
+get_version() {
+    #
+    # Build semver version string with build metadata
+    #
+    # Build version string from available info using following
+    # logic:
+    #
+    #  1. use VERSION (from config.mk)
+    #  2. if we are in git, override the version with last tag
+    #  3. if set, add PRERELEASE (from config.mk) as pre-release ID
+    #     (afer dash)
+    #  4. if we are at a later commit than the last tag, add branch
+    #     name and commit sha1 to build metadata (after plus sign)
+    #  5. if the tree is "dirty", i.e. has uncommited changes,
+    #     add "dirty" to build metadata
+    #
+    # The version is compatible with SemVer 2.0.0.
+    #
+    # Examples:
+    #
+    #     myprog v1.0.7                         # all clear
+    #     myprog v1.0.7-alpha                   # PRERELEASE="alpha"
+    #     myprog v1.0.7-alpha+g1aef811.master   # ^^ + some commits after
+    #     myprog v1.0.7-alpha+gf14fc4f.api2     # ^^ + on a feature branch
+    #     myprog v1.0.7-alpha+gf14fc4f.api2.dirty  # ^^ + tree edited
+    #     myprog v1.0.7-alpha+dirty             # tag OK but tree edited
+    #     myprog v1.0.7+dirty                   # ^^ but no pre-release id
+    #
+    # Note that versions with "dirty" should be perceived as kind of
+    # dangerous outside developer's own machine.  Versions with sha1 are
+    # safer but must not be released.
+    #
+    # I have considered decorating the git commit refs to make them
+    # sort of sortable (e.g. "r1.g1aef811"), but on second thought,
+    # I don't think it's good idea to give *any* semantics to meta-data
+    # at all.  First, there is no rule that r1<r2<r3; a commit can be
+    # removing what other just added and one change can be split to
+    # multiple commits.  Also, the whole thing breaks anyway once you
+    # rebase your branch (no, it's not a sin).  The sole purpose of
+    # meta-data is to *identify* the code, and provide safe path back
+    # to tree; commit refs are already perfect for that.
+    #
+    # FIXME:  Using PRERELEASE for release IDs may not be compatible with
+    #         release strategy implemented in release.sh
+    #
+    local version=$VERSION
+    local prerl=$PRERELEASE
+    grep ":" <<<"$prerl" && warn "colon in PRERELEASE may corrupt version data: $prerl"
+    if git rev-parse HEAD >&/dev/null;
+    then    # we are in git repo... so we can get smart
+        local lasttag=$(git tag | grep ^v | sort -V | tail -n1)
+        if ! git describe --tags --exact-match HEAD >&/dev/null;
+        then    # we are at a later commit than the last tag
+            local sha=g$(git log -1 --pretty=format:%h HEAD)
+            local curbranch=$(git rev-parse --abbrev-ref HEAD)
+            local commit="$curbranch.$sha"
+        fi
+        if test "$(git diff --shortstat 2>/dev/null)" != "";
+        then    # the tree is "dirty", i.e. has been edited
+            local dirty=dirty
+        fi
+        test -n "$lasttag" && version=${lasttag:1}
+        local suffix=""
+        case "$commit:$dirty" in
+            :)       suffix=""                ;;
+            :dirty)  suffix="+$dirty"         ;;
+            *:)      suffix="+$commit"        ;;
+            *:dirty) suffix="+$commit.$dirty" ;;
+        esac
+        test -b "$prerl" && suffix="-$prerl$suffix"
+        version="$version$suffix"
+    fi
+    echo "$version"
+}

src/include/deploy.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+
+check_env() {
+    #
+    # Check that environment variables have been set properly
+    #
+    PREFIX="$(readlink -m "$PREFIX")"
+    test -d "$PREFIX" || die "PREFIX points to non-existent directory: $PREFIX"
+}
+
+deploy_item() {
+    #
+    # Deploy item and make it look like wanted
+    #
+    # usage: deploy_item src dst [mode]
+    #
+    # Both src and dst must be names of actual items[1],
+    # whereas dst must not exist.  On update, dst is
+    # usually to be replaced but that is uninstall's
+    # job!
+    #
+    #  [1] Ie. src=foo and dst=/foo/bar does *always*
+    #      mean that foo will become 'bar'.  This is
+    #      different than traditional `cp` behavior,
+    #      when this depends if 'bar' already exists
+    #      as a directory.
+    #
+    # If mode is omitted or empty, MKIT_DEFAULT_MODE is
+    # used instead.
+    #
+    # Directories are copied recursively, and mode is
+    # applied only to files.
+    #
+    local src="$1"
+    local dst="$2"
+    local mode="${3:-$MKIT_DEFAULT_MODE}"
+    if test -d "$src";
+    then
+        cp -Tvr "$src" "$dst"
+        find "$dst" -type f -print0 | xargs -0 chmod -c "$mode"
+    else
+        command -p install -DTvm "$mode" "$src" "$dst"
+    fi
+}
+
+get_dst() {
+    #
+    # Find out target path for src file $2 of group $1
+    #
+    local grp=$1
+    local src=$2
+    echo "$(get_root "$grp")/$(ini 1value "files:$grp:$src")"
+}
+
+get_root() {
+    #
+    # Find out target rooot for group $1
+    #
+    local grp="$1"
+    local root=$(ini 1value "roots:$grp")
+    test -n "$root" || die "missing in config.ini: roots:$grp"
+    echo "$root"
+}
+
+install() {
+    #
+    # Install product
+    #
+    check_env
+    local dst group mode src
+    ini values "lists:group" \
+      | while read group;
+        do
+            mode=$(ini 1value "modes:$group")
+            ini lskeys "files:$group" \
+              | while read src;
+                do
+                    dst=$(get_dst "$group" "$src")
+                    deploy_item "$src" "$dst" "$mode"
+                done
+        done
+    test -f "$MKIT_LOCAL/autoclean" && clean
+    true
+}
+
+uninstall() {
+    #
+    # Uninstall product
+    #
+    check_env
+    local dst group src
+    ini values "lists:group" \
+      | while read group;
+        do
+            ini lskeys "files:$group" \
+              | while read src;
+                do
+                    dst=$(get_dst "$group" "$src")
+                    rm -vrf "$dst"
+                done
+        done
+}

src/include/ini.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+ini() {
+    #
+    # do ini operation
+    #
+    local op=$1
+    local arg=$2
+    local fn
+    local limit=_ini_cat
+    case $op in
+        lskeys) fn=_ini_lskeys   ;;
+        sec)    fn=_ini_grepsec  ;;
+        values) fn=_ini_greppath ;;
+        1value) fn=_ini_greppath; limit="tail -1" ;;
+        *)      die "incorrect use of \`ini()\`"
+    esac
+    <"$MKIT_INI" $fn "$arg" | $limit
+}
+
+_ini_cat() {
+    #
+    # A no-op for text stream
+    #
+    while read line;
+    do
+        printf -- "%s\n" "$line"
+    done
+}
+
+_ini_expand() {
+    #
+    # Expand reference value (prefix only)
+    #
+    local line suffix ref value
+    while read line;                        # [foo:bar]/path
+    do
+        suffix="${line#\[*\]}"              # /path
+        ref="${line%$suffix}"               # [foo:bar]
+        ref="${ref%\]}"                     # [foo:bar
+        ref="${ref#\[}"                     # foo:bar
+        value="$(ini 1value "$ref")"        # foo_bar_value
+        printf -- "%s\n" "$value$suffix"    # foo_bar_value/path
+    done
+}
+
+_ini_grepkey() {
+    #
+    # Read key from a section
+    #
+    local wnt=$1
+    grep '.' \
+      | grep -v '\s*#' \
+      | sed -e 's/ *= */=/; s/ +$//; s/^//;' \
+      | grep -e "^$wnt=" \
+      | cut -d= -f2- \
+      | _ini_maybe_expand
+}
+
+_ini_greppath() {
+    #
+    # Read key from the right section
+    #
+    # E.g. `files:share:my/lib.sh` should read
+    #
+    #     [files:share]
+    #         my/lib.sh   = proj/my/lib.sh
+    #
+    local wnt="$1"
+    local wntkey="${wnt##*:}"
+    local wntsec="${wnt%:$wntkey}"
+    if test "$wntsec" = 'ENV';
+    then
+        local override="${!wntkey}"
+        if test -n "$override";
+        then
+            echo "$override"
+            return
+        fi
+    fi
+    _ini_grepsec "$wntsec" | _ini_grepkey "$wntkey"
+}
+
+_ini_grepsec() {
+    #
+    # Read one INI section
+    #
+    local wnt="$1"
+    local ok=false
+    grep '.' \
+      | grep -v '\s*#' \
+      | while read line;
+        do
+            case "$line" in
+                \[$wnt\]) ok=true;  continue ;;
+                \[*\])    ok=false; continue ;;
+            esac
+            $ok || continue
+            printf -- "%s\n" "$line"
+        done \
+      | sed -e 's/ *= */=/; s/ +$//; s/^//;'
+}
+
+_ini_lskeys() {
+    #
+    # List keys from a section
+    #
+    local sct="$1"
+    _ini_grepsec "$sct" | cut -d= -f1 | sort | uniq
+}
+
+_ini_maybe_expand() {
+    #
+    # Decide whether or not to expand
+    #
+    if test "$MKIT_INI_EXPAND" -gt 0;
+    then
+        MKIT_INI_EXPAND=$(( --MKIT_INI_EXPAND )) _ini_expand
+    else
+        _ini_cat
+    fi
+}

src/include/mkit.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+. "$MKIT_DIR/include/build.sh"  || die "cannot import build.sh"
+. "$MKIT_DIR/include/deploy.sh" || die "cannot import deploy.sh"
+. "$MKIT_DIR/include/release.sh" || die "cannot import release.sh"
+. "$MKIT_DIR/include/ini.sh"    || die "cannot import ini.sh"
+
+MKIT_INI=${MKIT_INI:-mkit.ini}
+MKIT_INI_EXPAND=2
+MKIT_PKGNAME=$(ini 1value "ENV:PKGNAME")
+MKIT_PROJNAME=$(ini 1value "ENV:PROJNAME")
+MKIT_DEFAULT_MODE="644"
+
+mkit_init() {
+    #
+    # Do basic initialization
+    #
+    # Check for ini file, load variables from config.mk
+    #
+    test -f "$MKIT_INI" || die "cannot find mkit.ini: $MKIT_INI"
+    tmp=$(mktemp)
+    sed -e 's/ = /=/' < config.mk > "$tmp"
+    . "$tmp"
+    rm -f "$tmp"
+    test -n "$(tr -d '[:space:]' <<<"$MKIT_LOCAL")" \
+     || die "MKIT_LOCAL must be non-blank: '$MKIT_LOCAL'"
+}
+
+die() {
+    #
+    # Exit with message and non-zero exit status
+    #
+    echo "fatal: $*" >&2
+    exit 4
+}
+
+warn() {
+    #
+    # Print warning message
+    #
+    echo "$@" >&2
+}
+
+route() {
+    #
+    # Call correct function based on $1
+    #
+    case $1 in
+        build|build_manpages|clean|dist|install|release_?|uninstall)
+            $1
+            ;;
+        *)
+            echo "usage: $(basename "$0") build|clean|dist|install|uninstall" >&2
+    esac
+}

src/include/release.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+__make_ver() {
+    local level=$1
+    local old=$2
+    local oldx=${old%.*.*}
+    local oldz=${old#*.*.}
+    local tmpy=${old%.*}
+    local oldy=${tmpy#*.}
+    case $level in
+        x) new="$((oldx+1)).0.0"            ;;
+        y) new="$oldx.$((oldy+1)).0"        ;;
+        z) new="$oldx.$oldy.$((oldz+1))"    ;;
+        *) die "invalid release level: $1"  ;;
+    esac
+    echo "$new"
+}
+
+__release() {
+    #
+    # Prepare release
+    #
+    # Span release routines: make a signed tag, check branch
+    # and update release branch
+    #
+    # FIXME: Using PRERELEASE as build.sh:get_version() does may not be
+    #        compatible with this release strategy
+    #
+    local level=$1
+    git rev-parse HEAD >&/dev/null || die "cannot release outside git repo"
+    local lastfile=$(git diff-tree --no-commit-id --name-only -r HEAD)
+    local lasttag=$(git tag | grep ^v | sort -V | tail -n1)
+    local changelog="$(git log --oneline "$lasttag..HEAD")"
+    local curbranch=$(git rev-parse --abbrev-ref HEAD)
+    local dirt="$(git diff --shortstat 2>/dev/null)"
+
+    local newver=$(__make_ver "$level" "${lasttag#v}")
+    local newtag=v$newver
+    local higher=$(echo -e "$oldtag\n$newtag" | sort -V | tail -n1)
+
+    test "$lastfile" = config.mk \
+     || die "last change must be version bump in config.mk"
+
+    test -n "$lasttag" \
+     || die "cannot find last tag"
+
+    test "$newtag" = "$higher" \
+     || die "generated tag looks older: $oldtag<$newtag"
+
+    grep -qw "$newver" config.mk \
+     || die "new version not in config.mk: $newver"
+
+    grep '^....... WIP ' <<<"$changelog" \
+     && die "WIP commit since $lasttag"
+
+    grep -qw "$curbranch" <<<"$RELSRC" \
+     || die "you are not on RELSRC branch: $RELSRC"
+
+    test -z "$dirt" \
+     || die "tree is dirty: $dirt"
+
+    # FIXME: Have user prepare proper message with changelog
+
+    set -e
+    git tag -m "$MKIT_PROJNAME $newtag - $CODENAME" "$newtag"
+    git branch -f "$RELDST" "$newtag"
+}
+
+release_x() {
+    __release x
+}
+
+release_y() {
+    __release y
+}
+
+release_z() {
+    __release z
+}

src/make

@@ -0,0 +1,16 @@
+#!/bin/bash
+# mkit - simple install helper
+# See LICENSE file for copyright and license details.
+
+die() {
+    echo "$@" && exit 9
+}
+
+export MKIT_DIR=${MKIT_DIR:-mkit}
+export MKIT_LOCAL=${MKIT_LOCAL:-.mkit}
+
+. "$MKIT_DIR/include/mkit.sh" || die "failed to init; check if MKIT_DIR is set properly: $MKIT_DIR"
+
+mkit_init
+
+route "$@"

src/mkit.mk

@@ -0,0 +1,41 @@
+# ffoo - Fastfoo - Bash dot on steroids
+# See LICENSE file for copyright and license details.
+
+export MKIT_DIR
+
+all: options build
+
+options:
+	@echo build options:
+	@echo "VERSION  = ${VERSION}"
+	@echo "PRERELEASE = ${PRERELEASE}"
+	@echo "PREFIX   = ${PREFIX}"
+
+build:
+	@$(MKIT_DIR)/make build
+
+manpages: build
+	@$(MKIT_DIR)/make build_manpages
+
+clean:
+	@$(MKIT_DIR)/make clean
+
+dist: clean
+	@$(MKIT_DIR)/make dist
+
+install: all
+	@$(MKIT_DIR)/make install
+
+release_x:
+	@$(MKIT_DIR)/make release_x
+
+release_y:
+	@$(MKIT_DIR)/make release_y
+
+release_z:
+	@$(MKIT_DIR)/make release_z
+
+uninstall:
+	@$(MKIT_DIR)/make uninstall
+
+.PHONY: all options clean dist install uninstall release_x release_y release_z

utils/mkit/include/build.sh

@@ -0,0 +1,194 @@
+#!/bin/bash
+
+. "$MKIT_DIR/include/ini.sh" || die "cannot import ini.sh"
+
+
+build() {
+    #
+    # Add meat to all skeletons
+    #
+    local srcpath
+    find src -type f -name '*.skel' \
+     | while read srcpath;
+       do
+           build1 "$srcpath"
+       done
+}
+
+build1() {
+    #
+    # Process one skeleton
+    #
+    local srcpath dstpath
+    srcpath=$1
+    dstpath=${srcpath%.skel}
+    case $dstpath in
+        *.md) <"$srcpath" expand_includes | expand_variables >"$dstpath" ;;
+        *)    <"$srcpath"                   expand_variables >"$dstpath" ;;
+    esac
+    mkdir -p "$MKIT_LOCAL"
+    echo "$dstpath" >> "$MKIT_LOCAL/built.lst"
+}
+
+build_manpages() {
+    local manfile mdfile
+    if command -v ronn >/dev/null;
+    then
+        ini lskeys "files:man" \
+          | while read manfile;
+            do
+                mdfile="$manfile.md"
+                ronn -r "$mdfile"
+                mkdir -p "$MKIT_LOCAL"
+                echo "$manfile" >> "$MKIT_LOCAL/built.lst"
+            done
+    else
+        echo "ronn is not installed"
+        return 1
+    fi
+}
+
+clean() {
+    #
+    # Clean up tree after building
+    #
+    test -f "$MKIT_LOCAL/built.lst" && {
+        <"$MKIT_LOCAL/built.lst" xargs -r rm -f
+        rm -f "$MKIT_LOCAL/built.lst"
+        rmdir --ignore-fail-on-non-empty "$MKIT_LOCAL"
+    }
+    true
+}
+
+dist() {
+    #
+    # Create distributable tarball
+    #
+    local version=$(get_version)
+    local dirname=$MKIT_PKGNAME-$version
+    mkdir -p "$dirname"
+    local item
+    cp -R "$(ini values "lists:dist")" "$dirname"
+    sed -i -e "s/^VERSION = .*/VERSION = $version/" "$dirname/config.mk"
+    tar -cf "$dirname.tar" "$dirname"
+    gzip "$dirname.tar"
+    rm -rf "$dirname"
+}
+
+expand_includes() {
+    #
+    # Expand include directives
+    #
+    # Expand e.g. `<!-- include4: foo.sh -->` to include code of foo.sh
+    #
+    perl -we '
+        use strict;
+        my $text;
+        while (<>) {
+            chomp;
+            if (m/<!-- include4: (\S+) -->/) {
+                open my $fh, $1 or warn "cannot find: $1";
+                my $text = do { local($/); <$fh> };
+                close $fh;
+                $text =~ s/^(.)/    $1/gm;
+                chomp $text;
+                print "$text\n";
+            } else {
+                print "$_\n";
+            }
+        }
+    '
+}
+
+expand_variables() {
+    #
+    # Expand variable values
+    #
+    local script=$(mktemp --tmpdir mkit-tmp.XXXXXXXXXX)
+    local varname varvalue
+    ini lskeys "vars" \
+      | while read varname;
+        do
+            varvalue="$(ini 1value "vars:$varname" | sed -e 's/\$/\\$/' )"
+            echo "s|$varname|$varvalue|;" >> "$script"
+        done
+    echo "s|__CODENAME__|$CODENAME|;"     >> "$script"
+    echo "s|__VERSION__|$(get_version)|;" >> "$script"
+    perl -wp "$script"
+    rm "$script"
+}
+
+get_version() {
+    #
+    # Build semver version string with build metadata
+    #
+    # Build version string from available info using following
+    # logic:
+    #
+    #  1. use VERSION (from config.mk)
+    #  2. if we are in git, override the version with last tag
+    #  3. if set, add PRERELEASE (from config.mk) as pre-release ID
+    #     (afer dash)
+    #  4. if we are at a later commit than the last tag, add branch
+    #     name and commit sha1 to build metadata (after plus sign)
+    #  5. if the tree is "dirty", i.e. has uncommited changes,
+    #     add "dirty" to build metadata
+    #
+    # The version is compatible with SemVer 2.0.0.
+    #
+    # Examples:
+    #
+    #     myprog v1.0.7                         # all clear
+    #     myprog v1.0.7-alpha                   # PRERELEASE="alpha"
+    #     myprog v1.0.7-alpha+g1aef811.master   # ^^ + some commits after
+    #     myprog v1.0.7-alpha+gf14fc4f.api2     # ^^ + on a feature branch
+    #     myprog v1.0.7-alpha+gf14fc4f.api2.dirty  # ^^ + tree edited
+    #     myprog v1.0.7-alpha+dirty             # tag OK but tree edited
+    #     myprog v1.0.7+dirty                   # ^^ but no pre-release id
+    #
+    # Note that versions with "dirty" should be perceived as kind of
+    # dangerous outside developer's own machine.  Versions with sha1 are
+    # safer but must not be released.
+    #
+    # I have considered decorating the git commit refs to make them
+    # sort of sortable (e.g. "r1.g1aef811"), but on second thought,
+    # I don't think it's good idea to give *any* semantics to meta-data
+    # at all.  First, there is no rule that r1<r2<r3; a commit can be
+    # removing what other just added and one change can be split to
+    # multiple commits.  Also, the whole thing breaks anyway once you
+    # rebase your branch (no, it's not a sin).  The sole purpose of
+    # meta-data is to *identify* the code, and provide safe path back
+    # to tree; commit refs are already perfect for that.
+    #
+    # FIXME:  Using PRERELEASE for release IDs may not be compatible with
+    #         release strategy implemented in release.sh
+    #
+    local version=$VERSION
+    local prerl=$PRERELEASE
+    grep ":" <<<"$prerl" && warn "colon in PRERELEASE may corrupt version data: $prerl"
+    if git rev-parse HEAD >&/dev/null;
+    then    # we are in git repo... so we can get smart
+        local lasttag=$(git tag | grep ^v | sort -V | tail -n1)
+        if ! git describe --tags --exact-match HEAD >&/dev/null;
+        then    # we are at a later commit than the last tag
+            local sha=g$(git log -1 --pretty=format:%h HEAD)
+            local curbranch=$(git rev-parse --abbrev-ref HEAD)
+            local commit="$curbranch.$sha"
+        fi
+        if test "$(git diff --shortstat 2>/dev/null)" != "";
+        then    # the tree is "dirty", i.e. has been edited
+            local dirty=dirty
+        fi
+        test -n "$lasttag" && version=${lasttag:1}
+        local suffix=""
+        case "$commit:$dirty" in
+            :)       suffix=""                ;;
+            :dirty)  suffix="+$dirty"         ;;
+            *:)      suffix="+$commit"        ;;
+            *:dirty) suffix="+$commit.$dirty" ;;
+        esac
+        test -b "$prerl" && suffix="-$prerl$suffix"
+        version="$version$suffix"
+    fi
+    echo "$version"
+}

utils/mkit/include/deploy.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+
+check_env() {
+    #
+    # Check that environment variables have been set properly
+    #
+    PREFIX="$(readlink -m "$PREFIX")"
+    test -d "$PREFIX" || die "PREFIX points to non-existent directory: $PREFIX"
+}
+
+deploy_item() {
+    #
+    # Deploy item and make it look like wanted
+    #
+    # usage: deploy_item src dst [mode]
+    #
+    # Both src and dst must be names of actual items[1],
+    # whereas dst must not exist.  On update, dst is
+    # usually to be replaced but that is uninstall's
+    # job!
+    #
+    #  [1] Ie. src=foo and dst=/foo/bar does *always*
+    #      mean that foo will become 'bar'.  This is
+    #      different than traditional `cp` behavior,
+    #      when this depends if 'bar' already exists
+    #      as a directory.
+    #
+    # If mode is omitted or empty, MKIT_DEFAULT_MODE is
+    # used instead.
+    #
+    # Directories are copied recursively, and mode is
+    # applied only to files.
+    #
+    local src="$1"
+    local dst="$2"
+    local mode="${3:-$MKIT_DEFAULT_MODE}"
+    if test -d "$src";
+    then
+        cp -Tvr "$src" "$dst"
+        find "$dst" -type f -print0 | xargs -0 chmod -c "$mode"
+    else
+        command -p install -DTvm "$mode" "$src" "$dst"
+    fi
+}
+
+get_dst() {
+    #
+    # Find out target path for src file $2 of group $1
+    #
+    local grp=$1
+    local src=$2
+    echo "$(get_root "$grp")/$(ini 1value "files:$grp:$src")"
+}
+
+get_root() {
+    #
+    # Find out target rooot for group $1
+    #
+    local grp="$1"
+    local root=$(ini 1value "roots:$grp")
+    test -n "$root" || die "missing in config.ini: roots:$grp"
+    echo "$root"
+}
+
+install() {
+    #
+    # Install product
+    #
+    check_env
+    local dst group mode src
+    ini values "lists:group" \
+      | while read group;
+        do
+            mode=$(ini 1value "modes:$group")
+            ini lskeys "files:$group" \
+              | while read src;
+                do
+                    dst=$(get_dst "$group" "$src")
+                    deploy_item "$src" "$dst" "$mode"
+                done
+        done
+    test -f "$MKIT_LOCAL/autoclean" && clean
+    true
+}
+
+uninstall() {
+    #
+    # Uninstall product
+    #
+    check_env
+    local dst group src
+    ini values "lists:group" \
+      | while read group;
+        do
+            ini lskeys "files:$group" \
+              | while read src;
+                do
+                    dst=$(get_dst "$group" "$src")
+                    rm -vrf "$dst"
+                done
+        done
+}

utils/mkit/include/ini.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+ini() {
+    #
+    # do ini operation
+    #
+    local op=$1
+    local arg=$2
+    local fn
+    local limit=_ini_cat
+    case $op in
+        lskeys) fn=_ini_lskeys   ;;
+        sec)    fn=_ini_grepsec  ;;
+        values) fn=_ini_greppath ;;
+        1value) fn=_ini_greppath; limit="tail -1" ;;
+        *)      die "incorrect use of \`ini()\`"
+    esac
+    <"$MKIT_INI" $fn "$arg" | $limit
+}
+
+_ini_cat() {
+    #
+    # A no-op for text stream
+    #
+    while read line;
+    do
+        printf -- "%s\n" "$line"
+    done
+}
+
+_ini_expand() {
+    #
+    # Expand reference value (prefix only)
+    #
+    local line suffix ref value
+    while read line;                        # [foo:bar]/path
+    do
+        suffix="${line#\[*\]}"              # /path
+        ref="${line%$suffix}"               # [foo:bar]
+        ref="${ref%\]}"                     # [foo:bar
+        ref="${ref#\[}"                     # foo:bar
+        value="$(ini 1value "$ref")"        # foo_bar_value
+        printf -- "%s\n" "$value$suffix"    # foo_bar_value/path
+    done
+}
+
+_ini_grepkey() {
+    #
+    # Read key from a section
+    #
+    local wnt=$1
+    grep '.' \
+      | grep -v '\s*#' \
+      | sed -e 's/ *= */=/; s/ +$//; s/^//;' \
+      | grep -e "^$wnt=" \
+      | cut -d= -f2- \
+      | _ini_maybe_expand
+}
+
+_ini_greppath() {
+    #
+    # Read key from the right section
+    #
+    # E.g. `files:share:my/lib.sh` should read
+    #
+    #     [files:share]
+    #         my/lib.sh   = proj/my/lib.sh
+    #
+    local wnt="$1"
+    local wntkey="${wnt##*:}"
+    local wntsec="${wnt%:$wntkey}"
+    if test "$wntsec" = 'ENV';
+    then
+        local override="${!wntkey}"
+        if test -n "$override";
+        then
+            echo "$override"
+            return
+        fi
+    fi
+    _ini_grepsec "$wntsec" | _ini_grepkey "$wntkey"
+}
+
+_ini_grepsec() {
+    #
+    # Read one INI section
+    #
+    local wnt="$1"
+    local ok=false
+    grep '.' \
+      | grep -v '\s*#' \
+      | while read line;
+        do
+            case "$line" in
+                \[$wnt\]) ok=true;  continue ;;
+                \[*\])    ok=false; continue ;;
+            esac
+            $ok || continue
+            printf -- "%s\n" "$line"
+        done \
+      | sed -e 's/ *= */=/; s/ +$//; s/^//;'
+}
+
+_ini_lskeys() {
+    #
+    # List keys from a section
+    #
+    local sct="$1"
+    _ini_grepsec "$sct" | cut -d= -f1 | sort | uniq
+}
+
+_ini_maybe_expand() {
+    #
+    # Decide whether or not to expand
+    #
+    if test "$MKIT_INI_EXPAND" -gt 0;
+    then
+        MKIT_INI_EXPAND=$(( --MKIT_INI_EXPAND )) _ini_expand
+    else
+        _ini_cat
+    fi
+}

utils/mkit/include/mkit.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+. "$MKIT_DIR/include/build.sh"  || die "cannot import build.sh"
+. "$MKIT_DIR/include/deploy.sh" || die "cannot import deploy.sh"
+. "$MKIT_DIR/include/release.sh" || die "cannot import release.sh"
+. "$MKIT_DIR/include/ini.sh"    || die "cannot import ini.sh"
+
+MKIT_INI=${MKIT_INI:-mkit.ini}
+MKIT_INI_EXPAND=2
+MKIT_PKGNAME=$(ini 1value "ENV:PKGNAME")
+MKIT_PROJNAME=$(ini 1value "ENV:PROJNAME")
+MKIT_DEFAULT_MODE="644"
+
+mkit_init() {
+    #
+    # Do basic initialization
+    #
+    # Check for ini file, load variables from config.mk
+    #
+    test -f "$MKIT_INI" || die "cannot find mkit.ini: $MKIT_INI"
+    tmp=$(mktemp)
+    sed -e 's/ = /=/' < config.mk > "$tmp"
+    . "$tmp"
+    rm -f "$tmp"
+    test -n "$(tr -d '[:space:]' <<<"$MKIT_LOCAL")" \
+     || die "MKIT_LOCAL must be non-blank: '$MKIT_LOCAL'"
+}
+
+die() {
+    #
+    # Exit with message and non-zero exit status
+    #
+    echo "fatal: $*" >&2
+    exit 4
+}
+
+warn() {
+    #
+    # Print warning message
+    #
+    echo "$@" >&2
+}
+
+route() {
+    #
+    # Call correct function based on $1
+    #
+    case $1 in
+        build|build_manpages|clean|dist|install|release_?|uninstall)
+            $1
+            ;;
+        *)
+            echo "usage: $(basename "$0") build|clean|dist|install|uninstall" >&2
+    esac
+}

utils/mkit/include/release.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+__make_ver() {
+    local level=$1
+    local old=$2
+    local oldx=${old%.*.*}
+    local oldz=${old#*.*.}
+    local tmpy=${old%.*}
+    local oldy=${tmpy#*.}
+    case $level in
+        x) new="$((oldx+1)).0.0"            ;;
+        y) new="$oldx.$((oldy+1)).0"        ;;
+        z) new="$oldx.$oldy.$((oldz+1))"    ;;
+        *) die "invalid release level: $1"  ;;
+    esac
+    echo "$new"
+}
+
+__release() {
+    #
+    # Prepare release
+    #
+    # Span release routines: make a signed tag, check branch
+    # and update release branch
+    #
+    # FIXME: Using PRERELEASE as build.sh:get_version() does may not be
+    #        compatible with this release strategy
+    #
+    local level=$1
+    git rev-parse HEAD >&/dev/null || die "cannot release outside git repo"
+    local lastfile=$(git diff-tree --no-commit-id --name-only -r HEAD)
+    local lasttag=$(git tag | grep ^v | sort -V | tail -n1)
+    local changelog="$(git log --oneline "$lasttag..HEAD")"
+    local curbranch=$(git rev-parse --abbrev-ref HEAD)
+    local dirt="$(git diff --shortstat 2>/dev/null)"
+
+    local newver=$(__make_ver "$level" "${lasttag#v}")
+    local newtag=v$newver
+    local higher=$(echo -e "$oldtag\n$newtag" | sort -V | tail -n1)
+
+    test "$lastfile" = config.mk \
+     || die "last change must be version bump in config.mk"
+
+    test -n "$lasttag" \
+     || die "cannot find last tag"
+
+    test "$newtag" = "$higher" \
+     || die "generated tag looks older: $oldtag<$newtag"
+
+    grep -qw "$newver" config.mk \
+     || die "new version not in config.mk: $newver"
+
+    grep '^....... WIP ' <<<"$changelog" \
+     && die "WIP commit since $lasttag"
+
+    grep -qw "$curbranch" <<<"$RELSRC" \
+     || die "you are not on RELSRC branch: $RELSRC"
+
+    test -z "$dirt" \
+     || die "tree is dirty: $dirt"
+
+    # FIXME: Have user prepare proper message with changelog
+
+    set -e
+    git tag -m "$MKIT_PROJNAME $newtag - $CODENAME" "$newtag"
+    git branch -f "$RELDST" "$newtag"
+}
+
+release_x() {
+    __release x
+}
+
+release_y() {
+    __release y
+}
+
+release_z() {
+    __release z
+}

utils/mkit/make

@@ -0,0 +1,16 @@
+#!/bin/bash
+# mkit - simple install helper
+# See LICENSE file for copyright and license details.
+
+die() {
+    echo "$@" && exit 9
+}
+
+export MKIT_DIR=${MKIT_DIR:-mkit}
+export MKIT_LOCAL=${MKIT_LOCAL:-.mkit}
+
+. "$MKIT_DIR/include/mkit.sh" || die "failed to init; check if MKIT_DIR is set properly: $MKIT_DIR"
+
+mkit_init
+
+route "$@"

utils/mkit/mkit.mk

@@ -0,0 +1,41 @@
+# ffoo - Fastfoo - Bash dot on steroids
+# See LICENSE file for copyright and license details.
+
+export MKIT_DIR
+
+all: options build
+
+options:
+	@echo build options:
+	@echo "VERSION  = ${VERSION}"
+	@echo "PRERELEASE = ${PRERELEASE}"
+	@echo "PREFIX   = ${PREFIX}"
+
+build:
+	@$(MKIT_DIR)/make build
+
+manpages: build
+	@$(MKIT_DIR)/make build_manpages
+
+clean:
+	@$(MKIT_DIR)/make clean
+
+dist: clean
+	@$(MKIT_DIR)/make dist
+
+install: all
+	@$(MKIT_DIR)/make install
+
+release_x:
+	@$(MKIT_DIR)/make release_x
+
+release_y:
+	@$(MKIT_DIR)/make release_y
+
+release_z:
+	@$(MKIT_DIR)/make release_z
+
+uninstall:
+	@$(MKIT_DIR)/make uninstall
+
+.PHONY: all options clean dist install uninstall release_x release_y release_z

utils/tfkit/README.md

@@ -0,0 +1,15 @@
+TFKIT
+=====
+
+A simple portable (or rather "movable") test framework.
+
+ 1. Copy (or `git submodule`) *test* sub-folder to your place
+    where you want to run tests.
+
+ 2. Run your framework by `./tfkit/runtests`.
+
+ 3. Don't be surprised--you haven't added any tests yet :)
+
+ 4. Read tfkit/README.testing.md
+
+ 5. Start adding tests to `tests` folder

utils/tfkit/doc/README.md

@@ -0,0 +1,360 @@
+Tests
+=====
+
+Running tests is handled by tfkit/runtests:
+
+    $ tfkit/runtest [filter]
+
+*filter* is a regular expression to be applied to sub-test name, running
+only the matching ones.  See below for details.
+
+
+Writing tests
+-------------
+
+Tests can be written in any scripting language, although the built-in
+framework, written in Bash, provides some useful features for writing
+certain kind of relatively simple tests.
+
+The harness, though, assumes that:
+
+ *  Any direct sub-directory of `$TF_SUITE` directory ("tests" by default)
+    that contains at least *TF_RUN* executable becomes a test,
+
+ *  basename of this directory becomes the name of the test,
+
+ *  and return code from running the executable is reported
+    as result of the test, according to "Exit status" chapter below.
+
+
+Naming
+------
+
+Test name should start with name of the module that is tested and
+underscore.  If module name contains dots, they should be replaced with
+underscores as well.
+
+    core_sanity
+    mod_submod_function
+    ini_iniread
+
+are valid test names.
+
+
+Data
+----
+
+Should the test need any data, just leave it around in the test directory
+along with *TF_RUN*.
+
+Note that before running, the whole test directory is automatically
+copied to a temporary location (one per test), and should the test fail,
+copied back as a debugging artifact.  For this reason, *do not store
+huge amounts of data here*.  If you really need huge data, consider
+obtaining it (and throwing it away) within runtime of *TF_RUN*.
+
+
+Exit status
+-----------
+
+We try hard to follow this semantic:
+
+ *  *Zero* means *OK* -- test has been run and passed.
+
+ *  *One* means *Failure* -- test has been run but failed (e.g. found
+     a bug).
+
+ *  *Two* means *Bailout* --  test has decided not to run at all.
+
+ *  *Three* means *Error* -- there was error detected during execution,
+     but script was able to clean up properly.
+
+ *  *Four* means *Panic* -- there was other error but script *was not*
+     able to clean up properly.
+
+ *  Anything else should indicate other uncaught errors, including those
+    outside control of the program such as segfaults in the test code
+    or test being SIGKILLed.
+
+Notice that the higher the value is, the worse situation it indicates.
+Thus, if a test is composed of several sub-tests, you need to make sure
+to always **exit with the highest value** (subtest.sh does take care
+of this).
+
+See *common.sh* for functions and variables to help with handling exit
+statuses with this semantic.
+
+Also see Notes section for more details on exit statuses, including
+cheat sheet and dscussuion.
+
+
+Framework
+---------
+
+
+### harness.sh ###
+
+This part is not intended to be used in tests, but rather contains
+functions that help govern test discovery, preparation and execution as
+is described in previous chapters.  Feel free to poke around, of course.
+
+
+### subtest.sh ###
+
+As name suggests, this file defines few functions to handle subtests
+in *TF_RUN*.
+
+In order to make use of the subtests functionality, you will need to
+define two functions yourself:  `tf_enum_subtests` to enumerate names of
+tests you want to run, and `tf_name2cmd` to translate each name an actual
+command that would perform it and return with the the correct exit status.
+
+The minimal *TF_RUN* with two subtests could look like this:
+
+    #!/bin/bash
+
+    . $TF_DIR/include/subtest.sh
+
+    tf_enum_subtests() {
+        echo test1
+        echo test2
+        something && echo test3
+    }
+
+    tf_name2cmd() {
+        case $1 in
+            test1)  echo myprog foo ;;
+            test2)  echo myprog bar ;;
+        esac
+    }
+
+    tf_do_subtests
+
+At the end, `tf_do_subtests` acts as a launcher of the actual test.
+In short, it will
+
+ *  take each enumerated subtest from `tf_enum_subtests`,
+ *  source *TF_SETUP*, if such file is found,
+ *  translate te subtest name to a command,
+ *  launch the command,
+ *  source *TF_CLEANUP*, if such file is found,
+ *  and report "worst" exit status encountered.
+
+All but the first and last step is done by `tf_do_subtest`, so in some
+cases you may want to re-define this one as well.
+
+Note that subtest names need to be single words (`[a-zA-Z0-9_]`).
+
+
+### tools.sh ###
+
+This file contains various tools and utilities to help with testing.
+
+Curently there is only one function, `tf_testflt` designed to help write
+tests for simple unix filters.
+
+
+#### tf_testflt ####
+
+The idea is that tester specifies
+
+ *  test name,
+ *  command to launch the system under test,
+ *  a data stream to use as STDIN,
+ *  and expected STDOUT, STDERR, and exit status.
+
+and tf_testflt launches the command, collects tha data and evaluates
+and reports the result using unified diff.
+
+In its simplest form:
+
+    tf_testflt -n foo my_command arg
+
+the function will run `my_command arg` (not piping anything to it),
+and will expect it to finish with exit status 0 and empty both STDERR
+and STDOUT.
+
+Example of full form,
+
+    tf_testflt -n foo -i foo.in -O foo.stdout -E foo.stderr -S 2 myprog
+
+will pipe foo.in into `myprog`, expecting exit status of 2, and STDOUT and
+STDERR as above.  Notice that parameters specifying expected values are
+uppercase, and those specifying input values are lowercase.
+
+Specifying name is mandatory, because it's used in reporting messages,
+and as a basis for naming temporary result files: these are saved in
+*results* subdirectory and kept for further reference.
+
+
+### common.sh ###
+
+This includes simple functions and variables shared between both mentioned
+libraries.
+
+First group is designed to help support the exit status semantic:
+
+ *  The functions are `tf_exit_pass`, `tf_exit_fail`, `tf_exit_bailout`,
+    `tf_exit_error` and `tf_exit_panic` and each take any number of
+    parameters that are printed on stderr.
+
+ *  The variables are `TF_ES_OK`, `TF_ES_FAIL`, `TF_ES_BAILOUT`,
+    `TF_ES_ERROR` and `TF_ES_PANIC` and are supposed to be used with
+    `return` builtin, e.g. to return from `tf_exit_error`.
+
+Second group is useful to better control output:  functions `tf_warn`,
+`tf_debug` and `tf_think` are used to print stuff on STDERR.  Use of
+`tf_warn` is apparent, just as `tf_debug`, the latter being muted if
+`TF_DEBUG` is set to `false` (set it to `true` to turn on debugging).
+
+`tf_think` is used for progress info, and is muted unless `TF_VERBOSE`
+is set to `true`, which is by default.
+
+
+### Setup and cleanup ###
+
+Special files *TF_SETUP* and *TF_CLEANUP* (one of them or both) can be
+added along with *TF_RUN*.  These will be sourced before (*TF_SETUP*)
+and after every subtest (*TF_CLEANUP*).
+
+First, if any of these files are missing, it is considered as if the
+respective phase succeeded.  Second, if setup phase fails, test will
+be skipped and subtest exit status will be *TF_ES_BAILOUT*.   Last,
+if cleanup fails (no matter result of setup), subtests aborts with
+*TF_ES_PANIC* returned.  Be aware that in this case the actual test
+status, albeit useful, is lost.
+
+When coming from other test frameworks, this may feel harsh, but note
+that this has been designed with the idea that if a cleanup fails,
+it may render all further tests are automatically unsafe, because the
+environment is not as expected.
+
+To cope with this behavior, try to bear in mind following advice:
+
+ 1. Make sure you write setup/cleanup procedures with extreme care and
+    test them well.
+
+ 2. Do not do complicated and risky things in the setup/cleanup phases.
+
+ 3. If you need to do such things, consider doing them in the *TF_RUN*
+    instead of doing them for all subtests.
+
+ 4. You don't need to clean up everything, the contents of the testing dir
+    will be moved out from the test system.
+
+ 5. If there are scenarios you can safely fix or ignore, handle them in
+    a robust manner.
+
+
+Notes
+-----
+
+
+### bailout vs. `tf_enum_subtests` ###
+
+One more note to claify relation of bailout and `tf_enum_subtests`.
+As you may have noticed, there are two ways how to skip a test:
+return prematurely with `TF_ES_BAILOUT`, or suppress enumeration in
+`tf_enum_subtests`.  The problem is that the latter does not do anything
+to inform upper in the stack that a test has been skipped, which seems to
+break the principle described in the previous chapters.
+
+Don't confuse these mechanisms, though. Each is supposed to be used
+for distinct purpose.  Compare: by using the `tf_enum_subtests` you are
+saying that you actually **did not even want** to run the test in the
+first place.  By using `TF_ES_BAILOUT`, you are saying that you **wanted**
+to run the test but could not.
+
+A few common cases if that helps you:
+
+ *  If during the test you find out that for some reason it can't be
+    carried out (e.g. an external resource is not available, or
+    something outside the SUT is broken), use `TF_ES_BAILOUT`.
+
+ *  If you want to disable the test because for some long-term condition,
+    e.g. a known bug outside SUT but preventing execution of the test
+    is not fixed, use `tf_enum_subtests`.
+
+ *  If you want to filter out some sub-tests to only for some platforms,
+    e.g. 64-bit architecture, (IOW, you can safely check that a
+    sub-test would be totally pointless if run on this box), use
+    `tf_enum_subtests`.
+
+ *  If you want to disable (comment out test) that you might not have
+    implemented yet or is broken (and for some reason you still want
+    it to haunt the test code), use `tf_enum_subtests` and properly
+    comment the reasons in code.
+
+ *  If in doubt, use `TF_ES_BAILOUT`.
+
+
+### On exit statuses: three and above ###
+
+The difference in *error*, *panic* and higher values is subtle but
+important.  Follow me as I try to explain:
+
+ 1. If script has changed something on the system outside the working
+    directory, it is apparently expected to revert that change.
+
+ 2. Now if an error occurs, but the code responsible for cleaning up is
+    safely run, you can say there was *error but we have recovered*.
+
+ 3. But if the change can't be reverted safely, we know that we have
+    broken something and latter code may lead to weird results (including
+    masking bugs(!)), it's time to *panic* (in the code, not in real
+    life ;))
+
+ 4. And then there are corner cases like a bug in the script, OOM kill
+    or timeout when the status will be different and not really controlled
+    by the script.  Such cases will have to be treated the same way as
+    the "panic" case, but...
+
+ 5. the use of *panic* adds hint that the status has been set consciously
+    by the script, albeit exiting "in a hurry"--without proper clean up.
+
+Unfortunately there will be cases like above but with the error code less
+than four.   Example is a bash script syntax error, which returns 2, or
+Python exception which returns 1.  Yes, in such cases the information
+conveyed by the exit status is wrong and you should do everything to
+avoid it.
+
+Possibilities like "test has passed but then something blew up" exist,
+but conveying this information is responsibility of the test output.
+
+Following table can be used as a cheat-sheet:
+
+    .---------------------------------------------------------------.
+    | e |    state of         |                                     |
+    | s |---------------------| script says                         |
+    |   | SUT   | environment |                                     |
+    |---|-------|-------------|-------------------------------------|
+    | 0 | OK    | safe        | test passed, everything worked fine |
+    | 1 | buggy | safe        | test failed, everything worked fine |
+    | 2 | ???   | safe        | I decided not to run the test       |
+    | 3 | ???   | safe        | Something blew up but I managed to  |
+    |   |       |             | clean up (I promise!)               |
+    | 4 | ???   | broken      | Something blew up and I rushed out  |
+    |   |       |             | in panic                            |
+    | * | ???   | broken      | ...nothing (is dead)                |
+    '---------------------------------------------------------------'
+
+As you can see, following this semantic allows us to see both the state
+of the system under test (SUT) *and* the environment.
+
+Following table illustrates how different statuses map to different
+scenarios with regard to test result as well as state of the environment:
+
+    .--------------------------------------------------.
+    | environment |  test result   |  test result      |
+    |             | pass fail unkn | pass fail unkn    |
+    |-------------|----------------|-------------------|
+    | clean(ed)   |  0    1    3   |  OK  FAIL ERROR   |
+    | untouched   |  ~    ~    2   |  ~    ~   BAILOUT |
+    | mess        |  ~    ~    4   |  ~    ~   PANIC   |
+    | ?! (trap)   |  ~    ~    5   |  ~    ~   ~       |
+    | ?! (sig 9)  |  ~    ~    137 |  ~    ~   ~       |
+    | ?! (aliens) |  ~    ~    ?   |  ~    ~   ~       |
+    '-------------|----------------|-------------------|
+                  |  exit status   |  human-readable   |
+                  |                |  name (TF_ES_*)   |
+                  '------------------------------------'

utils/tfkit/doc/templates/grep_engine/TF_RUN

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+. "$TF_DIR/include/subtest.sh"
+. "$TF_DIR/include/tools.sh"
+
+tf_enum_subtests() {
+    echo fixed
+    echo basic
+    echo extended
+#   echo perl       # TODO: write test
+}
+
+tf_name2cmd() {
+    local name=$1
+    local t_in="test/ALL.stdin"
+    local o_out="oracle/$name.stdout"
+    local args
+    case $name in
+        fixed)       args="-F 'The mask *.* matches all.'" ;;
+        basic)       args="-G 'he.*'" ;;
+        extended)    args="-P '.*og?g'" ;;
+    esac
+    echo "tf_testflt -n $name -i $t_in -O $o_out grep $args"
+}
+
+tf_do_subtests

utils/tfkit/doc/templates/grep_engine/oracle/basic.stdout

@@ -0,0 +1,3 @@
+Linda and Nina work together.
+The mask *.* matches all.
+The mask *.docx matches Word documents.

utils/tfkit/doc/templates/grep_engine/oracle/extended.stdout

@@ -0,0 +1,3 @@
+Alice uses Google.
+Bob wears goggles.
+Linda and Nina work together.

utils/tfkit/doc/templates/grep_engine/oracle/fixed.stdout

@@ -0,0 +1 @@
+The mask *.* matches all.

utils/tfkit/doc/templates/grep_engine/test/ALL.stdin

@@ -0,0 +1,6 @@
+Alice uses Google.
+Bob wears goggles.
+Joe and John are friends.
+Linda and Nina work together.
+The mask *.* matches all.
+The mask *.docx matches Word documents.

utils/tfkit/include/common.sh

@@ -0,0 +1,135 @@
+#
+# Variables to hold exit status semantic
+#
+TF_ES_OK=0
+TF_ES_FAIL=1
+TF_ES_BAILOUT=2
+TF_ES_ERROR=3
+TF_ES_PANIC=4
+
+#
+# Option to turn on/off verbosity
+#
+TF_VERBOSE=${TF_VERBOSE:-true}
+
+#
+# Option to turn on/off debug mode
+#
+TF_DEBUG=${TF_DEBUG:-false}
+
+#
+# Regex to filter test names to run
+#
+TF_FILTER_TEST="${TF_FILTER_TEST:-.*}"
+
+#
+# Regex to filter subtest names to run
+#
+TF_FILTER_SUBTEST="${TF_FILTER_SUBTEST:-.*}"
+
+#
+# Enable color?
+#
+TF_COLOR=${TF_COLOR:-true}
+
+#
+# Color definition variables
+#
+TF_COLOR_BLACK="\033[0;30m"
+TF_COLOR_RED="\033[0;31m"
+TF_COLOR_GREEN="\033[0;32m"
+TF_COLOR_YELLOW="\033[0;33m"
+TF_COLOR_BLUE="\033[0;34m"
+TF_COLOR_MAGENTA="\033[0;35m"
+TF_COLOR_CYAN="\033[0;36m"
+TF_COLOR_WHITE="\033[0;37m"
+TF_COLOR_LBLACK="\033[1;30m"
+TF_COLOR_LRED="\033[1;31m"
+TF_COLOR_LGREEN="\033[1;32m"
+TF_COLOR_LYELLOW="\033[1;33m"
+TF_COLOR_LBLUE="\033[1;34m"
+TF_COLOR_LMAGENTA="\033[1;35m"
+TF_COLOR_LCYAN="\033[1;36m"
+TF_COLOR_LWHITE="\033[1;37m"
+TF_COLOR_NONE="\033[1;0m"
+
+
+tf_exit_ok() {
+    #
+    # Exit with OK status
+    #
+    exit $TF_ES_OK
+}
+
+tf_exit_fail() {
+    #
+    # Warn $1 and exit with status FAIL
+    #
+    tf_warn "$@"
+    exit $TF_ES_FAIL
+}
+
+tf_exit_bailout() {
+    #
+    # Warn $1 and exit with status FAIL
+    #
+    tf_warn "$@"
+    exit $TF_ES_BAILOUT
+}
+
+tf_exit_error() {
+    #
+    # Warn $1 and exit with status FAIL
+    #
+    tf_warn "$@"
+    exit $TF_ES_ERROR
+}
+
+tf_exit_panic() {
+    #
+    # Warn $1 and exit with status FAIL
+    #
+    tf_warn "$@"
+    exit $TF_ES_PANIC
+}
+
+tf_debug() {
+    #
+    # Emit debug message
+    #
+    $TF_DEBUG || return 0
+    local msg
+    for msg in "$@";
+    do
+        $TF_COLOR && echo -ne "$TF_COLOR_CYAN" >&2
+        echo "||| $msg" >&2;
+        $TF_COLOR && echo -ne "$TF_COLOR_NONE" >&2
+    done
+}
+
+tf_think() {
+    #
+    # Emit status/progress message
+    #
+    $TF_VERBOSE || return 0
+    local msg
+    for msg in "$@";
+    do
+        $TF_COLOR && echo -ne "$TF_COLOR_LBLACK" >&2
+        echo "$pfx$msg$sfx" >&2;
+        $TF_COLOR && echo -ne "$TF_COLOR_NONE" >&2
+    done
+}
+
+tf_warn() {
+    #
+    # Emit warning
+    #
+    local msg
+    for msg in "$@";
+    do
+        $TF_COLOR && echo -ne "$TF_COLOR_LRED" >&2
+        echo "$msg" >&2;
+        $TF_COLOR && echo -ne "$TF_COLOR_NONE" >&2
+    done
+}

utils/tfkit/include/harness.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# ffoo test harness
+# See LICENSE file for copyright and license details.
+
+. "$TF_DIR/include/common.sh"
+
+#
+# Default path to header generator
+#
+__TF_HDRCMD="$TF_SUITE/TF_HEADER"
+
+
+__tf_collect_if_needed() {
+    #
+    # Collect artifact if exit status suggests it
+    #
+    # Use test exit status $1 to help decide if artifacts are
+    # needed, and collect them if so.
+    #
+    # If TF_COLLECT is set to "always", collect regardless of
+    # the status.  If set to "never", do not collect anything.
+    # The default setting, "auto" collects unless status is 0
+    # or 1 (pass or bailout); in that case do nothing.
+    #
+    local tes=$1    # test exit status
+    local will      # should we collect artifacts?
+    case "$TF_COLLECT:$tes" in
+        always:*)   will=true ;;
+        never:*)    will=false ;;
+        auto:0)     will=false ;;
+        auto:2)     will=false ;;
+        auto:*)     will=true ;;
+        *)          tf_exit_bailout "bad value of TF_COLLECT: $TF_COLLECT" ;;
+    esac
+    $will || return 0
+    mkdir -p "$artifact_dir/$stamp"
+    cp -r "$tmpdir"/* "$artifact_dir/$stamp"
+}
+
+__tf_default_header() {
+    #
+    # Create default header
+    #
+    echo "(add $__TF_HDRCMD executable for own header)"
+}
+
+__tf_header() {
+    #
+    # create header to add to output before test
+    #
+    local field
+    local hdrcmd="$__TF_HDRCMD"
+    test -x "$hdrcmd" || hdrcmd="__tf_default_header"
+    $hdrcmd \
+      | while read field;
+        do
+            test -n "$field" || break
+            tf_think "# $field"
+        done
+    tf_think ""
+}
+
+tf_enum_tests() {
+    #
+    # List what looks like test; relative to $TF_SUITE
+    #
+    tf_debug "TF_SUITE='$TF_SUITE'"
+    test -d "$TF_SUITE" || return 0
+    find -L \
+        "$TF_SUITE" \
+        -mindepth 2 \
+        -maxdepth 2 \
+        -type f \
+        -perm /111 \
+        -name TF_RUN \
+      | cut -d/ -f2
+}
+
+tf_run_tests() {
+    #
+    # Discover and run tests
+    #
+    local es=0          # overall exit status
+    local tmpdir=""     # test temporary dir
+    local tname=""      # test name
+    local tes=0         # test result
+    local stamp=""      # test stamp to use as artifact name
+    local tf_dir tf_suite   # to keep absolute paths for TF_RUN
+    local artifact_dir="$(readlink -f "$TF_ARTIFACTS")"
+    __tf_header
+    tf_debug "TF_VERSION='$TF_VERSION'"
+    tf_dir="$(readlink -f "$TF_DIR")"
+    tf_suite="$(readlink -f "$TF_SUITE")"
+    es=0
+    for tname in $(tf_enum_tests | grep -e "$TF_FILTER_TEST");
+    do
+        tf_think "... $tname"
+        tmpdir=$(mktemp -d)
+        stamp=$(date "+artifacts-$tname-%Y%m%d-%H%M%S")
+        cp -r "$TF_SUITE/$tname/"* "$tmpdir"
+        pushd "$tmpdir" >/dev/null
+            TF_DIR="$tf_dir" TF_SUITE=$tf_suite TF_TNAME="$tname" \
+                ./TF_RUN
+            tes=$?
+            __tf_collect_if_needed $tes
+            test $tes -gt $es && es=$tes
+        popd >/dev/null
+        rm -rf "$tmpdir"
+        if test $tes -eq 0;
+        then
+            tf_think "''' $tname ($tes)"
+        else
+            tf_warn "??? $tname ($tes)"
+        fi
+    done
+    return $es
+}

utils/tfkit/include/subtest.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+. "$TF_DIR/include/common.sh"
+
+tf_enum_subtests() {
+    #
+    # Stub: enumerate subtests
+    #
+    tf_warn "implement tf_enum_subtests()!"
+    return $TF_ES_ERROR
+}
+
+tf_name2cmd() {
+    #
+    # Stub: expand test name to test command
+    #
+    tf_warn "implement tf_name2cmd()!"
+    return $TF_ES_ERROR
+}
+
+tf_do_subtest() {
+    #
+    # Run single subtest inc. setup/cleanup if present
+    #
+    local subtname="$1"     # this subtest name
+    local ses=0             # subtest exit status
+    local tcmd=""           # test command
+    local setup=true        # setup command
+    local cleanup=true      # cleanup command
+    test -f TF_SETUP   && setup=". TF_SETUP"
+    test -f TF_CLEANUP && cleanup=". TF_CLEANUP"
+    if $setup;
+    then
+        tcmd="$(tf_name2cmd "$subtname")"
+        tf_debug "tcmd='$tcmd'"
+        $tcmd; ses=$?
+    else
+        tf_warn "setup phase failed, skipping: $subtname"
+        ses=$TF_ES_ERROR
+    fi
+    if ! $cleanup;
+    then
+        tf_warn "cleanup phase failed: $subtname"
+        ses=$TF_ES_PANIC
+    fi
+    return $ses
+}
+
+tf_do_subtests() {
+    #
+    # Run all subtests and return highest status
+    #
+    local es=0              # final exit status ("worst" of subtests)
+    local subtname=""       # one subtest name
+    local tes=""            # one subtest exit status
+    local enumd=TF_ENUMERATED_SUBTESTS
+    local fltrd=TF_FILTERED_SUBTESTS
+    tf_enum_subtests >$enumd    || { tf_warn "error enumerating subtests"; return $TF_ES_BAILOUT; }
+    test -s $enumd              || { tf_warn "no subtests enumerated";     return $TF_ES_BAILOUT; }
+    grep -e "$TF_FILTER_SUBTEST" $enumd > $fltrd
+    test -s $fltrd  || tf_debug "TF_FILTER_SUBTEST ate everything: $TF_FILTER_SUBTEST"
+    for subtname in $(<$fltrd);
+    do
+        export TF_SUBTNAME=$subtname
+        tf_think "::: $TF_TNAME::$TF_SUBTNAME"
+        tf_do_subtest "$TF_SUBTNAME";
+        tes=$?
+        test $tes -gt $es            && es=$tes
+        test $tes -gt $TF_ES_OK      && tf_warn "!!! $TF_TNAME::$TF_SUBTNAME ($tes)"
+        test $tes -gt $TF_ES_BAILOUT && break
+    done
+    return $es
+}

utils/tfkit/include/tools.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+. "$TF_DIR/include/common.sh"
+
+# 1. exec: [test] -> [result]
+# 2. eval:           [result] == [oracle]
+
+tf_testflt() {
+    #
+    # Run a simple test for a unix filter
+    #
+    #     tf_testflt -n foo [-i foo.stdin] \
+    #                [-O ffoo.stdout] [-E ffoo.stderr] [-S 3] \
+    #                cmd arg...
+    #
+    # Will drop *result/NAME.stdout* and *result/NAME.stderr* (intentionally
+    # not cleaning up).
+    #
+
+    # defaults
+    #
+    local t_in="/dev/null"      # test: stdin
+    local t_name=""             # test: name
+                                # command is "$@" after arg parsing
+    local t_es="0"              # final test exit status
+    local o_out="/dev/null"     # oracle: stdout
+    local o_err="/dev/null"     # oracle: stderr
+    local o_es="0"              # oralce: exit status
+    local r_out r_err r_es      # result: ^ ^ ^ those 3
+
+    # get args
+    #
+    local orig_args="$0 $*"
+    tf_debug "orig_args=$orig_args"
+    local arg_err=false
+    while true; do case "$1" in
+        -i) t_in="$2";          shift 2 || { arg_err=true; break; } ;;
+        -n) t_name="$2";        shift 2 || { arg_err=true; break; } ;;
+        -O) o_out="$2";         shift 2 || { arg_err=true; break; } ;;
+        -E) o_err="$2";         shift 2 || { arg_err=true; break; } ;;
+        -S) o_es="$2";          shift 2 || { arg_err=true; break; } ;;
+        --)                     shift; break ;;
+        "")                            break ;;
+        -*) tf_warn "wrong testcli arg: $1"; return $TF_ES_BAILOUT ;;
+        *)                             break ;;
+    esac done
+    $arg_err && { tf_warn "error parsing arguments: $orig_args"; return $TF_ES_BAILOUT; }
+    tf_debug "t_in='$t_in'"
+    tf_debug "t_name='$t_name'"
+    tf_debug "o_out='$o_out'"
+    tf_debug "o_err='$o_err'"
+    tf_debug "o_es='$o_es'"
+    tf_debug "test command: $*"
+    test "$t_in" = "-" && t_in=/dev/stdin   # works better for check below
+    test -z "$t_name"  && { tf_warn "missing test name"             ; return $TF_ES_BAILOUT; }
+    test -z "$1"       && { tf_warn "missing test command"          ; return $TF_ES_BAILOUT; }
+    test -r "$t_in"    || { tf_warn "missing input file: $t_in"     ; return $TF_ES_BAILOUT; }
+    test -e "$o_out"   || { tf_warn "missing oracle stdout: $o_out" ; return $TF_ES_BAILOUT; }
+    test -e "$o_err"   || { tf_warn "missing oracle stderr: $o_err" ; return $TF_ES_BAILOUT; }
+    test "$o_es" -ge 0 || { tf_warn "invalid oracle status: $o_es"  ; return $TF_ES_BAILOUT; }
+
+    # prepare
+    #
+    mkdir -p result
+    r_out="result/$t_name.stdout"
+    r_err="result/$t_name.stderr"
+    tf_debug "r_out='$r_out'"
+    tf_debug "r_err='$r_err'"
+    touch "$r_out" || { tf_warn "cannot create tmp file: $r_out" ; return $TF_ES_BAILOUT; }
+    touch "$r_err" || { tf_warn "cannot create tmp file: $r_err" ; return $TF_ES_PANIC; }
+
+    # run
+    #
+    ( <"$t_in" eval "$@" >"$r_out" 2>"$r_err" ); r_es=$?
+    tf_debug "r_es='$r_es'"
+
+    # eval/report/exit
+    #
+    test $r_es = $o_es || { tf_warn "bad exit status: $r_es (need $o_es)" ; t_es=$TF_ES_FAIL; }
+    diff -u "$o_err" "$r_err" || t_es=$TF_ES_FAIL
+    diff -u "$o_out" "$r_out" || t_es=$TF_ES_FAIL
+    return $t_es
+}

utils/tfkit/runtests

@@ -0,0 +1,45 @@
+#!/bin/bash
+# ffoo test framework
+# See LICENSE file for copyright and license details.
+
+export TF_VERSION="0.0.4"
+
+die() {
+    echo "$@" && exit 9
+}
+
+export LC_ALL=C
+export TF_DIR=${TF_DIR:-$(dirname "$0")}
+export TF_SUITE="${TF_SUITE:-tests}"
+export TF_ARTIFACTS="${TF_ARTIFACTS:-tfkit-artifacts}"
+export TF_COLLECT="${TF_COLLECT:-auto}"
+
+
+usage() {
+    echo "usage: $(basename "$0") [-c|-C] [-t tests_re] [-s subtest_re] [-p binpath] [-v] [-d]" >&2
+    exit 2
+}
+
+version() {
+    echo "TFKit (A trivial test kit) $TF_VERSION"
+    exit 0
+}
+
+while true; do case "$1" in
+    -c|--collect)           TF_COLLECT=always; shift ;;
+    -C|--no-collect)        TF_COLLECT=never; shift ;;
+    -t|--filter-test)       export TF_FILTER_TEST="$2"; shift 2 ;;
+    -s|--filter-subtest)    export TF_FILTER_SUBTEST="$2"; shift 2 ;;
+    -p|--prefix)            export PATH="$(readlink -f "$2")/bin:$PATH"; shift 2 ;;
+    -d|--debug)             export TF_DEBUG=true; shift ;;
+    -v|--verbose)           export TF_VERBOSE=true; shift ;;
+    --version)              version ;;
+    --version-semver)       echo "$TF_VERSION"; exit 0 ;;
+    "") break ;;
+    *)  usage ;;
+esac done
+
+. "$TF_DIR/include/harness.sh" \
+ || die "cannot import harness; is TF_DIR set properly?: $TF_DIR"
+
+time tf_run_tests

utils/tfkit/tfkit.mk

@@ -0,0 +1,18 @@
+# tfkit - Fastfoo - Bash dot on steroids
+# See LICENSE file for copyright and license details.
+#
+# To use, set TF_DIR (make macro) in your Makefile and
+# include this file.
+#
+# Adds `test` target.
+#
+# Note that running "runtests" directly should have the same
+# effect but you will need to set TF_DIR and/or TF_SUITE properly
+# unless they are default (see README)
+
+export TF_DIR
+
+test:
+	@$(TF_DIR)/runtests
+
+.PHONY: test