Move most sfdoc code to separate module

Allow for re-use and more independent testing.

packaging/debian/control

@@ -82,6 +82,23 @@ Description: Logging Shellfu/Bash module
  .
  This sub-package contains 'pretty', universal Shellfu logging module.
 
+Package: shellfu-bash-sfdoc
+Architecture: all
+Depends: shellfu-bash, shellfu-bash-pretty
+Description: Shellfu/Bash module to export docs from Shellfu modules
+ Shellfu is an attempt to add modularity to your shell scripts.
+ .
+ With Shellfu you can develop your shell modules separately from your
+ scripts, and test, use, explore or study them without need to be aware
+ of details such as where the actual files are placed.
+ .
+ Shellfu is mostly intended for cases when there's need for non-trivial
+ amount of reliable body of shell scripts, and access to advanced modular
+ languages such as Python or Perl is limited.
+ .
+ This sub-package contains Shellfu/Bash module to export documentation
+ from other Shellfu modules that follow Shellfu coding style.
+
 Package: shellfu-devel
 Architecture: all
 Depends: shellfu-bash-pretty

packaging/debian/shellfu-bash-sfdoc.install

@@ -0,0 +1 @@
+/usr/share/shellfu/include-bash/sfdoc.sh

packaging/template.spec

@@ -56,6 +56,15 @@ Obsoletes: shellfu-bash-core < 0.10
 %description bash-pretty
 This sub-package contains 'pretty', universal Shellfu logging module.
 
+%package bash-sfdoc
+Summary: Shellfu/Bash module to export docs from Shellfu modules
+Requires: perl
+Requires: shellfu-bash
+Requires: shellfu-bash-pretty
+%description bash-sfdoc
+This sub-package contains Shellfu/Bash module to export documentation
+from other Shellfu modules that follow Shellfu coding style.
+
 %package devel
 Summary: Essential developer tools
 Requires: shellfu-bash-pretty
@@ -140,6 +149,9 @@ make test \
 %{_datadir}/%{name}/include-bash/_pretty_plain.sh
 %{_datadir}/%{name}/include-bash/pretty.sh
 
+%files bash-sfdoc
+%{_datadir}/%{name}/include-bash/sfdoc.sh
+
 %files devel
 %{_bindir}/sfembed
 

src/bin/sfdoc

@@ -4,6 +4,7 @@
 . "$(sfpath)" || exit 3
 
 shellfu import pretty
+shellfu import sfdoc
 
 
 usage() {
@@ -39,487 +40,6 @@ usage() {
         "in man(1) pager."
 }
 
-export_as_manpage() {
-    #
-    # Export module doc as manpage
-    #
-    local module=$1
-    local mfile
-    mfile=$(select_mfile "$module") || return 3
-    mver="v$(shellfu _read_directive "module-version" "$mfile" 2>/dev/null)"
-    test "$mver" == "v" && mver="(no version)"
-    export_as_pod "$module" \
-      | pod2man \
-            --name "${RealModuleName:-${module^^}}" \
-            --center "User Contributed Shellfu Documentation" \
-            --section 3x \
-            --date "$(date -I -r "$mfile")" \
-            --release "$module $mver" \
-            --utf8 -q \`
-}
-
-export_as_markdown() {
-    #
-    # Export module doc as Markdown
-    #
-    local module=$1
-    local module_name=${RealModuleName:-$module}
-    local variable
-    local function
-    local vheader_done=false
-    local fheader_done=false
-    debug -v module
-    echo "$module_name"
-    echo "$module_name" | tr -c '\n' '[=*]'
-    echo
-    get_doc "$module" | md_escapes
-    echo
-    for variable in $(sf_list v "$module" | cut -d. -f2);
-    do
-        debug -v variable
-        $vheader_done || {
-            echo
-            echo Variables
-            echo ---------
-            vheader_done=true
-        }
-        echo
-        echo "### \`\$$variable\` ###"
-        echo ""
-        get_doc "$module" "v:$variable" | md_escapes
-        echo
-    done
-    for function in $(sf_list f "$module" | cut -d. -f2);
-    do
-        debug -v function
-        $fheader_done || {
-            echo
-            echo Functions
-            echo ---------
-            fheader_done=true
-        }
-        echo
-        echo "### \`$function()\` ###"
-        echo ""
-        get_doc "$module" "f:$function" | md_escapes
-        echo
-    done
-}
-
-export_as_pod() {
-    #
-    # Export module doc as POD
-    #
-    local module=$1
-    local variable
-    local function
-    local vheader_done=false
-    local fheader_done=false
-    local indented=false
-    debug -v module
-    echo "true <<'=cut'"
-    echo "=pod"
-    echo
-    echo "=head1 NAME"
-    echo
-    echo "${RealModuleName:-$module} - $(get_doc "$module" | head -1)"
-    echo
-    echo "=head1 DESCRIPTION"
-    echo
-    get_doc "$module"
-    echo
-    for variable in $(sf_list v "$module" | cut -d. -f2);
-    do
-        debug -v variable
-        $vheader_done || {
-            echo
-            echo "=head1 VARIABLES"
-            vheader_done=true
-            echo
-            echo "=over 8"
-            echo
-            indented=true
-        }
-        echo
-        echo "=item I<\$$variable>"
-        echo
-        get_doc "$module" "v:$variable"
-        echo
-    done
-    $indented && {
-        echo "=back"
-        echo
-    }
-    for function in $(sf_list f "$module" | cut -d. -f2);
-    do
-        debug -v function
-        $fheader_done || {
-            echo
-            echo "=head1 FUNCTIONS"
-            fheader_done=true
-            echo
-            echo "=over 8"
-            echo
-            indented=true
-        }
-        echo
-        echo "=item I<$function()>"
-        echo
-        get_doc "$module" "f:$function"
-        echo
-    done
-    $indented && {
-        echo "=back"
-        echo
-    }
-    echo "=encoding $Encoding"
-    echo "=cut"
-}
-
-filter_body() {
-    #
-    # Filter part $1 of body on stdin
-    #
-    local part=$1
-    case "$part" in
-        f:*)  filter_fun "${part:2}" ;;
-        v:*)  filter_var "${part:2}" ;;
-        *) warn "bug: invalid part specification $part" ;;
-    esac
-}
-
-filter_doc() {
-    #
-    # Filter docstring for part $1 from object body on stdin
-    #
-    local part=$1
-    case $part in
-        "")  filter_mdoc ;;
-        v:*)  filter_vdoc ;;
-        f:*)  filter_fdoc ;;
-        *) warn "bug: invalid part specification: $part" ;;
-    esac
-}
-
-filter_fun() {
-    #
-    # From module body on stdin, filter out function named $1
-    #
-    # Function definition should look like:
-    #
-    #     foo_fun() {
-    #         foo=$1
-    #     }
-    #
-    # That is,
-    #
-    #  *  no `function` keyword,
-    #  *  name starts the first line,
-    #  *  name is followed by '() {' with no additional spaces,
-    #  *  function end is denoted by line with a single '}'.
-    #
-    local fn_name=$1
-    name=$fn_name perl -we '
-        undef $/;
-        my $name = $ENV{name};
-        my ($fbody) = <> =~ m/^($name\(\) \{.*?^\}$)/ms;
-        print "$fbody\n" if defined $fbody;
-    '
-}
-
-filter_fdoc() {
-    #
-    # Filter docstring from function body on stdin
-    #
-    # Look for:
-    #
-    #  1. line "^    #$" to open docstring - this must be
-    #     the first one after function name (`fun() {`)
-    #  2. block of consecutive docstring lines (i.e. matching
-    #     "^    # " or "^    #$"
-    #  3. last "^    #$" line to close docstring.
-    #  4. Next line must not match any of the patterns above.
-    #
-    # For example, stdin like this:
-    #
-    #     myfun() {
-    #         #
-    #         # Do my thing with foo $1
-    #         #
-    #         # Detailed description, possibly spanning
-    #         # several paragraph.
-    #         #
-    #         # The format here should be Markdown.
-    #         #
-    #         local foo=$1
-    #         printf %p "${foo:4}"
-    #     }
-    #
-    # would be read as:
-    #
-    #     Do my thing with foo $1
-    #
-    #     Detailed description, possibly spanning
-    #     several paragraph.
-    #
-    #     The format here should be Markdown.
-    #
-    # However if we added following line right before the `local`
-    # declaration
-    #
-    #         #FIXME: TODO/FIXME lines are not dropped properly
-    #
-    # it **will not** be considered part of the docstring. This is
-    # to allow for special comments tools like TODO/FIXME or
-    # data for lint-like tools that are not part of function
-    # description.
-    #
-    perl -we '
-        my $isdoc;
-        while (<>) {
-            if (m/^    #$/) {
-                $isdoc = 1;
-            } elsif (m/^    [^#]/) {
-                exit;
-            }
-            next unless $isdoc;
-            s/^    //;
-            print;
-        }' | strip_doc
-}
-
-filter_mdoc() {
-    #
-    # From module body on stdin, filter module doc
-    #
-    # Example:
-    #
-    #     #!/bin/bash
-    #     #
-    #     # legal stuff
-    #
-    #     shellfu import irrelevant_stuff
-    #
-    #     #
-    #     # Module is cool this
-    #     #
-    #     # Ok so this is my
-    #     # "module", heh...
-    #     #
-    #
-    perl -we '
-        sub ok_chunk {
-            my $chunk = shift;
-            my @lines = split "\n", $_;
-            chomp @lines;
-            return 0 unless (defined $lines[0] and $lines[0] eq "#");
-            return 0 unless $lines[$#lines] eq "#";
-            foreach (@lines) {
-                return 0 unless (m/^#$/ or m/^# /);
-            }
-            return 1
-        }
-        my @chunks;
-        my @lines;
-        $/ = "\n\n";
-        @chunks = <STDIN>;
-        foreach (@chunks) {
-            if (ok_chunk "$_") {
-                s/\n+$//;
-                print "$_\n";
-                exit 0;
-            }
-        }
-        exit 1
-    ' | strip_doc
-}
-
-filter_var() {
-    #
-    # From module body on stdin, filter variable definition named $1
-    #
-    # Look for:
-    #
-    #  1. Empty line, followed by line with single hash sign
-    #  2. Set of consecutive docstring lines (start with '# ')
-    #  3. Another "#" followed by one or more variable definitions (start
-    #     with $name or \w+, followed by equal sign), at least one of which
-    #     must be $1
-    #  4. Empty line or EOF
-    #
-    # and return what is found.
-    #
-    # For example:
-    #
-    #     #
-    #     # My variable
-    #     #
-    #     my_var=foo
-    #
-    # is found if $1 is 'my_var'
-    #
-    # A more advanced example:
-    #
-    #     #
-    #     # Bar variables
-    #     #
-    #     # These serve similar purpose; the difference is
-    #     # obvious
-    #     #
-    #     bar_1=one
-    #     bar_2=two
-    #     bar_3=three
-    #
-    # is found if $1 is either 'bar_1', 'bar_2' or 'bar_3'.  This allows
-    # for a docstring being shared among closely related variables.
-    #
-    local oname=$1    # object name
-    local state       # void, edge, dstr, vstr, junk
-    local o_end       # object ended; is complete
-    local cache       # buffer of hope
-    # parse and keep each var docstring in cache;
-    # then decide whether or not print the cache
-    cache=$(mktemp -t shellfu_doc.filter_var.XXXXXXXX)
-    state=junk
-    o_end=false
-    while read -r line;
-    do
-        case $state:$line in
-            # looks like void
-            void:)          state=void; o_end=false ;;
-            edge:)          state=void; o_end=true  ;;
-            dstr:)          state=void; o_end=true  ;;
-            vstr:)          state=void; o_end=true  ;;
-            junk:)          state=void; o_end=false ;;
-            # looks like edge
-            void:"#")       state=edge; o_end=false ;;
-            edge:"#")       state=dstr; o_end=false ;;
-            dstr:"#")       state=dstr; o_end=false ;;
-            vstr:"#")       state=dstr; o_end=false ;;
-            junk:"#")       state=junk; o_end=false ;;
-            # looks like docstring
-            void:"# "*)     state=junk; o_end=false ;;
-            edge:"# "*)     state=dstr; o_end=false ;;
-            dstr:"# "*)     state=dstr; o_end=false ;;
-            vstr:"# "*)     state=dstr; o_end=false ;;
-            junk:"# "*)     state=junk; o_end=false ;;
-            # looks like junk
-            void:"#"*)      state=junk; o_end=false ;;
-            edge:"#"*)      state=junk; o_end=false ;;
-            dstr:"#"*)      state=junk; o_end=false ;;
-            vstr:"#"*)      state=junk; o_end=false ;;
-            junk:"#"*)      state=junk; o_end=false ;;
-            # looks like variable string
-            void:*=*)       state=vstr; o_end=false ;;
-            edge:*=*)       state=junk; o_end=false ;;
-            dstr:*=*)       state=vstr; o_end=false ;;
-            vstr:*=*)       state=vstr; o_end=false ;;
-            junk:*=*)       state=junk; o_end=false ;;
-            # looks like something else
-            *)              state=junk; o_end=false ;;
-        esac
-        case $state in
-            edge|dstr|vstr) echo "$line" >> "$cache" ;;
-        esac
-        if $o_end;
-        then
-            if grep -q "^$oname=" "$cache";   # it's our wanted object
-            then
-                cat "$cache"
-                rm "$cache"
-                return 0
-            fi
-            rm "$cache"
-        fi
-    done
-    rm "$cache"
-    return 1
-}
-
-filter_vdoc() {
-    #
-    # From variable definition body on stdin, filter doc
-    #
-    # Example:
-    #
-    #     #
-    #     # Bar variables
-    #     #
-    #     # These serve similar purpose; the difference is
-    #     # obvious
-    #     #
-    #     bar_1=one
-    #     bar_2=two
-    #     bar_3=three
-    #
-    grep '^#' | strip_doc
-}
-
-get_doc() {
-    #
-    # Show doc for part $2 of module $1
-    #
-    local module=$1
-    local part=$2
-    get_part "$module" "$part" | filter_doc "$part"
-}
-
-get_part() {
-    #
-    # Print part $2 (f|v:name) of module $1
-    #
-    # Part has format
-    #
-    #     TYPE:NAME
-    #
-    # where TYPE is 'v' for variables and f for functions, and
-    # NAME is name of the part.
-    #
-    # If part is not specified, whole module file is printed.
-    #
-    local module=$1
-    local part=$2
-    local mfile
-    mfile=$(select_mfile "$module") || return 3
-    test -n "$mfile" || return 1
-    test -n "$part" || { cat "$mfile"; return $?; }
-    <"$mfile" filter_body "$part"
-}
-
-md_escapes() {
-    #
-    # Do additional escapes for Markdown
-    #
-    # In docstrings, references to variables are often done
-    # without backticks;  add these to prevent interpreting
-    # underscores in variable names as cursive.
-    #
-    perl -ne '
-        my @bigwords = ();
-        my @newwords = ();
-        if (m|^    |) {
-            print;                              # probably code -- leave be
-        } else {
-            @bigwords = split " ";
-            foreach (@bigwords) {
-                if (m|`\w+\(\)`|) {
-                    push @newwords, $_;              # function: escaped
-                } elsif (m|(.*)\b(\w+)\(\)(.*)|) {
-                    push @newwords, "$1`$2()`$3";    # function: needs escape
-                } elsif (m|`\$\w+`|) {
-                    push @newwords, $_;              # variable: escaped
-                } elsif (m|(.*)\$(\w+)(.*)|) {
-                    push @newwords, "$1`\$$2`$3";    # variable: needs escape
-                } else {
-                    push @newwords, $_;              # a normal word
-                }
-            }
-            print join " ", @newwords;
-            print "\n";
-        }
-    '
-}
-
 select_mfile() {
     #
     # Find and/or verify file that holds module $1
@@ -530,143 +50,80 @@ select_mfile() {
         */*)    mfile="$module" ;;
         *)      mfile=$(shellfu _select_mfile "$module") ;;
     esac
+    debug -v mfile
     test -n "$mfile" || { warn "no such module found: $module"; return 3; }
     test -f "$mfile" || { warn "no such file found: $mfile"; return 3; }
     echo "$mfile"
 }
 
-sf_list() {
-    #
-    # List all objects of type $1 in module $2
-    #
-    local otype=$1
-    local module=$2
-    local mfile
-    mfile=$(select_mfile "$module") || return 3
-    case $otype in
-            f)
-                grep -HE '^[[:alnum:]_]+\(\) \{' "$mfile" \
-                  | sed -e 's|(.*||; s|\.sh:|.|; s|^.*/||'
-                ;;
-            v)
-                grep -HE '^[[:alnum:]_]+=' "$mfile" \
-                  | sed -e 's|=.*||; s|\.sh:|.|; s|^.*/||'
-                ;;
-            *)
-                warn "bug: invalid object type: $otype"
-                ;;
-    esac \
-      | sf_flt_hidden
-}
-
-sf_list_modules() {
-    #
-    # List all module files
-    #
-    shellfu _list_mfiles \
-      | sed 's|\.sh$||; s|.*/||;' \
-      | sf_flt_hidden
-}
-
-sf_flt_hidden() {
-    $ShowHidden && cat && return
-    grep -v ^_ | grep -v \\._
-}
-
-strip_doc() {
-    #
-    # Strip doc out of Shellfu docstring
-    #
-    # From e.g.
-    #
-    #     #
-    #     # foo
-    #     #
-    #     # bar
-    #     # baz
-    #     #
-    #
-    # where first and last line are mandatory and each line
-    # must be either '#' or '# *', get:
-    #
-    #     foo
-    #
-    #     bar
-    #     baz
-    #
-    # just as when you're parsing this text by eyes.
-    #
-    head -n -1 \
-      | tail -n +2 \
-      | sed -e 's/^#$//; s/^# //;'
-}
-
 main() {
-    local ShowHidden        # show hidden objects?
     local action            # what to do
     local format            # export format
     local module            # module name or path/to/module.sh
     local m                 # module helper var
     local RealModuleName    # name to override eg. if accessing via filename
-    local Encoding          # encoding
-    ShowHidden=false
+    local encoding          # encoding
+    local mpath             # path to module file
     action=man
     format=markdown
-    Encoding=utf8
+    encoding=utf8
     #shellcheck disable=SC2034
     while true; do case "$1" in
         -d|--debug)     PRETTY_DEBUG=true;          shift ;;
-        -a|--all)       ShowHidden=true;            shift ;;
+        -a|--all)       SFDOC_SHOW_HIDDEN=true;     shift ;;
         -I|--include)   SHELLFU_PATH="$2:$SHELLFU_PATH"; shift 2 || usage ;;
         -l|--ls)        action=lsx;                 shift; break ;;
         -L|--lsmod)     action=lsm;                 shift; break ;;
         --lsvar)        action=lsv;                 shift; break ;;
         --lsfun)        action=lsf;                 shift; break ;;
         -e|--export)    action=exp; format="$2";    shift 2 || usage; break ;;
-        --encoding)     Encoding="$2";              shift 2 || usage ;;
+        --encoding)     encoding="$2";              shift 2 || usage ;;
         --name)         RealModuleName="$2";        shift 2 || usage ;;
         -*)                                         usage ;;
         *)                                          break ;;
     esac done
     module="$1"; shift
-    debug -v SHELLFU_INCLUDE SHELLFU_PATH
-    debug -v action ShowHidden format module RealModuleName
+    debug -v SHELLFU_INCLUDE SHELLFU_PATH SFDOC_SHOW_HIDDEN
+    debug -v action format module RealModuleName
     case $action:$module in
         lsx:*|lsm:*)    true ;;
         *:)             usage ;;
     esac
+    case $action in
+        lsm)    : ;;
+        *)      mpath=$(select_mfile "$module") || die ;;
+    esac
     case $action in
         exp)    # --export
             grep -qw "$format" <<<manpage,markdown,pod \
              || die "unknown format: $format"
-            "export_as_$format" "$module"
+            sfdoc__export "$format" "${RealModuleName:-$module}" "$mpath"
             ;;
         lsm)    # --lsmod
-            sf_list_modules | sort
+            sfdoc__ls_m | sort
             ;;
         lsv)    # --lsvar MODULE
-            sf_list v "$module" | sort
+            sfdoc__ls v "$mpath" | sort
             ;;
         lsf)    # --lsfun MODULE
-            sf_list f "$module" | sort
+            sfdoc__ls f "$mpath" | sort
             ;;
         lsx)    # --ls [MODULE]
             case $module in
-                "") sf_list_modules ;;
-                *)  echo "$module" ;;
+                "") shellfu _list_mfiles ;;
+                *)  echo "$mpath" ;;
             esac \
               | while read -r m;
                 do
-                    sf_list v "$m"
-                    sf_list f "$m"
+                    sfdoc__ls v "$m"
+                    sfdoc__ls f "$m"
                 done \
               | sort
             ;;
         man)
             which pod2man &>/dev/null \
              || die "pod2man is missing cannot use man page mode"
-            select_mfile "$module" >/dev/null || return 3
-            export_as_manpage "$module" \
+            sfdoc__export manpage "${RealModuleName:-$module}" "$mpath" \
               | man -l -
             ;;
     esac

src/include-bash/sfdoc.sh

@@ -0,0 +1,567 @@
+#!/bin/bash
+
+shellfu import pretty
+
+#
+# Show hidden objects (modules, functions, variables)?
+#
+SFDOC_SHOW_HIDDEN=${SFDOC_SHOW_HIDDEN:-false}
+
+sfdoc__export() {
+    #
+    # Export module doc as manpage
+    #
+    local format=$1             # format to export in
+    local MName=$2              # name to use in headers
+    local MPath=$3              # path to file to export
+    local Encoding=${4:-utf8}   # file encoding
+    local MVer                  # module version (if found)
+    local exfun                 # export function
+    local usage="usage: sfdoc__export FMT NAME PATH [ENCODING]"
+    test -n "$format"   || { warn "$usage"; return 2; }
+    test -n "$MPath"    || { warn "$usage"; return 2; }
+    test -n "$MName"    || { warn "$usage"; return 2; }
+    exfun="__sfdoc__export_as_$format"
+    type -t "$exfun" >/dev/null || {
+        warn "unknown format: $format"
+        return 2
+    }
+    MVer="v$(shellfu _read_directive "module-version" "$MPath" 2>/dev/null)"
+    test "$MVer" == "v" && MVer="(no version)"
+    debug -v format MName MPath Encoding MVer
+    "$exfun"
+}
+
+sfdoc__ls() {
+    #
+    # List all objects of type $1 in module file $2
+    #
+    local otype=$1
+    local mpath=$2
+    case $otype in
+            f)
+                grep -HE '^[[:alnum:]_]+\(\) \{' "$mpath" \
+                  | sed -e 's|(.*||; s|\.sh:|.|; s|^.*/||'
+                ;;
+            v)
+                grep -HE '^[[:alnum:]_]+=' "$mpath" \
+                  | sed -e 's|=.*||; s|\.sh:|.|; s|^.*/||'
+                ;;
+            *)
+                warn "bug: invalid object type: $otype"
+                ;;
+    esac \
+      | __sfdoc__filter_hidden
+}
+
+sfdoc__ls_m() {
+    #
+    # List all module files
+    #
+    shellfu _list_mfiles \
+      | sed 's|\.sh$||; s|.*/||;' \
+      | __sfdoc__filter_hidden
+}
+
+__sfdoc__export_as_manpage() {
+    #
+    # Export module $Module doc as manpage
+    #
+    __sfdoc__export_as_pod \
+      | pod2man \
+            --name "${MName^^}" \
+            --center "User Contributed Shellfu Documentation" \
+            --section 3x \
+            --date "$(date -I -r "$MPath")" \
+            --release "$MName $MVer" \
+            --utf8 -q \`
+}
+
+__sfdoc__export_as_markdown() {
+    #
+    # Export module $Module doc as Markdown
+    #
+    local variable
+    local function
+    local vheader_done=false
+    local fheader_done=false
+    debug -v module
+    echo "$MName"
+    echo "$MName" | tr -c '\n' '[=*]'
+    echo
+    __sfdoc__get_doc | __sfdoc__md_escapes
+    echo
+    for variable in $(sfdoc__ls v "$MPath" | cut -d. -f2);
+    do
+        debug -v variable
+        $vheader_done || {
+            echo
+            echo Variables
+            echo ---------
+            vheader_done=true
+        }
+        echo
+        echo "### \`\$$variable\` ###"
+        echo ""
+        __sfdoc__get_doc "v:$variable" | __sfdoc__md_escapes
+        echo
+    done
+    for function in $(sfdoc__ls f "$MPath" | cut -d. -f2);
+    do
+        debug -v function
+        $fheader_done || {
+            echo
+            echo Functions
+            echo ---------
+            fheader_done=true
+        }
+        echo
+        echo "### \`$function()\` ###"
+        echo ""
+        __sfdoc__get_doc "f:$function" | __sfdoc__md_escapes
+        echo
+    done
+}
+
+__sfdoc__export_as_pod() {
+    #
+    # Export module $Module doc as POD
+    #
+    local variable
+    local function
+    local vheader_done=false
+    local fheader_done=false
+    local indented=false
+    echo "true <<'=cut'"
+    echo "=pod"
+    echo
+    echo "=head1 NAME"
+    echo
+    echo "$MName - $(__sfdoc__get_doc | head -1)"
+    echo
+    echo "=head1 DESCRIPTION"
+    echo
+    __sfdoc__get_doc
+    echo
+    for variable in $(sfdoc__ls v "$MPath" | cut -d. -f2);
+    do
+        debug -v variable
+        $vheader_done || {
+            echo
+            echo "=head1 VARIABLES"
+            vheader_done=true
+            echo
+            echo "=over 8"
+            echo
+            indented=true
+        }
+        echo
+        echo "=item I<\$$variable>"
+        echo
+        __sfdoc__get_doc "v:$variable"
+        echo
+    done
+    $indented && {
+        echo "=back"
+        echo
+    }
+    for function in $(sfdoc__ls f "$MPath" | cut -d. -f2);
+    do
+        debug -v function
+        $fheader_done || {
+            echo
+            echo "=head1 FUNCTIONS"
+            fheader_done=true
+            echo
+            echo "=over 8"
+            echo
+            indented=true
+        }
+        echo
+        echo "=item I<$function()>"
+        echo
+        __sfdoc__get_doc "f:$function"
+        echo
+    done
+    $indented && {
+        echo "=back"
+        echo
+    }
+    echo "=encoding $Encoding"
+    echo "=cut"
+}
+
+__sfdoc__filter_body() {
+    #
+    # Filter part $1 of body on stdin
+    #
+    local part=$1
+    case "$part" in
+        f:*)  __sfdoc__filter_fun "${part:2}" ;;
+        v:*)  __sfdoc__filter_var "${part:2}" ;;
+        *) warn "bug: invalid part specification $part" ;;
+    esac
+}
+
+__sfdoc__filter_doc() {
+    #
+    # Filter docstring for part $1 from object body on stdin
+    #
+    local part=$1
+    case $part in
+        "")  __sfdoc__filter_mdoc ;;
+        v:*)  __sfdoc__filter_vdoc ;;
+        f:*)  __sfdoc__filter_fdoc ;;
+        *) warn "bug: invalid part specification: $part" ;;
+    esac
+}
+
+__sfdoc__filter_fun() {
+    #
+    # From module body on stdin, filter out function named $1
+    #
+    # Function definition should look like:
+    #
+    #     foo_fun() {
+    #         foo=$1
+    #     }
+    #
+    # That is,
+    #
+    #  *  no `function` keyword,
+    #  *  name starts the first line,
+    #  *  name is followed by '() {' with no additional spaces,
+    #  *  function end is denoted by line with a single '}'.
+    #
+    local fn_name=$1
+    name=$fn_name perl -we '
+        undef $/;
+        my $name = $ENV{name};
+        my ($fbody) = <> =~ m/^($name\(\) \{.*?^\}$)/ms;
+        print "$fbody\n" if defined $fbody;
+    '
+}
+
+__sfdoc__filter_fdoc() {
+    #
+    # Filter docstring from function body on stdin
+    #
+    # Look for:
+    #
+    #  1. line "^    #$" to open docstring - this must be
+    #     the first one after function name (`fun() {`)
+    #  2. block of consecutive docstring lines (i.e. matching
+    #     "^    # " or "^    #$"
+    #  3. last "^    #$" line to close docstring.
+    #  4. Next line must not match any of the patterns above.
+    #
+    # For example, stdin like this:
+    #
+    #     myfun() {
+    #         #
+    #         # Do my thing with foo $1
+    #         #
+    #         # Detailed description, possibly spanning
+    #         # several paragraph.
+    #         #
+    #         # The format here should be Markdown.
+    #         #
+    #         local foo=$1
+    #         printf %p "${foo:4}"
+    #     }
+    #
+    # would be read as:
+    #
+    #     Do my thing with foo $1
+    #
+    #     Detailed description, possibly spanning
+    #     several paragraph.
+    #
+    #     The format here should be Markdown.
+    #
+    # However if we added following line right before the `local`
+    # declaration
+    #
+    #         #FIXME: TODO/FIXME lines are not dropped properly
+    #
+    # it **will not** be considered part of the docstring. This is
+    # to allow for special comments tools like TODO/FIXME or
+    # data for lint-like tools that are not part of function
+    # description.
+    #
+    perl -we '
+        my $isdoc;
+        while (<>) {
+            if (m/^    #$/) {
+                $isdoc = 1;
+            } elsif (m/^    [^#]/) {
+                exit;
+            }
+            next unless $isdoc;
+            s/^    //;
+            print;
+        }' | __sfdoc__strip_doc
+}
+
+__sfdoc__filter_hidden() {
+    #
+    # From objects on stdin, print only visible unless $SFDOC_SHOW_HIDDEN
+    #
+    $SFDOC_SHOW_HIDDEN && cat && return
+    grep -v ^_ | grep -v \\._
+}
+
+__sfdoc__filter_mdoc() {
+    #
+    # From module body on stdin, filter module doc
+    #
+    # Example:
+    #
+    #     #!/bin/bash
+    #     #
+    #     # legal stuff
+    #
+    #     shellfu import irrelevant_stuff
+    #
+    #     #
+    #     # Module is cool this
+    #     #
+    #     # Ok so this is my
+    #     # "module", heh...
+    #     #
+    #
+    perl -we '
+        sub ok_chunk {
+            my $chunk = shift;
+            my @lines = split "\n", $_;
+            chomp @lines;
+            return 0 unless (defined $lines[0] and $lines[0] eq "#");
+            return 0 unless $lines[$#lines] eq "#";
+            foreach (@lines) {
+                return 0 unless (m/^#$/ or m/^# /);
+            }
+            return 1
+        }
+        my @chunks;
+        my @lines;
+        $/ = "\n\n";
+        @chunks = <STDIN>;
+        foreach (@chunks) {
+            if (ok_chunk "$_") {
+                s/\n+$//;
+                print "$_\n";
+                exit 0;
+            }
+        }
+        exit 1
+    ' | __sfdoc__strip_doc
+}
+
+__sfdoc__filter_var() {
+    #
+    # From module body on stdin, filter variable definition named $1
+    #
+    # Look for:
+    #
+    #  1. Empty line, followed by line with single hash sign
+    #  2. Set of consecutive docstring lines (start with '# ')
+    #  3. Another "#" followed by one or more variable definitions (start
+    #     with $name or \w+, followed by equal sign), at least one of which
+    #     must be $1
+    #  4. Empty line or EOF
+    #
+    # and return what is found.
+    #
+    # For example:
+    #
+    #     #
+    #     # My variable
+    #     #
+    #     my_var=foo
+    #
+    # is found if $1 is 'my_var'
+    #
+    # A more advanced example:
+    #
+    #     #
+    #     # Bar variables
+    #     #
+    #     # These serve similar purpose; the difference is
+    #     # obvious
+    #     #
+    #     bar_1=one
+    #     bar_2=two
+    #     bar_3=three
+    #
+    # is found if $1 is either 'bar_1', 'bar_2' or 'bar_3'.  This allows
+    # for a docstring being shared among closely related variables.
+    #
+    local oname=$1    # object name
+    local state       # void, edge, dstr, vstr, junk
+    local o_end       # object ended; is complete
+    local cache       # buffer of hope
+    # parse and keep each var docstring in cache;
+    # then decide whether or not print the cache
+    cache=$(mktemp -t shellfu_doc.__sfdoc__filter_var.XXXXXXXX)
+    state=junk
+    o_end=false
+    while read -r line;
+    do
+        case $state:$line in
+            # looks like void
+            void:)          state=void; o_end=false ;;
+            edge:)          state=void; o_end=true  ;;
+            dstr:)          state=void; o_end=true  ;;
+            vstr:)          state=void; o_end=true  ;;
+            junk:)          state=void; o_end=false ;;
+            # looks like edge
+            void:"#")       state=edge; o_end=false ;;
+            edge:"#")       state=dstr; o_end=false ;;
+            dstr:"#")       state=dstr; o_end=false ;;
+            vstr:"#")       state=dstr; o_end=false ;;
+            junk:"#")       state=junk; o_end=false ;;
+            # looks like docstring
+            void:"# "*)     state=junk; o_end=false ;;
+            edge:"# "*)     state=dstr; o_end=false ;;
+            dstr:"# "*)     state=dstr; o_end=false ;;
+            vstr:"# "*)     state=dstr; o_end=false ;;
+            junk:"# "*)     state=junk; o_end=false ;;
+            # looks like junk
+            void:"#"*)      state=junk; o_end=false ;;
+            edge:"#"*)      state=junk; o_end=false ;;
+            dstr:"#"*)      state=junk; o_end=false ;;
+            vstr:"#"*)      state=junk; o_end=false ;;
+            junk:"#"*)      state=junk; o_end=false ;;
+            # looks like variable string
+            void:*=*)       state=vstr; o_end=false ;;
+            edge:*=*)       state=junk; o_end=false ;;
+            dstr:*=*)       state=vstr; o_end=false ;;
+            vstr:*=*)       state=vstr; o_end=false ;;
+            junk:*=*)       state=junk; o_end=false ;;
+            # looks like something else
+            *)              state=junk; o_end=false ;;
+        esac
+        case $state in
+            edge|dstr|vstr) echo "$line" >> "$cache" ;;
+        esac
+        if $o_end;
+        then
+            if grep -q "^$oname=" "$cache";   # it's our wanted object
+            then
+                cat "$cache"
+                rm "$cache"
+                return 0
+            fi
+            rm "$cache"
+        fi
+    done
+    rm "$cache"
+    return 1
+}
+
+__sfdoc__filter_vdoc() {
+    #
+    # From variable definition body on stdin, filter doc
+    #
+    # Example:
+    #
+    #     #
+    #     # Bar variables
+    #     #
+    #     # These serve similar purpose; the difference is
+    #     # obvious
+    #     #
+    #     bar_1=one
+    #     bar_2=two
+    #     bar_3=three
+    #
+    grep '^#' | __sfdoc__strip_doc
+}
+
+__sfdoc__get_doc() {
+    #
+    # Show doc for part $2 of module $1
+    #
+    local part=$1
+    __sfdoc__get_part "$part" | __sfdoc__filter_doc "$part"
+}
+
+__sfdoc__get_part() {
+    #
+    # Print part $1 (f|v:name) of module $MPath
+    #
+    # Part has format
+    #
+    #     TYPE:NAME
+    #
+    # where TYPE is 'v' for variables and f for functions, and
+    # NAME is name of the part.
+    #
+    # If part is not specified, whole module file is printed.
+    #
+    local part=$1
+    test -n "$part" || { cat "$MPath"; return $?; }
+    <"$MPath" __sfdoc__filter_body "$part"
+}
+
+__sfdoc__md_escapes() {
+    #
+    # Do additional escapes for Markdown
+    #
+    # In docstrings, references to variables are often done
+    # without backticks;  add these to prevent interpreting
+    # underscores in variable names as cursive.
+    #
+    perl -ne '
+        my @bigwords = ();
+        my @newwords = ();
+        if (m|^    |) {
+            print;                              # probably code -- leave be
+        } else {
+            @bigwords = split " ";
+            foreach (@bigwords) {
+                if (m|`\w+\(\)`|) {
+                    push @newwords, $_;              # function: escaped
+                } elsif (m|(.*)\b(\w+)\(\)(.*)|) {
+                    push @newwords, "$1`$2()`$3";    # function: needs escape
+                } elsif (m|`\$\w+`|) {
+                    push @newwords, $_;              # variable: escaped
+                } elsif (m|(.*)\$(\w+)(.*)|) {
+                    push @newwords, "$1`\$$2`$3";    # variable: needs escape
+                } else {
+                    push @newwords, $_;              # a normal word
+                }
+            }
+            print join " ", @newwords;
+            print "\n";
+        }
+    '
+}
+
+__sfdoc__strip_doc() {
+    #
+    # Strip doc out of Shellfu docstring
+    #
+    # From e.g.
+    #
+    #     #
+    #     # foo
+    #     #
+    #     # bar
+    #     # baz
+    #     #
+    #
+    # where first and last line are mandatory and each line
+    # must be either '#' or '# *', get:
+    #
+    #     foo
+    #
+    #     bar
+    #     baz
+    #
+    # just as when you're parsing this text by eyes.
+    #
+    head -n -1 \
+      | tail -n +2 \
+      | sed -e 's/^#$//; s/^# //;'
+}

tests/sfdoc/TF_RUN

@@ -28,6 +28,10 @@ tf_do_subtest() {
     #
     local name=$1
     local cmd
+    local o_es=0
+    case $name in
+        nonesuch)      o_es=3 ;;
+    esac
     case $name in
         lsx_tricky)    cmd="sfdoc --ls test/include/tricky.sh" ;;
         lsv_tricky)    cmd="sfdoc --lsvar test/include/tricky.sh" ;;
@@ -36,7 +40,7 @@ tf_do_subtest() {
         included_path) cmd="sfdoc -I test/other -e markdown path" ;;
         *)             cmd="sfdoc -e markdown $name" ;;
     esac
-    tf_testflt -n "$name" -O "oracle/$name.stdout" -E "oracle/$name.stderr" "$cmd"
+    tf_testflt -n "$name" -O "oracle/$name.stdout" -E "oracle/$name.stderr" -S "$o_es" "$cmd"
 }
 
 tf_do_subtests

tests/sfdoc/oracle/nonesuch.stderr

@@ -1,3 +1,2 @@
 no such module found: nonesuch
-no such module found: nonesuch
-no such module found: nonesuch
+

tests/sfdoc/oracle/nonesuch.stdout

@@ -1,4 +0,0 @@
-nonesuch
-========
-
-

tests/shellfu_api/TF_RUN

@@ -19,6 +19,7 @@ flt_ours() {
         -e exit \
         -e termcolors \
         -e pretty \
+        -e sfdoc \
         -e mdfmt \
     #FIXME: remove the filter when TFKit learns to test in sandbox
 }

tests/shellfu_api/oracle/functions.stdout

@@ -73,3 +73,21 @@ pretty.mkhelp
 pretty.mkusage
 pretty.think
 pretty.warn
+sfdoc.__sfdoc__export_as_manpage
+sfdoc.__sfdoc__export_as_markdown
+sfdoc.__sfdoc__export_as_pod
+sfdoc.__sfdoc__filter_body
+sfdoc.__sfdoc__filter_doc
+sfdoc.__sfdoc__filter_fdoc
+sfdoc.__sfdoc__filter_fun
+sfdoc.__sfdoc__filter_hidden
+sfdoc.__sfdoc__filter_mdoc
+sfdoc.__sfdoc__filter_var
+sfdoc.__sfdoc__filter_vdoc
+sfdoc.__sfdoc__get_doc
+sfdoc.__sfdoc__get_part
+sfdoc.__sfdoc__md_escapes
+sfdoc.__sfdoc__strip_doc
+sfdoc.sfdoc__export
+sfdoc.sfdoc__ls
+sfdoc.sfdoc__ls_m

tests/shellfu_api/oracle/modules.stdout

@@ -8,4 +8,5 @@ exit
 inigrep
 mdfmt
 pretty
+sfdoc
 termcolors

tests/shellfu_api/oracle/variables.stdout

@@ -16,6 +16,7 @@ pretty.PRETTY_DEBUG
 pretty.PRETTY_DEBUG_EXCLUDE
 pretty.PRETTY_USAGE
 pretty.PRETTY_VERBOSE
+sfdoc.SFDOC_SHOW_HIDDEN
 termcolors.TERMCOLORS_BLACK
 termcolors.TERMCOLORS_BLUE
 termcolors.TERMCOLORS_CYAN