Browse Source

Use upstream repo and nested workdir layout for tfkit

Alois Mahdal 9 years ago
parent
commit
db13e2c3f9

+ 2
- 0
.gitignore View File

@@ -1,3 +1,5 @@
1 1
 .autoclean
2 2
 saturnin-*.tar.gz
3 3
 tfkit-artifacts
4
+tfkit
5
+bin/tfkit_init

+ 1
- 3
Makefile View File

@@ -2,6 +2,7 @@
2 2
 # See LICENSE file for copyright and license details.
3 3
 
4 4
 include config.mk
5
+include tfkit.mk
5 6
 
6 7
 all: options build
7 8
 
@@ -35,9 +36,6 @@ release_y:
35 36
 release_z:
36 37
 	@mkit/make release_z
37 38
 
38
-test:
39
-	@tfkit/runtests
40
-
41 39
 uninstall:
42 40
 	@mkit/make uninstall
43 41
 

+ 53
- 0
tfkit.mk View File

@@ -0,0 +1,53 @@
1
+# Makefile module to deploy tfkit
2
+#
3
+# Will add bin/tfkit_init utility, and use it to guess URI to tfkit.git,
4
+# and clone tfkit to current directory.
5
+#
6
+# Adds targets: test to run tfkit tests, tfkit to deploy tfkit, tfkit_update to
7
+# pull tfkit updates, and tfkit_delete to remove the tfkit code.
8
+
9
+
10
+test: tfkit
11
+	@tfkit/runtests
12
+
13
+tfkit: bin/tfkit_init
14
+	@bin/tfkit_init
15
+
16
+tfkit_delete:
17
+	@rm -rf tfkit
18
+	@rm -f bin/tfkit_init
19
+	@rmdir bin 2>/dev/null || true
20
+
21
+bin/tfkit_init:
22
+	@mkdir -p bin
23
+	@echo '#!/bin/sh' >bin/tfkit_init
24
+	@echo '' >>bin/tfkit_init
25
+	@echo 'die() {' >>bin/tfkit_init
26
+	@echo '    echo "$$1" >&2' >>bin/tfkit_init
27
+	@echo '    exit 3' >>bin/tfkit_init
28
+	@echo '}' >>bin/tfkit_init
29
+	@echo '' >>bin/tfkit_init
30
+	@echo 'if test -d tfkit/.git;' >>bin/tfkit_init
31
+	@echo 'then' >>bin/tfkit_init
32
+	@echo '    exit 0' >>bin/tfkit_init
33
+	@echo 'else' >>bin/tfkit_init
34
+	@echo '' >>bin/tfkit_init
35
+	@echo '    origin=""' >>bin/tfkit_init
36
+	@echo '    git remote | grep -q origin && origin=origin' >>bin/tfkit_init
37
+	@echo '    test $$(git remote | wc -l) -eq 1 && origin=$$(git remote)' >>bin/tfkit_init
38
+	@echo '    test -n "$$origin" || die "could not guess remote name"' >>bin/tfkit_init
39
+	@echo '' >>bin/tfkit_init
40
+	@echo '' >>bin/tfkit_init
41
+	@echo '    uri=$$(git remote -v | grep "^$$origin.*(fetch)" | tr $$"\t" " " | cut -d\  -f2)' >>bin/tfkit_init
42
+	@echo '    tfkit_uri=$$(dirname $$uri)/tfkit.git' >>bin/tfkit_init
43
+	@echo '' >>bin/tfkit_init
44
+	@echo '    git clone $$tfkit_uri || die "failed to clone tfkit"' >>bin/tfkit_init
45
+	@echo '' >>bin/tfkit_init
46
+	@echo 'fi' >>bin/tfkit_init
47
+	@chmod +x bin/tfkit_init
48
+
49
+tfkit_update: tfkit
50
+	@cd tfkit
51
+	@git --git-dir tfkit/.git pull
52
+
53
+.PHONY: test

+ 0
- 15
tfkit/README.md View File

@@ -1,15 +0,0 @@
1
-TFKIT
2
-=====
3
-
4
-A simple portable (or rather "movable") test framework.
5
-
6
- 1. Copy (or `git submodule`) *test* sub-folder to your place
7
-    where you want to run tests.
8
-
9
- 2. Run your framework by `./tfkit/runtests`.
10
-
11
- 3. Don't be surprised--you haven't added any tests yet :)
12
-
13
- 4. Read tfkit/README.testing.md
14
-
15
- 5. Start adding tests to `tests` folder

+ 0
- 360
tfkit/README.testing.md View File

@@ -1,360 +0,0 @@
1
-Tests
2
-=====
3
-
4
-Running tests is handled by tfkit/runtests:
5
-
6
-    $ tfkit/runtest [filter]
7
-
8
-*filter* is a regular expression to be applied to sub-test name, running
9
-only the matching ones.  See below for details.
10
-
11
-
12
-Writing tests
13
--------------
14
-
15
-Tests can be written in any scripting language, although the built-in
16
-framework, written in Bash, provides some useful features for writing
17
-certain kind of relatively simple tests.
18
-
19
-The harness, though, assumes that:
20
-
21
- *  Any direct sub-directory of $TF_SUITE directory ("tests" by default)
22
-    that contains at least *TF_RUN* executable becomes a test,
23
-
24
- *  basename of this directory becomes the name of the test,
25
-
26
- *  and return code from running the executable is reported
27
-    as result of the test, according to "Exit status" chapter below.
28
-
29
-
30
-Naming
31
-------
32
-
33
-Test name should start with name of the module that is tested and
34
-underscore.  If module name contains dots, they should be replaced with
35
-underscores as well.
36
-
37
-    core_sanity
38
-    mod_submod_function
39
-    ini_iniread
40
-
41
-are valid test names.
42
-
43
-
44
-Data
45
-----
46
-
47
-Should the test need any data, just leave it around in the test directory
48
-along with TF_RUN.
49
-
50
-Note that before running, the whole test directory is automatically
51
-copied to a temporary location (one per test), and should the test fail,
52
-copied back as a debugging artifact.  For this reason, *do not store
53
-huge amounts of data here*.  If you really need huge data, consider
54
-obtaining it (and throwing it away) within runtime of TF_RUN.
55
-
56
-
57
-Exit status
58
------------
59
-
60
-We try hard to follow this semantic:
61
-
62
- *  *Zero* means *OK* -- test has been run and passed.
63
-
64
- *  *One* means *Failure* -- test has been run but failed (e.g. found
65
-     a bug).
66
-
67
- *  *Two* means *Bailout* --  test has decided not to run at all.
68
-
69
- *  *Three* means *Error* -- there was error detected during execution,
70
-     but script was able to clean up properly.
71
-
72
- *  *Four* means *Panic* -- there was other error but script *was not*
73
-     able to clean up properly.
74
-
75
- *  Anything else should indicate other uncaught errors, including those
76
-    outside control of the program such as segfaults in the test code
77
-    or test being SIGKILLed.
78
-
79
-Notice that the higher the value is, the worse situation it indicates.
80
-Thus, if a test is composed of several sub-tests, you need to make sure
81
-to always **exit with the highest value** (subtest.sh does take care
82
-of this).
83
-
84
-See *common.sh* for functions and variables to help with handling exit
85
-statuses with this semantic.
86
-
87
-Also see Notes section for more details on exit statuses, including
88
-cheat sheet and dscussuion.
89
-
90
-
91
-Framework
92
----------
93
-
94
-
95
-### harness.sh ###
96
-
97
-This part is not intended to be used in tests, but rather contains
98
-functions that help govern test discovery, preparation and execution as
99
-is described in previous chapters.  Feel free to poke around, of course.
100
-
101
-
102
-### subtest.sh ###
103
-
104
-As name suggests, this file defines few functions to handle subtests
105
-in *TF_RUN*.
106
-
107
-In order to make use of the subtests functionality, you will need to
108
-define two functions yourself:  `tf_enum_subtests` to enumerate names of
109
-tests you want to run, and `tf_name2cmd` to translate each name an actual
110
-command that would perform it and return with the the correct exit status.
111
-
112
-The minimal *TF_RUN* with two subtests could look like this:
113
-
114
-    #!/bin/bash
115
-
116
-    . $TF_DIR/include/subtest.sh
117
-
118
-    tf_enum_subtests() {
119
-        echo test1
120
-        echo test2
121
-        something && echo test3
122
-    }
123
-
124
-    tf_name2cmd() {
125
-        case $1 in
126
-            test1)  echo myprog foo ;;
127
-            test2)  echo myprog bar ;;
128
-        esac
129
-    }
130
-
131
-    tf_do_subtests
132
-
133
-At the end, `tf_do_subtests` acts as a launcher of the actual test.
134
-In short, it will
135
-
136
- *  take each enumerated subtest from `tf_enum_subtests`,
137
- *  source TF_SETUP, if such file is found,
138
- *  translate te subtest name to a command,
139
- *  launch the command,
140
- *  run TF_CLEANUP, if such file is found,
141
- *  and report "worst" exit status encountered.
142
-
143
-All but the first and last step is done by `tf_do_subtest`, so in some
144
-cases you may want to re-define this one as well.
145
-
146
-Note that subtest names need to be single words (`[a-zA-Z0-9_]`).
147
-
148
-
149
-### tools.sh ###
150
-
151
-This file contains various tools and utilities to help with testing.
152
-
153
-Curently there is only one function, `tf_testflt` designed to help write
154
-tests for simple unix filters.
155
-
156
-
157
-#### tf_testflt ####
158
-
159
-The idea is that tester specifies
160
-
161
- *  test name,
162
- *  command to launch the system under test,
163
- *  a data stream to use as STDIN,
164
- *  and expected STDOUT, STDERR, and exit status.
165
-
166
-and tf_testflt launches the command, collects tha data and evaluates
167
-and reports the result using unified diff.
168
-
169
-In its simplest form:
170
-
171
-    tf_testflt -n foo my_command arg
172
-
173
-the function will run `my_command arg` (not piping anything to it),
174
-and will expect it to finish with exit status 0 and empty both STDERR
175
-and STDOUT.
176
-
177
-Example of full form,
178
-
179
-    tf_testflt -n foo -i foo.in -O foo.stdout -E foo.stderr -S 2 myprog
180
-
181
-will pipe foo.in into `myprog`, expecting exit status of 2, and STDOUT and
182
-STDERR as above.  Notice that parameters specifying expected values are
183
-uppercase, and those specifying input values are lowercase.
184
-
185
-Specifying name is mandatory, because it's used in reporting messages,
186
-and as a basis for naming temporary result files: these are saved in
187
-*results* subdirectory and kept for further reference.
188
-
189
-
190
-### common.sh ###
191
-
192
-This includes simple functions and variables shared between both mentioned
193
-libraries.
194
-
195
-First group is designed to help support the exit status semantic:
196
-
197
- *  The functions are `tf_exit_pass`, `tf_exit_fail`, `tf_exit_bailout`,
198
-    `tf_exit_error` and `tf_exit_panic` and each take any number of
199
-    parameters that are printed on stderr.
200
-
201
- *  The variables are `TF_ES_OK`, `TF_ES_FAIL`, `TF_ES_BAILOUT`,
202
-    `TF_ES_ERROR` and `TF_ES_PANIC` and are supposed to be used with
203
-    `return` builtin, e.g. to return from `tf_exit_error`.
204
-
205
-Second group is useful to better control output:  functions `tf_warn`,
206
-`tf_debug` and `tf_think` are used to print stuff on STDERR.  Use of
207
-`tf_warn` is apparent, just as `tf_debug`, the latter being muted if
208
-`TF_DEBUG` is set to `false` (set it to `true` to turn on debugging).
209
-
210
-`tf_think` is used for progress info, and is muted unless `TF_VERBOSE`
211
-is set to `true`, which is by defaiult.
212
-
213
-
214
-### Setup and cleanup ###
215
-
216
-Special files *TF_SETUP* and *TF_CLEANUP* (one of them or both) can be
217
-added along with *TF_RUN*.  These will be sourced before (*TF_SETUP*)
218
-and after every subtest (*TF_CLEANUP*).
219
-
220
-First, if any of these files are missing, it is considered as if the
221
-respective phase succeeded.  Second, if setup phase fails, test will
222
-be skipped and subtest exit status will be *TF_ES_BAILOUT*.   Last,
223
-if cleanup fails (no matter result of setup), subtests aborts with
224
-*TF_ES_PANIC* returned.  Be aware that in this case the actual test
225
-status, albeit useful, is lost.
226
-
227
-When coming from other test frameworks, this may feel harsh, but note
228
-that this has been designed with the idea that if a cleanup fails,
229
-it may render all further tests are automatically unsafe, because the
230
-environment is not as expected.
231
-
232
-To cope with this behavior, try to bear in mind following advice:
233
-
234
- 1. Make sure you write setup/cleanup procedures with extreme care and
235
-    test them well.
236
-
237
- 2. Do not do complicated and risky things in the setup/cleanup phases.
238
-
239
- 3. If you need to do such things, consider doing them in the *TF_RUN*
240
-    instead of doing them for all subtests.
241
-
242
- 4. You don't need to clean up everything, the contents of the testing dir
243
-    will be moved out from the test system.
244
-
245
- 5. If there are scenarios you can safely fix or ignore, handle them in
246
-    a robust manner.
247
-
248
-
249
-Notes
250
------
251
-
252
-
253
-### bailout vs. `tf_enum_subtests` ###
254
-
255
-One more note to claify relation of bailout and `tf_enum_subtests`.
256
-As you may have noticed, there are two ways how to skip a test:
257
-return prematurely with `TF_ES_BAILOUT`, or suppress enumeration in
258
-`tf_enum_subtests`.  The problem is that the latter does not do anything
259
-to inform upper in the stack that a test has been skipped, which seems to
260
-break the principle described in the previous chapters.
261
-
262
-Don't confuse these mechanisms, though. Each is supposed to be used
263
-for distinct purpose.  Compare: by using the `tf_enum_subtests` you are
264
-saying that you actually **did not even want** to run the test in the
265
-first place.  By using `TF_ES_BAILOUT`, you are saying that you **wanted**
266
-to run the test but could not.
267
-
268
-A few common cases if that helps you:
269
-
270
- *  If during the test you find out that for some reason it can't be
271
-    carried out (e.g.   an external resource is not available, or
272
-    something outside the SUT is broken), use `TF_ES_BAILOUT`.
273
-
274
- *  If you want to disable the test because for some long-term condition,
275
-    e.g. a known bug outside SUT but preventing execution of the test
276
-    is not fixed, use `tf_enum_subtests`.
277
-
278
- *  If you want to filter out some sub-tests to only for some platforms,
279
-    e.g. 64-bit architecture, (IOW, you can safely check that a
280
-    sub-test would be totally pointless if run on this box), use
281
-    `tf_enum_subtests`.
282
-
283
- *  If you want to disable (comment out test) that you might not have
284
-    implemented yet or is broken (and for some reason you still want
285
-    it to haunt the test code), use `tf_enum_subtests` and properly
286
-    comment the reasons in code.
287
-
288
- *  If in doubt, use `TF_ES_BAILOUT`.
289
-
290
-
291
-### On exit statuses: three and above ###
292
-
293
-The difference in *error*, *panic* and higher values is subtle but
294
-important.  Follow me as I try to explain:
295
-
296
- 1. If script has changed something on the system outside the working
297
-    directory, it is apparently expected to revert that change.
298
-
299
- 2. Now if an error occurs, but the code responsible for cleaning up is
300
-    safely run, you can say there was *error but we have recovered*.
301
-
302
- 3. But if the change can't be reverted safely, we know that we have
303
-    broken something and latter code may lead to weird results (including
304
-    masking bugs(!)), it's time to *panic* (in the code, not in real
305
-    life ;))
306
-
307
- 4. And then there are be corner cases like a bug in the script, OOM kill
308
-    or timeout when the status will be different and not really controlled
309
-    by the script.  Such cases will have to be treated the same way as
310
-    the "panic" case, but...
311
-
312
- 5. the use of *panic* adds hint that the status has been set consciously
313
-    by the script, albeit exiting "in a hurry"--without proper clean up.
314
-
315
-Unfortunately there will be cases like above but with the error code less
316
-than four.   Example is a bash script syntax error, which returns 2, or
317
-Python exception which returns 1.  Yes, in such cases the information
318
-conveyed by the exit status is wrong and you should do everything to
319
-avoid it.
320
-
321
-Possibilities like "test has passed but then something blew up" exist,
322
-but conveying this information is responsibility of the test output.
323
-
324
-Following table can be used as a cheat-sheet:
325
-
326
-    .---------------------------------------------------------------.
327
-    | e |    state of         |                                     |
328
-    | s |---------------------| script says                         |
329
-    |   | SUT   | environment |                                     |
330
-    |---|-------|-------------|-------------------------------------|
331
-    | 0 | OK    | safe        | test passed, everything worked fine |
332
-    | 1 | buggy | safe        | test failed, everything worked fine |
333
-    | 2 | ???   | safe        | I decided not to run the test       |
334
-    | 3 | ???   | safe        | Something blew up but I managed to  |
335
-    |   |       |             | clean up (I promise!)               |
336
-    | 4 | ???   | broken      | Something blew up and I rushed out  |
337
-    |   |       |             | in panic                            |
338
-    | * | ???   | broken      | ...nothing (is dead)                |
339
-    '---------------------------------------------------------------'
340
-
341
-As you can see, following this semantic allows us to see both the state
342
-of the system under test (SUT) *and* the environment.
343
-
344
-Following table illustrates how different statuses map to different
345
-scenarios with regard to test result as well as state of the environment:
346
-
347
-    .--------------------------------------------------.
348
-    | environment |  test result   |  test result      |
349
-    |             | pass fail unkn | pass fail unkn    |
350
-    |-------------|----------------|-------------------|
351
-    | clean(ed)   |  0    1    3   |  OK  FAIL ERROR   |
352
-    | untouched   |  ~    ~    2   |  ~    ~   BAILOUT |
353
-    | mess        |  ~    ~    4   |  ~    ~   PANIC   |
354
-    | ?! (trap)   |  ~    ~    5   |  ~    ~   ~       |
355
-    | ?! (sig 9)  |  ~    ~    137 |  ~    ~   ~       |
356
-    | ?! (aliens) |  ~    ~    ?   |  ~    ~   ~       |
357
-    '-------------|----------------|-------------------|
358
-                  |  exit status   |  human-readable   |
359
-                  |                |  name (TF_ES_*)   |
360
-                  '------------------------------------'

+ 0
- 26
tfkit/doc/templates/grep_engine/TF_RUN View File

@@ -1,26 +0,0 @@
1
-#!/bin/bash
2
-
3
-. $TF_DIR/include/subtest.sh
4
-. $TF_DIR/include/tools.sh
5
-
6
-tf_enum_subtests() {
7
-    echo fixed
8
-    echo basic
9
-    echo extended
10
-#   echo perl       # TODO: write test
11
-}
12
-
13
-tf_name2cmd() {
14
-    local name=$1
15
-    local t_in="test/ALL.stdin"
16
-    local o_out="oracle/$name.stdout"
17
-    local args
18
-    case $name in
19
-        fixed)       args="-F 'The mask *.* matches all.'" ;;
20
-        basic)       args="-G 'he.*'" ;;
21
-        extended)    args="-P '.*og?g'" ;;
22
-    esac
23
-    echo "tf_testflt -n $name -i $t_in -O $o_out grep $args"
24
-}
25
-
26
-tf_do_subtests

+ 0
- 3
tfkit/doc/templates/grep_engine/oracle/basic.stdout View File

@@ -1,3 +0,0 @@
1
-Linda and Nina work together.
2
-The mask *.* matches all.
3
-The mask *.docx matches Word documents.

+ 0
- 3
tfkit/doc/templates/grep_engine/oracle/extended.stdout View File

@@ -1,3 +0,0 @@
1
-Alice uses Google.
2
-Bob wears goggles.
3
-Linda and Nina work together.

+ 0
- 1
tfkit/doc/templates/grep_engine/oracle/fixed.stdout View File

@@ -1 +0,0 @@
1
-The mask *.* matches all.

+ 0
- 6
tfkit/doc/templates/grep_engine/test/ALL.stdin View File

@@ -1,6 +0,0 @@
1
-Alice uses Google.
2
-Bob wears goggles.
3
-Joe and John are friends.
4
-Linda and Nina work together.
5
-The mask *.* matches all.
6
-The mask *.docx matches Word documents.

+ 0
- 135
tfkit/include/common.sh View File

@@ -1,135 +0,0 @@
1
-#
2
-# Variables to hold exit status semantic
3
-#
4
-TF_ES_OK=0
5
-TF_ES_FAIL=1
6
-TF_ES_BAILOUT=2
7
-TF_ES_ERROR=3
8
-TF_ES_PANIC=4
9
-
10
-#
11
-# Option to turn on/off verbosity
12
-#
13
-TF_VERBOSE=${TF_VERBOSE:-true}
14
-
15
-#
16
-# Option to turn on/off debug mode
17
-#
18
-TF_DEBUG=${TF_DEBUG:-false}
19
-
20
-#
21
-# Regex to filter test names to run
22
-#
23
-TF_FILTER_TEST="${TF_FILTER_TEST:-.*}"
24
-
25
-#
26
-# Regex to filter subtest names to run
27
-#
28
-TF_FILTER_SUBTEST="${TF_FILTER_SUBTEST:-.*}"
29
-
30
-#
31
-# Enable color?
32
-#
33
-TF_COLOR=${TF_COLOR:-true}
34
-
35
-#
36
-# Color definition variables
37
-#
38
-TF_COLOR_BLACK="\033[0;30m"
39
-TF_COLOR_RED="\033[0;31m"
40
-TF_COLOR_GREEN="\033[0;32m"
41
-TF_COLOR_YELLOW="\033[0;33m"
42
-TF_COLOR_BLUE="\033[0;34m"
43
-TF_COLOR_MAGENTA="\033[0;35m"
44
-TF_COLOR_CYAN="\033[0;36m"
45
-TF_COLOR_WHITE="\033[0;37m"
46
-TF_COLOR_LBLACK="\033[1;30m"
47
-TF_COLOR_LRED="\033[1;31m"
48
-TF_COLOR_LGREEN="\033[1;32m"
49
-TF_COLOR_LYELLOW="\033[1;33m"
50
-TF_COLOR_LBLUE="\033[1;34m"
51
-TF_COLOR_LMAGENTA="\033[1;35m"
52
-TF_COLOR_LCYAN="\033[1;36m"
53
-TF_COLOR_LWHITE="\033[1;37m"
54
-TF_COLOR_NONE="\033[1;0m"
55
-
56
-
57
-tf_exit_ok() {
58
-    #
59
-    # Exit with OK status
60
-    #
61
-    exit $TF_ES_OK
62
-}
63
-
64
-tf_exit_fail() {
65
-    #
66
-    # Warn $1 and exit with status FAIL
67
-    #
68
-    tf_warn "$@"
69
-    exit $TF_ES_FAIL
70
-}
71
-
72
-tf_exit_bailout() {
73
-    #
74
-    # Warn $1 and exit with status FAIL
75
-    #
76
-    tf_warn "$@"
77
-    exit $TF_ES_BAILOUT
78
-}
79
-
80
-tf_exit_error() {
81
-    #
82
-    # Warn $1 and exit with status FAIL
83
-    #
84
-    tf_warn "$@"
85
-    exit $TF_ES_ERROR
86
-}
87
-
88
-tf_exit_panic() {
89
-    #
90
-    # Warn $1 and exit with status FAIL
91
-    #
92
-    tf_warn "$@"
93
-    exit $TF_ES_PANIC
94
-}
95
-
96
-tf_debug() {
97
-    #
98
-    # Emit debug message
99
-    #
100
-    $TF_DEBUG || return 0
101
-    local msg
102
-    for msg in "$@";
103
-    do
104
-        $TF_COLOR && echo -ne "$TF_COLOR_CYAN" >&2
105
-        echo "||| $1" >&2;
106
-        $TF_COLOR && echo -ne "$TF_COLOR_NONE" >&2
107
-    done
108
-}
109
-
110
-tf_think() {
111
-    #
112
-    # Emit status/progress message
113
-    #
114
-    $TF_VERBOSE || return 0
115
-    local msg
116
-    for msg in "$@";
117
-    do
118
-        $TF_COLOR && echo -ne "$TF_COLOR_LBLACK" >&2
119
-        echo "$pfx$1$sfx" >&2;
120
-        $TF_COLOR && echo -ne "$TF_COLOR_NONE" >&2
121
-    done
122
-}
123
-
124
-tf_warn() {
125
-    #
126
-    # Emit warning
127
-    #
128
-    local msg
129
-    for msg in "$@";
130
-    do
131
-        $TF_COLOR && echo -ne "$TF_COLOR_LRED" >&2
132
-        echo "$1" >&2;
133
-        $TF_COLOR && echo -ne "$TF_COLOR_NONE" >&2
134
-    done
135
-}

+ 0
- 113
tfkit/include/harness.sh View File

@@ -1,113 +0,0 @@
1
-#!/bin/bash
2
-# ffoo test harness
3
-# See LICENSE file for copyright and license details.
4
-
5
-. $TF_DIR/include/common.sh
6
-
7
-#
8
-# Default path to header generator
9
-#
10
-__TF_HDRCMD="$TF_SUITE/TF_HEADER"
11
-
12
-
13
-__tf_collect_if_needed() {
14
-    #
15
-    # Collect artifact if exit status suggests it
16
-    #
17
-    # Use test exit status $1 to help decide if artifacts are
18
-    # needed, and collect them if so.
19
-    #
20
-    # If TF_COLLECT is set to "always", collect regardless of
21
-    # the status.  Otherwise, collect unless status is 0 or 1
22
-    # (pass or bailout); in that case do nothing.
23
-    #
24
-    local tes=$1    # test exit status
25
-    local will      # should we collect artifacts?
26
-    case "$TF_COLLECT:$tes" in
27
-        always:*)   will=true ;;
28
-        *:0)        will=false ;;
29
-        *:2)        will=false ;;
30
-        *)          will=true ;;
31
-    esac
32
-    $will || return 0
33
-    mkdir -p "$artifact_dir/$stamp"
34
-    cp -r $tmpdir/* "$artifact_dir/$stamp"
35
-}
36
-
37
-__tf_default_header() {
38
-    #
39
-    # Create default header
40
-    #
41
-    echo "(add $__TF_HDRCMD executable for own header)"
42
-}
43
-
44
-__tf_header() {
45
-    #
46
-    # create header to add to output before test
47
-    #
48
-    local field
49
-    local hdrcmd="$__TF_HDRCMD"
50
-    test -x "$hdrcmd" || hdrcmd="__tf_default_header"
51
-    $hdrcmd \
52
-      | while read field;
53
-        do
54
-            test -n "$field" || break
55
-            tf_think "# $field"
56
-        done
57
-    tf_think ""
58
-}
59
-
60
-tf_enum_tests() {
61
-    #
62
-    # List what looks like test; relative to $TF_SUITE
63
-    #
64
-    tf_debug "TF_SUITE='$TF_SUITE'"
65
-    test -d $TF_SUITE || return 0
66
-    find -L \
67
-        $TF_SUITE \
68
-        -mindepth 2 \
69
-        -maxdepth 2 \
70
-        -type f \
71
-        -perm /111 \
72
-        -name TF_RUN \
73
-      | cut -d/ -f2
74
-}
75
-
76
-tf_run_tests() {
77
-    #
78
-    # Discover and run tests
79
-    #
80
-    local artifact_dir="$PWD/tfkit-artifacts"
81
-    local es=0          # overall exit status
82
-    local tmpdir=""     # test temporary dir
83
-    local tname=""      # test name
84
-    local tes=0         # test result
85
-    local stamp=""      # test stamp to use as artifact name
86
-    local tf_dir tf_suite   # to keep absolute paths for TF_RUN
87
-    __tf_header
88
-    tf_dir="$(readlink -f "$TF_DIR")"
89
-    tf_suite="$(readlink -f "$TF_SUITE")"
90
-    es=0
91
-    for tname in $(tf_enum_tests | grep -e "$TF_FILTER_TEST");
92
-    do
93
-        tf_think "... $tname"
94
-        tmpdir=$(mktemp -d)
95
-        stamp=$(date +artifacts-$tname-%Y%m%d-%H%M%S)
96
-        cp -r "$TF_SUITE/$tname/"* "$tmpdir"
97
-        pushd $tmpdir >/dev/null
98
-            TF_DIR="$tf_dir" TF_SUITE=$tf_suite TF_TNAME="$tname" \
99
-                ./TF_RUN
100
-            tes=$?
101
-            __tf_collect_if_needed $tes
102
-            test $tes -gt $es && es=$tes
103
-        popd >/dev/null
104
-        rm -rf $tmpdir
105
-        if test $tes -eq 0;
106
-        then
107
-            tf_think "''' $tname ($tes)"
108
-        else
109
-            tf_warn "??? $tname ($tes)"
110
-        fi
111
-    done
112
-    return $es
113
-}

+ 0
- 73
tfkit/include/subtest.sh View File

@@ -1,73 +0,0 @@
1
-#!/bin/bash
2
-
3
-. $TF_DIR/include/common.sh
4
-
5
-tf_enum_subtests() {
6
-    #
7
-    # Stub: enumerate subtests
8
-    #
9
-    tf_warn "implement tf_enum_subtests()!"
10
-    return $TF_ES_ERROR
11
-}
12
-
13
-tf_name2cmd() {
14
-    #
15
-    # Stub: expand test name to test command
16
-    #
17
-    tf_warn "implement tf_name2cmd()!"
18
-    return $TF_ES_ERROR
19
-}
20
-
21
-tf_do_subtest() {
22
-    #
23
-    # Run single subtest inc. setup/cleanup if present
24
-    #
25
-    local subtname=$1       # this subtest name
26
-    local ses=0             # subtest exit status
27
-    local tcmd=""           # test command
28
-    local setup=true        # setup function
29
-    local cleanup=true      # cleanup command
30
-    test -f TF_SETUP   && setup=". TF_SETUP"
31
-    test -f TF_CLEANUP && cleanup=". TF_CLEANUP"
32
-    if $setup;
33
-    then
34
-        tcmd="$(tf_name2cmd $subtname)"
35
-        tf_debug "tcmd='$tcmd'"
36
-        $tcmd; ses=$?
37
-    else
38
-        tf_warn "setup phase failed, skipping: $subtname"
39
-        ses=$TF_ES_ERROR
40
-    fi
41
-    if ! $cleanup;
42
-    then
43
-        tf_warn "cleanup phase failed: $subtname"
44
-        ses=$TF_ES_PANIC
45
-    fi
46
-    return $ses
47
-}
48
-
49
-tf_do_subtests() {
50
-    #
51
-    # Run all subtests and return highest status
52
-    #
53
-    local es=0              # final exit status ("worst" of subtests)
54
-    local subtname=""       # one subtest name
55
-    local tes=""            # one subtest exit status
56
-    local enumd=TF_ENUMERATED_SUBTESTS
57
-    local fltrd=TF_FILTERED_SUBTESTS
58
-    tf_enum_subtests >$enumd    || { tf_warn "error enumerating subtests"; return $TF_ES_BAILOUT; }
59
-    test -s $enumd              || { tf_warn "no subtests enumerated";     return $TF_ES_BAILOUT; }
60
-    grep -e "$TF_FILTER_SUBTEST" $enumd > $fltrd
61
-    test -s $fltrd  || tf_debug "TF_FILTER_SUBTEST ate everything: $TF_FILTER_SUBTEST"
62
-    for subtname in $(<$fltrd);
63
-    do
64
-        export TF_SUBTNAME=$subtname
65
-        tf_think "::: $TF_TNAME::$TF_SUBTNAME"
66
-        tf_do_subtest "$TF_SUBTNAME";
67
-        tes=$?
68
-        test $tes -gt $es            && es=$tes
69
-        test $tes -gt $TF_ES_OK      && tf_warn "!!! $TF_TNAME::$TF_SUBTNAME ($tes)"
70
-        test $tes -gt $TF_ES_BAILOUT && break
71
-    done
72
-    return $es
73
-}

+ 0
- 78
tfkit/include/tools.sh View File

@@ -1,78 +0,0 @@
1
-#!/bin/bash
2
-
3
-. $TF_DIR/include/common.sh
4
-
5
-# 1. exec: [test] -> [result]
6
-# 2. eval:           [result] == [oracle]
7
-
8
-tf_testflt() {
9
-    #
10
-    # Run a simple test for a unix filter
11
-    #
12
-    #     tf_testflt -n foo [-i foo.stdin] \
13
-    #                [-O ffoo.stdout] [-E ffoo.stderr] [-S 3] \
14
-    #                cmd arg...
15
-    #
16
-    # Will drop *result/NAME.stdout* and *result/NAME.stderr* (intentionally
17
-    # not cleaning up).
18
-    #
19
-
20
-    # defaults
21
-    #
22
-    local t_in="/dev/null"      # test: stdin
23
-    local t_name=""             # test: name
24
-                                # command is "$@" after arg parsing
25
-    local t_es="0"              # final test exit status
26
-    local o_out="/dev/null"     # oracle: stdout
27
-    local o_err="/dev/null"     # oracle: stderr
28
-    local o_es="0"              # oralce: exit status
29
-    local r_out r_err r_es      # result: ^ ^ ^ those 3
30
-
31
-    # get args
32
-    #
33
-    while true; do case "$1" in
34
-        -i) t_in="$2";          shift 2 ;;
35
-        -n) t_name="$2";        shift 2 ;;
36
-        -O) o_out="$2";         shift 2 ;;
37
-        -E) o_err="$2";         shift 2 ;;
38
-        -S) o_es="$2";          shift 2 ;;
39
-        --)                     shift; break ;;
40
-        "")                            break ;;
41
-        -*) tf_warn "wrong testcli arg: $1"; return $TF_ES_BAILOUT ;;
42
-        *)                             break ;;
43
-    esac done
44
-    tf_debug "t_in='$t_in'"
45
-    tf_debug "t_name='$t_name'"
46
-    tf_debug "o_out='$o_out'"
47
-    tf_debug "o_err='$o_err'"
48
-    tf_debug "o_es='$o_es'"
49
-    tf_debug "test command: $*"
50
-    test "$t_in" = "-" && t_in=/dev/stdin   # works better for check below
51
-    test -z "$t_name"  && { tf_warn "missing test name"             ; return $TF_ES_BAILOUT; }
52
-    test -z "$1"       && { tf_warn "missing test command"          ; return $TF_ES_BAILOUT; }
53
-    test -r "$t_in"    || { tf_warn "missing input file: $t_in"     ; return $TF_ES_BAILOUT; }
54
-    test -e "$o_out"   || { tf_warn "missing oracle stdout: $o_out" ; return $TF_ES_BAILOUT; }
55
-    test -e "$o_err"   || { tf_warn "missing oracle stderr: $o_err" ; return $TF_ES_BAILOUT; }
56
-
57
-    # prepare
58
-    #
59
-    mkdir -p result
60
-    r_out="result/$t_name.stdout"
61
-    r_err="result/$t_name.stderr"
62
-    tf_debug "r_out='$r_out'"
63
-    tf_debug "r_err='$r_err'"
64
-    touch $r_out || { tf_warn "cannot create tmp file: $r_out" ; return $TF_ES_BAILOUT; }
65
-    touch $r_err || { tf_warn "cannot create tmp file: $r_err" ; return $TF_ES_PANIC; }
66
-
67
-    # run
68
-    #
69
-    ( <$t_in eval "$@" >$r_out 2>$r_err ); r_es=$?
70
-    tf_debug "r_es='$r_es'"
71
-
72
-    # eval/report/exit
73
-    #
74
-    test $r_es = $o_es || { tf_warn "bad exit status: $r_es (need $o_es)" ; t_es=$TF_ES_FAIL; }
75
-    diff -u $o_err $r_err || t_es=$TF_ES_FAIL
76
-    diff -u $o_out $r_out || t_es=$TF_ES_FAIL
77
-    return $t_es
78
-}

+ 0
- 33
tfkit/runtests View File

@@ -1,33 +0,0 @@
1
-#!/bin/bash
2
-# ffoo test framework
3
-# See LICENSE file for copyright and license details.
4
-
5
-
6
-die() {
7
-    echo "$@" && exit 9
8
-}
9
-
10
-export LC_ALL=C
11
-export TF_DIR=${TF_DIR:-tfkit}
12
-export TF_SUITE="${TF_SUITE:-tests}"
13
-
14
-. $TF_DIR/include/harness.sh || die "this must be run from project root"
15
-
16
-
17
-usage() {
18
-    echo "usage: $(basename $0) [-c] [-t tests_re] [-s subtest_re] [-p binpath] [-v] [-d]" >&2
19
-    exit 2
20
-}
21
-
22
-while true; do case "$1" in
23
-    -c|--collect)           export TF_COLLECT=always; shift ;;
24
-    -t|--filter-test)       export TF_FILTER_TEST="$2"; shift 2 ;;
25
-    -s|--filter-subtest)    export TF_FILTER_SUBTEST="$2"; shift 2 ;;
26
-    -p|--prefix)            export PATH="$(readlink -f "$2")/bin:$PATH"; shift 2 ;;
27
-    -d|--debug)             export TF_DEBUG=true; shift ;;
28
-    -v|--verbose)           export TF_VERBOSE=true; shift ;;
29
-    "") break ;;
30
-    *)  usage ;;
31
-esac done
32
-
33
-time tf_run_tests