|
|
@@ -18,6 +18,11 @@ shellfu import termcolors
|
|
18
|
18
|
# included within the log, see BEHAVIOR EVIDENCE ID section for more
|
|
19
|
19
|
# details.
|
|
20
|
20
|
#
|
|
|
21
|
+# * `-c CASEID` Case ID; together with $JAT__TESTID can be used to
|
|
|
22
|
+# uniquely identify a particular test case, with granularity up to
|
|
|
23
|
+# single assert. CASEID set here is joined with any CASEIDs set
|
|
|
24
|
+# by active phase.
|
|
|
25
|
+#
|
|
21
|
26
|
# * `-h HINT` Human-readable description used to describe meaning of
|
|
22
|
27
|
# the assertion to reader. Note that HINT should be worded in such
|
|
23
|
28
|
# way that it makes claim about the reality that matches expectation.
|
|
|
@@ -28,6 +33,10 @@ shellfu import termcolors
|
|
28
|
33
|
#
|
|
29
|
34
|
# * `-C DIR` Change to directory DIR for duration of the phase.
|
|
30
|
35
|
#
|
|
|
36
|
+# * `-c CASEID` Inherit CASEID as Case ID to all asserts in this
|
|
|
37
|
+# phase. Individual asserts can extend the final case id for
|
|
|
38
|
+# extra granularity and profit.
|
|
|
39
|
+#
|
|
31
|
40
|
#
|
|
32
|
41
|
# STRINGS
|
|
33
|
42
|
# =======
|
|
|
@@ -149,6 +158,7 @@ jat__cmd() {
|
|
149
|
158
|
local __jat__o_es=0
|
|
150
|
159
|
local __jat__hint
|
|
151
|
160
|
local __jat__beids=()
|
|
|
161
|
+ local __jat__caseid
|
|
152
|
162
|
local __jat__etype=TEST_ERROR
|
|
153
|
163
|
local __jat__r_out
|
|
154
|
164
|
local __jat__r_err
|
|
|
@@ -159,6 +169,7 @@ jat__cmd() {
|
|
159
|
169
|
#
|
|
160
|
170
|
while true; do case $1 in
|
|
161
|
171
|
-b) __jat__beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
|
|
|
172
|
+ -c) __jat__caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
|
|
162
|
173
|
-h) __jat__hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
|
|
163
|
174
|
-e) __jat__r_err=$2; shift 2 || { __jat__usage "missing R_ERR"; return 2; } ;;
|
|
164
|
175
|
-o) __jat__r_out=$2; shift 2 || { __jat__usage "missing R_OUT"; return 2; } ;;
|
|
|
@@ -188,7 +199,7 @@ jat__cmd() {
|
|
188
|
199
|
else
|
|
189
|
200
|
__jat__etype=FAIL
|
|
190
|
201
|
fi
|
|
191
|
|
- __jat__assert $__jat__etype "$__jat__hint" "${__jat__beids[@]}" \
|
|
|
202
|
+ __jat__assert $__jat__etype "$__jat__hint" "$__jat__caseid" "${__jat__beids[@]}" \
|
|
192
|
203
|
-- "t.cmd=${__jat__cmd[*]}" "o.es_expr=$__jat__o_es" "r.es=$__jat__r_es"
|
|
193
|
204
|
return "$__jat__r_es"
|
|
194
|
205
|
}
|
|
|
@@ -218,10 +229,12 @@ jat__cmp() {
|
|
218
|
229
|
local Op # operator
|
|
219
|
230
|
local OVal # "oracle" value
|
|
220
|
231
|
local hint # log hint
|
|
|
232
|
+ local caseid # case id
|
|
221
|
233
|
local beids=() # Behavior Evidence IDs
|
|
222
|
234
|
local cmpes # comparison exit status
|
|
223
|
235
|
while true; do case $1 in
|
|
224
|
236
|
-b) beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
|
|
|
237
|
+ -c) caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
|
|
225
|
238
|
-h) hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
|
|
226
|
239
|
*) break ;;
|
|
227
|
240
|
esac done
|
|
|
@@ -234,11 +247,11 @@ jat__cmp() {
|
|
234
|
247
|
__jat__cmp_match; cmpes=$?
|
|
235
|
248
|
case $cmpes in
|
|
236
|
249
|
0)
|
|
237
|
|
- __jat__assert PASS "$hint" "${beids[@]}" \
|
|
|
250
|
+ __jat__assert PASS "$hint" "$caseid" "${beids[@]}" \
|
|
238
|
251
|
-- "r.val=$RVal" "t.op=$Op" "o.val=$OVal"
|
|
239
|
252
|
;;
|
|
240
|
253
|
1)
|
|
241
|
|
- __jat__assert FAIL "$hint" "${beids[@]}" \
|
|
|
254
|
+ __jat__assert FAIL "$hint" "$caseid" "${beids[@]}" \
|
|
242
|
255
|
-- "r.val=$RVal" "t.op=$Op" "o.val=$OVal"
|
|
243
|
256
|
;;
|
|
244
|
257
|
*)
|
|
|
@@ -278,6 +291,7 @@ jat__eval() {
|
|
278
|
291
|
# call is incomplete, the exit status will be 2. If there is syntax error
|
|
279
|
292
|
# in CODE, the exit status will be 3.
|
|
280
|
293
|
#
|
|
|
294
|
+ local __jat__caseid
|
|
281
|
295
|
local __jat__code=""
|
|
282
|
296
|
local __jat__r_es
|
|
283
|
297
|
local __jat__r_esf
|
|
|
@@ -291,6 +305,7 @@ jat__eval() {
|
|
291
|
305
|
#
|
|
292
|
306
|
while true; do case $1 in
|
|
293
|
307
|
-b) __jat__beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
|
|
|
308
|
+ -c) __jat__caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
|
|
294
|
309
|
-h) __jat__hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
|
|
295
|
310
|
-s) __jat__r_esf=$2; shift 2 || { __jat__usage "missing R_ESF"; return 2; } ;;
|
|
296
|
311
|
-S) __jat__o_es=$2; shift 2 || { __jat__usage "missing ES_EXPR"; return 2; } ;;
|
|
|
@@ -318,7 +333,7 @@ jat__eval() {
|
|
318
|
333
|
else
|
|
319
|
334
|
__jat__etype=FAIL
|
|
320
|
335
|
fi
|
|
321
|
|
- __jat__assert $__jat__etype "$__jat__hint" "${__jat__beids[@]}" \
|
|
|
336
|
+ __jat__assert $__jat__etype "$__jat__hint" "$__jat__caseid" "${__jat__beids[@]}" \
|
|
322
|
337
|
-- "t.code=$__jat__code" "o.es_expr=$__jat__o_es" "r.es=$__jat__r_es"
|
|
323
|
338
|
return "$__jat__r_es"
|
|
324
|
339
|
}
|
|
|
@@ -338,15 +353,17 @@ jat__fail() {
|
|
338
|
353
|
# See COMMON ARGUMENTS section for common assert options.
|
|
339
|
354
|
#
|
|
340
|
355
|
local hint # log hint
|
|
|
356
|
+ local caseid # case ID
|
|
341
|
357
|
local beids=() # Behavior Evidence IDs
|
|
342
|
358
|
while true; do case $1 in
|
|
343
|
359
|
-b) beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
|
|
|
360
|
+ -c) caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
|
|
344
|
361
|
-h) hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
|
|
345
|
362
|
*) break ;;
|
|
346
|
363
|
esac done
|
|
347
|
364
|
test -n "$1" && { __jat__usage "extra arguments: $*"; return 2; }
|
|
348
|
365
|
debug -v hint beids
|
|
349
|
|
- __jat__assert FAIL "$hint" "${beids[@]}"
|
|
|
366
|
+ __jat__assert FAIL "$hint" "$caseid" "${beids[@]}"
|
|
350
|
367
|
}
|
|
351
|
368
|
|
|
352
|
369
|
jat__filebackup() {
|
|
|
@@ -536,14 +553,16 @@ jat__pass() {
|
|
536
|
553
|
#
|
|
537
|
554
|
local hint # log hint
|
|
538
|
555
|
local beids=() # Behavior Evidence IDs
|
|
|
556
|
+ local caseid # case ID
|
|
539
|
557
|
while true; do case $1 in
|
|
540
|
558
|
-b) beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
|
|
|
559
|
+ -c) caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
|
|
541
|
560
|
-h) hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
|
|
542
|
561
|
*) break ;;
|
|
543
|
562
|
esac done
|
|
544
|
563
|
test -n "$1" && { __jat__usage "extra arguments: $*"; return 2; }
|
|
545
|
564
|
debug -v hint beids
|
|
546
|
|
- __jat__assert PASS "$hint" "${beids[@]}"
|
|
|
565
|
+ __jat__assert PASS "$hint" "$caseid" "${beids[@]}"
|
|
547
|
566
|
}
|
|
548
|
567
|
|
|
549
|
568
|
jat__promise_asserts() {
|
|
|
@@ -555,7 +574,7 @@ jat__promise_asserts() {
|
|
555
|
574
|
__jat__usage "no NUM?"
|
|
556
|
575
|
return 2
|
|
557
|
576
|
}
|
|
558
|
|
- __jat__log_event PROMISE "assert number: $num" \
|
|
|
577
|
+ __jat__log_event PROMISE "assert number: $num" "" \
|
|
559
|
578
|
-- "num=$num"
|
|
560
|
579
|
}
|
|
561
|
580
|
|
|
|
@@ -576,7 +595,7 @@ jat__sinit() {
|
|
576
|
595
|
export JAT__YLOG
|
|
577
|
596
|
debug -v reload __JAT__SDIR
|
|
578
|
597
|
if $reload; then
|
|
579
|
|
- __jat__log_event SINFO "reloaded session" \
|
|
|
598
|
+ __jat__log_event SINFO "reloaded session" "" \
|
|
580
|
599
|
-- \
|
|
581
|
600
|
"JAT__LOG_FMT=$__JAT__LOG_FMT" \
|
|
582
|
601
|
"JAT__VERSION=$__JAT__SELF_VERSION"
|
|
|
@@ -790,7 +809,7 @@ __jat__log_event() {
|
|
790
|
809
|
#
|
|
791
|
810
|
# Usage:
|
|
792
|
811
|
#
|
|
793
|
|
- # __jat__log_event ETYPE HINT [BEID].. -- [KEY=VALUE]..
|
|
|
812
|
+ # __jat__log_event ETYPE HINT CASEID [BEID].. -- [KEY=VALUE]..
|
|
794
|
813
|
#
|
|
795
|
814
|
# ETYPE can be FAIL, PASS, TEST_ERROR or SINFO (the latter two
|
|
796
|
815
|
# are for events unrelated to SUT, like internal errors or
|
|
|
@@ -807,13 +826,19 @@ __jat__log_event() {
|
|
807
|
826
|
# is relevant to test method, result or oracle, respectively.
|
|
808
|
827
|
local etype=$1; shift
|
|
809
|
828
|
local hint=$1; shift
|
|
|
829
|
+ local acaseid=$1; shift
|
|
810
|
830
|
local arg
|
|
811
|
831
|
local beids=()
|
|
812
|
832
|
local pairs=()
|
|
813
|
833
|
local real_beids=()
|
|
|
834
|
+ local caseid="$JAT__TESTID"
|
|
|
835
|
+ local pcaseid
|
|
814
|
836
|
local reading=beids
|
|
815
|
837
|
local pair
|
|
816
|
838
|
local origin=${FUNCNAME[1]}
|
|
|
839
|
+ pcaseid=$(__jat__sd_keyR P.caseid)
|
|
|
840
|
+ test -n "$pcaseid" && caseid+=":$pcaseid"
|
|
|
841
|
+ test -n "$acaseid" && caseid+=":$acaseid"
|
|
817
|
842
|
case $etype in
|
|
818
|
843
|
PASS|FAIL|SINFO|TEST_ERROR|TEST_WARNING|PROMISE) : ;;
|
|
819
|
844
|
*) __jat__show_error "bad ETYPE, changing to TEST_ERROR: $etype"
|
|
|
@@ -882,6 +907,7 @@ __jat__log_event() {
|
|
882
|
907
|
echo " origin: $origin"
|
|
883
|
908
|
echo " etype: $etype"
|
|
884
|
909
|
echo " stamp: $(__jat__newstamp)"
|
|
|
910
|
+ echo " caseid: $caseid"
|
|
885
|
911
|
__jat__yamls hint "$hint"
|
|
886
|
912
|
__jat__yamla beids "${real_beids[@]}"
|
|
887
|
913
|
__jat__yamld data "${pairs[@]}"
|
|
|
@@ -919,9 +945,11 @@ __jat__pstart() {
|
|
919
|
945
|
local type # phase name
|
|
920
|
946
|
local name # ^^ name
|
|
921
|
947
|
local pdir # ^^ directory
|
|
|
948
|
+ local pcaseid # ^^ case id, if assigned
|
|
922
|
949
|
local oldphase # current phase id
|
|
923
|
950
|
while true; do case $1 in
|
|
924
|
951
|
-t) type="$2"; shift 2 || return 2 ;;
|
|
|
952
|
+ -c) pcaseid="$2"; shift 2 || return 2 ;;
|
|
925
|
953
|
-C) pdir="$2"; shift 2 || return 2 ;;
|
|
926
|
954
|
*) break ;;
|
|
927
|
955
|
esac done
|
|
|
@@ -935,6 +963,7 @@ __jat__pstart() {
|
|
935
|
963
|
__jat__sd_keyw phase "$(__jat__bumpid phasid)"
|
|
936
|
964
|
__jat__sd_keyw P.type "$type"
|
|
937
|
965
|
__jat__sd_keyw P.name "$name"
|
|
|
966
|
+ __jat__sd_keyw P.caseid "$pcaseid"
|
|
938
|
967
|
__jat__show_pstart "$name"
|
|
939
|
968
|
__jat__log_event SINFO
|
|
940
|
969
|
test -n "$pdir" && {
|
|
|
@@ -1168,7 +1197,7 @@ __jat__usage() {
|
|
1168
|
1197
|
for patt in "${patts[@]}"; do
|
|
1169
|
1198
|
__jat__show_error "usage: $parent $patt"
|
|
1170
|
1199
|
done
|
|
1171
|
|
- __jat__log_event TEST_ERROR "bad usage: $parent()" \
|
|
|
1200
|
+ __jat__log_event TEST_ERROR "bad usage: $parent()" "" \
|
|
1172
|
1201
|
-- "PATTERNS=${patts[*]}"
|
|
1173
|
1202
|
}
|
|
1174
|
1203
|
|
Alois Mahdal
6 years ago