jats
/
shellfu-bash-jat
Tarkkaile 1
Äänestä 0
Fork 0
Koodi Julkaisut 4 Activity
Selaa lähdekoodia

Add support for case ids

Alois Mahdal 6 vuotta sitten
vanhempi
80b5fdb845
commit
c1df289eff
4 muutettua tiedostoa jossa 63 lisäystä ja 10 poistoa
Jaettu näkymä Näytä diff tilastot
  1. 39
    10
      src/jat.sh.skel
  2. 8
    0
      tests/basic/oracle/ylog_both.stdout
  3. 8
    0
      tests/basic/oracle/ylog_fail.stdout
  4. 8
    0
      tests/basic/oracle/ylog_pass.stdout

+ 39
- 10
src/jat.sh.skel Näytä tiedosto 

@@ -18,6 +18,11 @@ shellfu import termcolors
18 18 
 #     included within the log, see BEHAVIOR EVIDENCE ID section for more
19 19 
 #     details.
20 20 
 #
21 
+#  *  `-c CASEID`   Case ID; together with $JAT__TESTID can be used to
22 
+#     uniquely identify a particular test case, with granularity up to
23 
+#     single assert.  CASEID set here is joined with any CASEIDs set
24 
+#     by active phase.
25 
+#
21 26 
 #  *  `-h HINT`   Human-readable description used to describe meaning of
22 27 
 #     the assertion to reader.  Note that HINT should be worded in such
23 28 
 #     way that it makes claim about the reality that matches expectation.
 
@@ -28,6 +33,10 @@ shellfu import termcolors
28 33 
 #
29 34 
 #  *  `-C DIR`  Change to directory DIR for duration of the phase.
30 35 
 #
36 
+#  *  `-c CASEID`  Inherit CASEID as Case ID to all asserts in this
37 
+#     phase.  Individual asserts can extend the final case id for
38 
+#     extra granularity and profit.
39 
+#
31 40 
 #
32 41 
 # STRINGS
33 42 
 # =======
 
@@ -149,6 +158,7 @@ jat__cmd() {
149 158 
     local __jat__o_es=0
150 159 
     local __jat__hint
151 160 
     local __jat__beids=()
161 
+    local __jat__caseid
152 162 
     local __jat__etype=TEST_ERROR
153 163 
     local __jat__r_out
154 164 
     local __jat__r_err
 
@@ -159,6 +169,7 @@ jat__cmd() {
159 169 
     #
160 170 
     while true; do case $1 in
161 171 
         -b) __jat__beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
172 
+        -c) __jat__caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
162 173 
         -h) __jat__hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
163 174 
         -e) __jat__r_err=$2; shift 2 || { __jat__usage "missing R_ERR"; return 2; } ;;
164 175 
         -o) __jat__r_out=$2; shift 2 || { __jat__usage "missing R_OUT"; return 2; } ;;
 
@@ -188,7 +199,7 @@ jat__cmd() {
188 199 
     else
189 200 
         __jat__etype=FAIL
190 201 
     fi
191 
-    __jat__assert $__jat__etype "$__jat__hint" "${__jat__beids[@]}" \
202 
+    __jat__assert $__jat__etype "$__jat__hint" "$__jat__caseid" "${__jat__beids[@]}" \
192 203 
         -- "t.cmd=${__jat__cmd[*]}" "o.es_expr=$__jat__o_es" "r.es=$__jat__r_es"
193 204 
     return "$__jat__r_es"
194 205 
 }
 
@@ -218,10 +229,12 @@ jat__cmp() {
218 229 
     local Op        # operator
219 230 
     local OVal      # "oracle" value
220 231 
     local hint      # log hint
232 
+    local caseid    # case id
221 233 
     local beids=()  # Behavior Evidence IDs
222 234 
     local cmpes     # comparison exit status
223 235 
     while true; do case $1 in
224 236 
         -b) beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
237 
+        -c) caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
225 238 
         -h) hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
226 239 
         *) break ;;
227 240 
     esac done
 
@@ -234,11 +247,11 @@ jat__cmp() {
234 247 
     __jat__cmp_match; cmpes=$?
235 248 
     case $cmpes in
236 249 
         0)
237 
-            __jat__assert PASS "$hint" "${beids[@]}" \
250 
+            __jat__assert PASS "$hint" "$caseid" "${beids[@]}" \
238 251 
                 -- "r.val=$RVal" "t.op=$Op" "o.val=$OVal"
239 252 
             ;;
240 253 
         1)
241 
-            __jat__assert FAIL "$hint" "${beids[@]}" \
254 
+            __jat__assert FAIL "$hint" "$caseid" "${beids[@]}" \
242 255 
                 -- "r.val=$RVal" "t.op=$Op" "o.val=$OVal"
243 256 
             ;;
244 257 
         *)
 
@@ -278,6 +291,7 @@ jat__eval() {
278 291 
     # call is incomplete, the exit status will be 2.  If there is syntax error
279 292 
     # in CODE, the exit status will be 3.
280 293 
     #
294 
+    local __jat__caseid
281 295 
     local __jat__code=""
282 296 
     local __jat__r_es
283 297 
     local __jat__r_esf
 
@@ -291,6 +305,7 @@ jat__eval() {
291 305 
     #
292 306 
     while true; do case $1 in
293 307 
         -b) __jat__beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
308 
+        -c) __jat__caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
294 309 
         -h) __jat__hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
295 310 
         -s) __jat__r_esf=$2; shift 2 || { __jat__usage "missing R_ESF"; return 2; } ;;
296 311 
         -S) __jat__o_es=$2; shift 2 || { __jat__usage "missing ES_EXPR"; return 2; } ;;
 
@@ -318,7 +333,7 @@ jat__eval() {
318 333 
     else
319 334 
         __jat__etype=FAIL
320 335 
     fi
321 
-    __jat__assert $__jat__etype "$__jat__hint" "${__jat__beids[@]}" \
336 
+    __jat__assert $__jat__etype "$__jat__hint" "$__jat__caseid" "${__jat__beids[@]}" \
322 337 
         -- "t.code=$__jat__code" "o.es_expr=$__jat__o_es" "r.es=$__jat__r_es"
323 338 
     return "$__jat__r_es"
324 339 
 }
 
@@ -338,15 +353,17 @@ jat__fail() {
338 353 
     # See COMMON ARGUMENTS section for common assert options.
339 354 
     #
340 355 
     local hint      # log hint
356 
+    local caseid    # case ID
341 357 
     local beids=()  # Behavior Evidence IDs
342 358 
     while true; do case $1 in
343 359 
         -b) beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
360 
+        -c) caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
344 361 
         -h) hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
345 362 
         *) break ;;
346 363 
     esac done
347 364 
     test -n "$1" && { __jat__usage "extra arguments: $*"; return 2; }
348 365 
     debug -v hint beids
349 
-    __jat__assert FAIL "$hint" "${beids[@]}"
366 
+    __jat__assert FAIL "$hint" "$caseid" "${beids[@]}"
350 367 
 }
351 368
352 369 
 jat__filebackup() {
 
@@ -536,14 +553,16 @@ jat__pass() {
536 553 
     #
537 554 
     local hint      # log hint
538 555 
     local beids=()  # Behavior Evidence IDs
556 
+    local caseid    # case ID
539 557 
     while true; do case $1 in
540 558 
         -b) beids+=("$2"); shift 2 || { __jat__usage "missing BEID"; return 2; } ;;
559 
+        -c) caseid="$2"; shift 2 || { __jat__usage "missing CASEID"; return 2; } ;;
541 560 
         -h) hint=$2; shift 2 || { __jat__usage "missing HINT"; return 2; } ;;
542 561 
         *) break ;;
543 562 
     esac done
544 563 
     test -n "$1" && { __jat__usage "extra arguments: $*"; return 2; }
545 564 
     debug -v hint beids
546 
-    __jat__assert PASS "$hint" "${beids[@]}"
565 
+    __jat__assert PASS "$hint" "$caseid" "${beids[@]}"
547 566 
 }
548 567
549 568 
 jat__promise_asserts() {
 
@@ -555,7 +574,7 @@ jat__promise_asserts() {
555 574 
         __jat__usage "no NUM?"
556 575 
         return 2
557 576 
     }
558 
-    __jat__log_event PROMISE "assert number: $num" \
577 
+    __jat__log_event PROMISE "assert number: $num" "" \
559 578 
         -- "num=$num"
560 579 
 }
561 580 
 
@@ -576,7 +595,7 @@ jat__sinit() {
576 595 
     export JAT__YLOG
577 596 
     debug -v reload __JAT__SDIR
578 597 
     if $reload; then
579 
-        __jat__log_event SINFO "reloaded session" \
598 
+        __jat__log_event SINFO "reloaded session" "" \
580 599 
             -- \
581 600 
             "JAT__LOG_FMT=$__JAT__LOG_FMT" \
582 601 
             "JAT__VERSION=$__JAT__SELF_VERSION"
 
@@ -790,7 +809,7 @@ __jat__log_event() {
790 809 
     #
791 810 
     # Usage:
792 811 
     #
793 
-    #     __jat__log_event ETYPE HINT [BEID].. -- [KEY=VALUE]..
812 
+    #     __jat__log_event ETYPE HINT CASEID [BEID].. -- [KEY=VALUE]..
794 813 
     #
795 814 
     # ETYPE can be FAIL, PASS, TEST_ERROR or SINFO (the latter two
796 815 
     # are for events unrelated to SUT, like internal errors or
 
@@ -807,13 +826,19 @@ __jat__log_event() {
807 826 
     # is relevant to test method, result or oracle, respectively.
808 827 
     local etype=$1; shift
809 828 
     local hint=$1;    shift
829 
+    local acaseid=$1; shift
810 830 
     local arg
811 831 
     local beids=()
812 832 
     local pairs=()
813 833 
     local real_beids=()
834 
+    local caseid="$JAT__TESTID"
835 
+    local pcaseid
814 836 
     local reading=beids
815 837 
     local pair
816 838 
     local origin=${FUNCNAME[1]}
839 
+    pcaseid=$(__jat__sd_keyR P.caseid)
840 
+    test -n "$pcaseid" && caseid+=":$pcaseid"
841 
+    test -n "$acaseid" && caseid+=":$acaseid"
817 842 
     case $etype in
818 843 
         PASS|FAIL|SINFO|TEST_ERROR|TEST_WARNING|PROMISE) : ;;
819 844 
         *)  __jat__show_error "bad ETYPE, changing to TEST_ERROR: $etype"
 
@@ -882,6 +907,7 @@ __jat__log_event() {
882 907 
         echo "    origin: $origin"
883 908 
         echo "    etype: $etype"
884 909 
         echo "    stamp: $(__jat__newstamp)"
910 
+        echo "    caseid: $caseid"
885 911 
         __jat__yamls hint "$hint"
886 912 
         __jat__yamla beids "${real_beids[@]}"
887 913 
         __jat__yamld data "${pairs[@]}"
 
@@ -919,9 +945,11 @@ __jat__pstart() {
919 945 
     local type      # phase name
920 946 
     local name      # ^^ name
921 947 
     local pdir      # ^^ directory
948 
+    local pcaseid   # ^^ case id, if assigned
922 949 
     local oldphase  # current phase id
923 950 
     while true; do case $1 in
924 951 
         -t) type="$2"; shift 2 || return 2 ;;
952 
+        -c) pcaseid="$2"; shift 2 || return 2 ;;
925 953 
         -C) pdir="$2"; shift 2 || return 2 ;;
926 954 
         *)  break ;;
927 955 
     esac done
 
@@ -935,6 +963,7 @@ __jat__pstart() {
935 963 
     __jat__sd_keyw phase "$(__jat__bumpid phasid)"
936 964 
     __jat__sd_keyw P.type "$type"
937 965 
     __jat__sd_keyw P.name "$name"
966 
+    __jat__sd_keyw P.caseid "$pcaseid"
938 967 
     __jat__show_pstart "$name"
939 968 
     __jat__log_event SINFO
940 969 
     test -n "$pdir" && {
 
@@ -1168,7 +1197,7 @@ __jat__usage() {
1168 1197 
     for patt in "${patts[@]}"; do
1169 1198 
         __jat__show_error "usage: $parent $patt"
1170 1199 
     done
1171 
-    __jat__log_event TEST_ERROR "bad usage: $parent()" \
1200 
+    __jat__log_event TEST_ERROR "bad usage: $parent()" "" \
1172 1201 
         -- "PATTERNS=${patts[*]}"
1173 1202 
 }
1174 1203

+ 8
- 0
tests/basic/oracle/ylog_both.stdout Näytä tiedosto 

@@ -9,6 +9,7 @@ events:
9 9 
         origin: jat__sinit
10 10 
         etype: SINFO
11 11 
         stamp: time-2
12 
+        caseid: _jat_anon_test_
12 13 
         hint: |-
13 14 
             started new session
14 15 
         beids: []
 
@@ -21,6 +22,7 @@ events:
21 22 
         origin: __jat__pstart
22 23 
         etype: SINFO
23 24 
         stamp: time-3
25 
+        caseid: _jat_anon_test_
24 26 
         hint: ~
25 27 
         beids: []
26 28 
         data: {}
 
@@ -32,6 +34,7 @@ events:
32 34 
         origin: jat__log_info
33 35 
         etype: SINFO
34 36 
         stamp: time-4
37 
+        caseid: _jat_anon_test_
35 38 
         hint: |-
36 39 
             CMD: true
37 40 
         beids: []
 
@@ -44,6 +47,7 @@ events:
44 47 
         origin: __jat__assert
45 48 
         etype: PASS
46 49 
         stamp: time-5
50 
+        caseid: _jat_anon_test_
47 51 
         hint: |-
48 52 
             true is true
49 53 
         beids: []
 
@@ -62,6 +66,7 @@ events:
62 66 
         origin: jat__log_info
63 67 
         etype: SINFO
64 68 
         stamp: time-6
69 
+        caseid: _jat_anon_test_
65 70 
         hint: |-
66 71 
             CMD: false
67 72 
         beids: []
 
@@ -74,6 +79,7 @@ events:
74 79 
         origin: __jat__assert
75 80 
         etype: FAIL
76 81 
         stamp: time-7
82 
+        caseid: _jat_anon_test_
77 83 
         hint: |-
78 84 
             false is true
79 85 
         beids: []
 
@@ -92,6 +98,7 @@ events:
92 98 
         origin: jat__pend
93 99 
         etype: SINFO
94 100 
         stamp: time-8
101 
+        caseid: _jat_anon_test_
95 102 
         hint: ~
96 103 
         beids: []
97 104 
         data: {}
 
@@ -103,6 +110,7 @@ events:
103 110 
         origin: jat__sfinish
104 111 
         etype: SINFO
105 112 
         stamp: time-9
113 
+        caseid: _jat_anon_test_
106 114 
         hint: |-
107 115 
             finishing session
108 116 
         beids: []

+ 8
- 0
tests/basic/oracle/ylog_fail.stdout Näytä tiedosto 

@@ -9,6 +9,7 @@ events:
9 9 
         origin: jat__sinit
10 10 
         etype: SINFO
11 11 
         stamp: time-2
12 
+        caseid: _jat_anon_test_
12 13 
         hint: |-
13 14 
             started new session
14 15 
         beids: []
 
@@ -21,6 +22,7 @@ events:
21 22 
         origin: __jat__pstart
22 23 
         etype: SINFO
23 24 
         stamp: time-3
25 
+        caseid: _jat_anon_test_
24 26 
         hint: ~
25 27 
         beids: []
26 28 
         data: {}
 
@@ -32,6 +34,7 @@ events:
32 34 
         origin: jat__log_info
33 35 
         etype: SINFO
34 36 
         stamp: time-4
37 
+        caseid: _jat_anon_test_
35 38 
         hint: |-
36 39 
             CMD: true
37 40 
         beids: []
 
@@ -44,6 +47,7 @@ events:
44 47 
         origin: __jat__assert
45 48 
         etype: FAIL
46 49 
         stamp: time-5
50 
+        caseid: _jat_anon_test_
47 51 
         hint: |-
48 52 
             true is false
49 53 
         beids: []
 
@@ -62,6 +66,7 @@ events:
62 66 
         origin: jat__log_info
63 67 
         etype: SINFO
64 68 
         stamp: time-6
69 
+        caseid: _jat_anon_test_
65 70 
         hint: |-
66 71 
             CMD: false
67 72 
         beids: []
 
@@ -74,6 +79,7 @@ events:
74 79 
         origin: __jat__assert
75 80 
         etype: FAIL
76 81 
         stamp: time-7
82 
+        caseid: _jat_anon_test_
77 83 
         hint: |-
78 84 
             false is true
79 85 
         beids: []
 
@@ -92,6 +98,7 @@ events:
92 98 
         origin: jat__pend
93 99 
         etype: SINFO
94 100 
         stamp: time-8
101 
+        caseid: _jat_anon_test_
95 102 
         hint: ~
96 103 
         beids: []
97 104 
         data: {}
 
@@ -103,6 +110,7 @@ events:
103 110 
         origin: jat__sfinish
104 111 
         etype: SINFO
105 112 
         stamp: time-9
113 
+        caseid: _jat_anon_test_
106 114 
         hint: |-
107 115 
             finishing session
108 116 
         beids: []

+ 8
- 0
tests/basic/oracle/ylog_pass.stdout Näytä tiedosto 

@@ -9,6 +9,7 @@ events:
9 9 
         origin: jat__sinit
10 10 
         etype: SINFO
11 11 
         stamp: time-2
12 
+        caseid: _jat_anon_test_
12 13 
         hint: |-
13 14 
             started new session
14 15 
         beids: []
 
@@ -21,6 +22,7 @@ events:
21 22 
         origin: __jat__pstart
22 23 
         etype: SINFO
23 24 
         stamp: time-3
25 
+        caseid: _jat_anon_test_
24 26 
         hint: ~
25 27 
         beids: []
26 28 
         data: {}
 
@@ -32,6 +34,7 @@ events:
32 34 
         origin: jat__log_info
33 35 
         etype: SINFO
34 36 
         stamp: time-4
37 
+        caseid: _jat_anon_test_
35 38 
         hint: |-
36 39 
             CMD: true
37 40 
         beids: []
 
@@ -44,6 +47,7 @@ events:
44 47 
         origin: __jat__assert
45 48 
         etype: PASS
46 49 
         stamp: time-5
50 
+        caseid: _jat_anon_test_
47 51 
         hint: |-
48 52 
             true is true
49 53 
         beids: []
 
@@ -62,6 +66,7 @@ events:
62 66 
         origin: jat__log_info
63 67 
         etype: SINFO
64 68 
         stamp: time-6
69 
+        caseid: _jat_anon_test_
65 70 
         hint: |-
66 71 
             CMD: false
67 72 
         beids: []
 
@@ -74,6 +79,7 @@ events:
74 79 
         origin: __jat__assert
75 80 
         etype: PASS
76 81 
         stamp: time-7
82 
+        caseid: _jat_anon_test_
77 83 
         hint: |-
78 84 
             false is false
79 85 
         beids: []
 
@@ -92,6 +98,7 @@ events:
92 98 
         origin: jat__pend
93 99 
         etype: SINFO
94 100 
         stamp: time-8
101 
+        caseid: _jat_anon_test_
95 102 
         hint: ~
96 103 
         beids: []
97 104 
         data: {}
 
@@ -103,6 +110,7 @@ events:
103 110 
         origin: jat__sfinish
104 111 
         etype: SINFO
105 112 
         stamp: time-9
113 
+        caseid: _jat_anon_test_
106 114 
         hint: |-
107 115 
             finishing session
108 116 
         beids: []