tokenize_test.go 3.5KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104
  1. package tokenize
  2. import "fmt"
  3. import "testing"
  4. func Test_tokenize(t *testing.T) {
  5. var test_cases = []struct {
  6. test_str string
  7. want_result tokenizeResult
  8. }{
  9. // emptiness
  10. {"", tokenizeResult{tokens: []string{}}},
  11. // unquoted whitespace
  12. {" ", tokenizeResult{tokens: []string{}}},
  13. {"\t", tokenizeResult{tokens: []string{}}},
  14. {"\n", tokenizeResult{tokens: []string{}}},
  15. {" \t\n", tokenizeResult{tokens: []string{}}},
  16. {" \tfoo", tokenizeResult{tokens: []string{"foo"}}},
  17. {"foo ", tokenizeResult{tokens: []string{"foo"}}},
  18. {"foo bar", tokenizeResult{tokens: []string{"foo", "bar"}}},
  19. {"foo\nbar", tokenizeResult{tokens: []string{"foo", "bar"}}},
  20. {"foo\tbar", tokenizeResult{tokens: []string{"foo", "bar"}}},
  21. // single quotes
  22. {"'", tokenizeResult{tokens: []string{}, code: tokenizeResultCodeMissingEndSingleQuote}},
  23. {"''", tokenizeResult{tokens: []string{""}}},
  24. {"fo''o", tokenizeResult{tokens: []string{"foo"}}},
  25. {"foo '' bar", tokenizeResult{tokens: []string{"foo", "", "bar"}}},
  26. {"foo 'and' bar", tokenizeResult{tokens: []string{"foo", "and", "bar"}}},
  27. {"foo '\\\t\n' bar", tokenizeResult{tokens: []string{"foo", "\\\t\n", "bar"}}},
  28. {"foo ' space bar '", tokenizeResult{tokens: []string{"foo", " space bar "}}},
  29. {"foo 'John \"Spaceman\" Doe'", tokenizeResult{tokens: []string{"foo", "John \"Spaceman\" Doe"}}},
  30. //
  31. }
  32. for _, tc := range test_cases {
  33. t.Run(fmt.Sprintf("%q", tc.test_str), func(t *testing.T) {
  34. have_result := tokenize(tc.test_str)
  35. if have_result.code != tc.want_result.code {
  36. t.Errorf("unexpected result .code: got %s, want %s in %v", have_result.code, tc.want_result.code, have_result)
  37. return
  38. }
  39. if have_result.err_loc != tc.want_result.err_loc {
  40. t.Errorf("unexpected result .err_loc: got %d, want %d in %v", have_result.err_loc, tc.want_result.err_loc, have_result)
  41. return
  42. }
  43. if len(have_result.tokens) != len(tc.want_result.tokens) {
  44. t.Errorf("unexpected number of result .tokens: got %d, want %d in %v", len(have_result.tokens), len(tc.want_result.tokens), have_result)
  45. return
  46. }
  47. for i := range have_result.tokens {
  48. if have_result.tokens[i] == tc.want_result.tokens[i] {
  49. continue
  50. }
  51. t.Errorf("unexpected token in result .tokens[%d]: got %q, want %q in %v", i, have_result.tokens[i], tc.want_result.tokens[i], have_result)
  52. return
  53. }
  54. })
  55. }
  56. }
  57. func Test_reader_tossUntilNeitherOf(t *testing.T) {
  58. var test_cases = []struct {
  59. test_data string
  60. test_needles string
  61. test_startpos uint
  62. want_endpos uint
  63. want_ok bool
  64. }{
  65. {"", "", 0, 0, false},
  66. {"", "x", 0, 0, false},
  67. {"", "xy", 0, 0, false},
  68. {"x", "", 0, 0, false},
  69. {"x", "x", 0, 1, true},
  70. {"x", "xy", 0, 1, true},
  71. {"x", "yx", 0, 1, true},
  72. {"xa", "x", 0, 1, true},
  73. {"xa", "xy", 0, 1, true},
  74. {"xa", "yx", 0, 1, true},
  75. {"xya", "x", 0, 1, true},
  76. {"xya", "xy", 0, 2, true},
  77. {"xya", "yx", 0, 2, true},
  78. {"xxya", "x", 1, 2, true},
  79. {"xxya", "xy", 1, 3, true},
  80. {"xxya", "yx", 1, 3, true},
  81. }
  82. for _, tc := range test_cases {
  83. t.Run(fmt.Sprintf("%q[%d:]-%q", tc.test_data, tc.test_startpos, tc.test_needles), func(t *testing.T) {
  84. test_reader := reader{data: tc.test_data}
  85. test_reader.pos = tc.test_startpos
  86. have_ok := test_reader.tossUntilNeitherOf(tc.test_needles)
  87. if test_reader.pos != tc.want_endpos {
  88. t.Errorf("unexpected position after toss: got %d, want %d", test_reader.pos, tc.want_endpos)
  89. return
  90. }
  91. if have_ok != tc.want_ok {
  92. t.Errorf("unexpected ok: got %v, want %v", have_ok, tc.want_ok)
  93. return
  94. }
  95. })
  96. }
  97. }