Implement #GU3

app/tokenize/tokenize.go

@@ -47,6 +47,27 @@ func (self *reader) takeUntil(needle string) (string, bool) {
 	return out, true
 }
 
+func (self *reader) takeUntilAnyOf(needles string) (string, bool) {
+	self.checkPos()
+	to_take := uint(0)
+	found := false
+	for {
+		idx := self.pos + to_take
+		if idx == self.len() {
+			break
+		}
+		if hasByte(self.data[idx], needles) {
+			break
+		}
+		to_take += 1
+		found = true
+	}
+	end := self.pos + to_take
+	out := self.data[self.pos:end]
+	self.pos = end
+	return out, found
+}
+
 func (self *reader) tossUntilNeitherOf(needles string) bool {
 	self.checkPos()
 	tossed := uint(0)
@@ -79,6 +100,16 @@ func (self *reader) tossChar() {
 	self.pos += 1
 }
 
+func (self *reader) tossChars(n uint) bool {
+	self.checkPos()
+	end := self.pos + n
+	if end > self.len() {
+		return false
+	}
+	self.pos = end
+	return true
+}
+
 func (self *reader) done() bool {
 	self.checkPos()
 	return self.pos == self.len()
@@ -150,6 +181,45 @@ const (
 	tokenizeResultCodeMissingEscapedCharacter
 )
 
+func tokenize_dq(str string) (string, tokenizeResultCode, uint) {
+	buf := ""
+	rdr := reader{data: str}
+	for {
+		dq_char, ok := rdr.peekChar()
+		if !ok {
+			return "", tokenizeResultCodeMissingEndDoubleQuote, rdr.pos
+		}
+		switch dq_char {
+		case '\\':
+			rdr.tossChar()
+			next_char, ok := rdr.peekChar()
+			if !ok {
+				return "", tokenizeResultCodeMissingEscapedCharacter, rdr.pos - 1
+			}
+			switch next_char {
+			case '"':
+				rdr.tossChar()
+				buf = buf + string(next_char)
+			case '\\':
+				rdr.tossChar()
+				buf = buf + string(next_char)
+			default:
+				buf = buf + "\\"
+			}
+		case '"':
+			rdr.tossChar()
+			return buf, tokenizeResultCodeOk, rdr.pos
+		default:
+			dq_chars, found := rdr.takeUntilAnyOf("\\\"")
+			if !found {
+				return "", tokenizeResultCodeMissingEndDoubleQuote, rdr.pos
+			}
+			buf = buf + dq_chars
+		}
+	}
+
+}
+
 func tokenize(str string) tokenizeResult {
 	rdr := reader{data: str}
 	b := make_builder()
@@ -178,12 +248,20 @@ func tokenize(str string) tokenizeResult {
 			rdr.tossChar() // the second `'`
 		case '"':
 			rdr.tossChar() // first `"`
-			new_chars, found := rdr.takeUntil("\"")
-			if !found {
-				return tokenizeResult{code: tokenizeResultCodeMissingEndDoubleQuote, err_loc: rdr.pos - 1}
+			dq_chars, code, dq_rdr_taken := tokenize_dq(rdr.data[rdr.pos:])
+			switch code {
+			case tokenizeResultCodeOk:
+				ok := rdr.tossChars(dq_rdr_taken)
+				if !ok {
+					panic(fmt.Sprintf(
+						"invalid tokenize_dq() result: claims to have taken %d chars but there were only %d",
+						dq_rdr_taken, rdr.len()-rdr.pos,
+					))
+				}
+				b.bufAppend(dq_chars)
+			default:
+				return tokenizeResult{code: code, err_loc: dq_rdr_taken}
 			}
-			b.bufAppend(new_chars)
-			rdr.tossChar() // the second `"`
 		case '\\':
 			rdr.tossChar()
 			if rdr.done() {

app/tokenize/tokenize_test.go

@@ -39,7 +39,7 @@ func Test_tokenize(t *testing.T) {
 		{"fo\"\"o", tokenizeResult{tokens: []string{"foo"}}},
 		{"foo \"\" bar", tokenizeResult{tokens: []string{"foo", "", "bar"}}},
 		{"foo \"and\" bar", tokenizeResult{tokens: []string{"foo", "and", "bar"}}},
-		{"foo \"\\\t\n\" bar", tokenizeResult{tokens: []string{"foo", "\\\t\n", "bar"}}},
+		{"foo \"\\\\\t\n\" bar", tokenizeResult{tokens: []string{"foo", "\\\t\n", "bar"}}},
 		{"foo \" space bar \"", tokenizeResult{tokens: []string{"foo", " space bar "}}},
 		{"foo \"Joe's lunch\"", tokenizeResult{tokens: []string{"foo", "Joe's lunch"}}},
 
@@ -58,6 +58,19 @@ func Test_tokenize(t *testing.T) {
 		{"foo \\\\ bar", tokenizeResult{tokens: []string{"foo", "\\", "bar"}}},
 		{"foo \\'bar\\' baz", tokenizeResult{tokens: []string{"foo", "'bar'", "baz"}}},
 
+		// backslash within double quotes
+		{"\"\\", tokenizeResult{tokens: []string{}, code: tokenizeResultCodeMissingEscapedCharacter}},
+		{"\"\\\"", tokenizeResult{tokens: []string{}, code: tokenizeResultCodeMissingEndDoubleQuote, err_loc: 2}},
+		{"\"  \\\"  \"", tokenizeResult{tokens: []string{"  \"  "}}},
+		{"\"  \\\\  \"", tokenizeResult{tokens: []string{"  \\  "}}},
+		{"\"  \\x  \"", tokenizeResult{tokens: []string{"  \\x  "}}},
+		{"fo\"o\\\\b\"ar", tokenizeResult{tokens: []string{"foo\\bar"}}},
+		{"fo\"o \\\\ \"bar\" b\"az", tokenizeResult{tokens: []string{"foo \\ bar baz"}}},
+
+		// CC cases
+		{"cat \"/tmp/fox/'f 2'\" \"/tmp/fox/'f  \\73'\" \"/tmp/fox/'f \\21\\'\"",
+			tokenizeResult{tokens: []string{"cat", "/tmp/fox/'f 2'", "/tmp/fox/'f  \\73'", "/tmp/fox/'f \\21\\'"}},
+		},
 		//
 	}
 	for _, tc := range test_cases {
@@ -80,6 +93,9 @@ func Test_tokenize(t *testing.T) {
 					continue
 				}
 				t.Errorf("unexpected token in result .tokens[%d]: got %q, want %q in %v", i, have_result.tokens[i], tc.want_result.tokens[i], have_result)
+				t.Errorf(" .. test_str: ⟅%s⟆", tc.test_str)
+				t.Errorf(" .. got:      ⟅%s⟆", have_result.tokens[i])
+				t.Errorf(" .. want:     ⟅%s⟆", tc.want_result.tokens[i])
 				return
 			}
 		})
@@ -127,3 +143,50 @@ func Test_reader_tossUntilNeitherOf(t *testing.T) {
 		})
 	}
 }
+
+func Test_reader_takeUntilAnyOf(t *testing.T) {
+	var test_cases = []struct {
+		test_startpos uint
+		test_data     string
+		test_needles  string
+		want_endpos   uint
+		want_ok       bool
+	}{
+		{0, "", "", 0, false},
+		{0, "", "x", 0, false},
+		{0, "", "xy", 0, false},
+		{0, "x", "", 1, true},
+		{0, "x", "x", 0, false},
+		{0, "x", "xy", 0, false},
+		{0, "x", "yx", 0, false},
+		{0, "xa", "x", 0, false},
+		{0, "xa", "xy", 0, false},
+		{0, "xa", "yx", 0, false},
+		{0, "xya", "x", 0, false},
+		{0, "xya", "xy", 0, false},
+		{0, "xya", "yx", 0, false},
+		{0, "ax", "x", 1, true},
+		{0, "ax", "xy", 1, true},
+		{0, "ax", "yx", 1, true},
+		{0, "axy", "x", 1, true},
+		{0, "axy", "xy", 1, true},
+		{0, "axy", "yx", 1, true},
+		{0, "abxa", "x", 2, true},
+		{0, "aboa", "x", 4, true},
+	}
+	for _, tc := range test_cases {
+		t.Run(fmt.Sprintf("%q[%d:]-%q", tc.test_data, tc.test_startpos, tc.test_needles), func(t *testing.T) {
+			test_reader := reader{data: tc.test_data}
+			test_reader.pos = tc.test_startpos
+			_, have_ok := test_reader.takeUntilAnyOf(tc.test_needles)
+			if test_reader.pos != tc.want_endpos {
+				t.Errorf("unexpected position after take: got %d, want %d", test_reader.pos, tc.want_endpos)
+				return
+			}
+			if have_ok != tc.want_ok {
+				t.Errorf("unexpected ok: got %v, want %v", have_ok, tc.want_ok)
+				return
+			}
+		})
+	}
+}