Implement #TG6

app/tokenize/tokenize.go

@@ -126,6 +126,8 @@ func (self tokenizeResultCode) String() string {
 		return "tokenizeResultCodeOk"
 	case tokenizeResultCodeMissingEndSingleQuote:
 		return "tokenizeResultCodeMissingEndSingleQuote"
+	case tokenizeResultCodeMissingEndDoubleQuote:
+		return "tokenizeResultCodeMissingEndDoubleQuote"
 	default:
 		return fmt.Sprintf("unknown!tokenizeResultCode(%d)", self)
 	}
@@ -134,6 +136,7 @@ func (self tokenizeResultCode) String() string {
 const (
 	tokenizeResultCodeOk tokenizeResultCode = iota
 	tokenizeResultCodeMissingEndSingleQuote
+	tokenizeResultCodeMissingEndDoubleQuote
 )
 
 func tokenize(str string) tokenizeResult {
@@ -162,6 +165,14 @@ func tokenize(str string) tokenizeResult {
 			}
 			b.bufAppend(new_chars)
 			rdr.tossChar() // the second `'`
+		case '"':
+			rdr.tossChar() // first `"`
+			new_chars, found := rdr.takeUntil("\"")
+			if !found {
+				return tokenizeResult{code: tokenizeResultCodeMissingEndDoubleQuote, err_loc: rdr.pos - 1}
+			}
+			b.bufAppend(new_chars)
+			rdr.tossChar() // the second `"`
 		default:
 			b.bufAppendChar(this_byte)
 			rdr.tossChar()
@@ -178,6 +189,8 @@ func Tokenize(str string) ([]string, error) {
 		return res.tokens, nil
 	case tokenizeResultCodeMissingEndSingleQuote:
 		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndSingleQuote, loc: res.err_loc}
+	case tokenizeResultCodeMissingEndDoubleQuote:
+		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndDoubleQuote, loc: res.err_loc}
 	default:
 		return nil, TokenizeError{code: TokenizeErrorCodeGeneral, loc: res.err_loc}
 	}
@@ -192,12 +205,15 @@ type TokenizeErrorCode uint8
 const (
 	TokenizeErrorCodeGeneral TokenizeErrorCode = iota
 	TokenizeErrorCodeMissingEndSingleQuote
+	TokenizeErrorCodeMissingEndDoubleQuote
 )
 
 func (e TokenizeError) Error() string {
 	switch e.code {
 	case TokenizeErrorCodeMissingEndSingleQuote:
 		return fmt.Sprintf("unterminated single-quote: at %d", e.loc)
+	case TokenizeErrorCodeMissingEndDoubleQuote:
+		return fmt.Sprintf("unterminated double-quote: at %d", e.loc)
 	default:
 		return fmt.Sprintf("unknown TokenizeError code: .code=%d .loc=%d", e.code, e.loc)
 	}

app/tokenize/tokenize_test.go

@@ -33,6 +33,16 @@ func Test_tokenize(t *testing.T) {
 		{"foo ' space bar '", tokenizeResult{tokens: []string{"foo", " space bar "}}},
 		{"foo 'John \"Spaceman\" Doe'", tokenizeResult{tokens: []string{"foo", "John \"Spaceman\" Doe"}}},
 
+		// double quotes
+		{"\"", tokenizeResult{tokens: []string{}, code: tokenizeResultCodeMissingEndDoubleQuote}},
+		{"\"\"", tokenizeResult{tokens: []string{""}}},
+		{"fo\"\"o", tokenizeResult{tokens: []string{"foo"}}},
+		{"foo \"\" bar", tokenizeResult{tokens: []string{"foo", "", "bar"}}},
+		{"foo \"and\" bar", tokenizeResult{tokens: []string{"foo", "and", "bar"}}},
+		{"foo \"\\\t\n\" bar", tokenizeResult{tokens: []string{"foo", "\\\t\n", "bar"}}},
+		{"foo \" space bar \"", tokenizeResult{tokens: []string{"foo", " space bar "}}},
+		{"foo \"Joe's lunch\"", tokenizeResult{tokens: []string{"foo", "Joe's lunch"}}},
+
 		//
 	}
 	for _, tc := range test_cases {