Implement #YT5

app/tokenize/tokenize.go

@@ -26,6 +26,12 @@ func (self *reader) peekByte() (byte, bool) {
 	return self.data[self.pos], true
 }
 
+func (self *reader) takeChar() string {
+	self.checkPos()
+	self.pos += 1
+	return self.data[self.pos-1 : self.pos]
+}
+
 func (self *reader) takeUntil(needle string) (string, bool) {
 	self.checkPos()
 	idx := strings.Index(self.data[self.pos:], needle)
@@ -126,6 +132,8 @@ func (self tokenizeResultCode) String() string {
 		return "tokenizeResultCodeMissingEndSingleQuote"
 	case tokenizeResultCodeMissingEndDoubleQuote:
 		return "tokenizeResultCodeMissingEndDoubleQuote"
+	case tokenizeResultCodeMissingEscapedCharacter:
+		return "tokenizeResultCodeMissingEscapedCharacter"
 	default:
 		return fmt.Sprintf("unknown!tokenizeResultCode(%d)", self)
 	}
@@ -135,6 +143,7 @@ const (
 	tokenizeResultCodeOk tokenizeResultCode = iota
 	tokenizeResultCodeMissingEndSingleQuote
 	tokenizeResultCodeMissingEndDoubleQuote
+	tokenizeResultCodeMissingEscapedCharacter
 )
 
 func tokenize(str string) tokenizeResult {
@@ -171,6 +180,13 @@ func tokenize(str string) tokenizeResult {
 			}
 			b.bufAppend(new_chars)
 			rdr.tossChar() // the second `"`
+		case '\\':
+			rdr.tossChar()
+			if rdr.done() {
+				return tokenizeResult{code: tokenizeResultCodeMissingEscapedCharacter, err_loc: rdr.pos - 1}
+			}
+			new_chars := rdr.takeChar()
+			b.bufAppend(new_chars)
 		default:
 			b.bufAppendChar(this_byte)
 			rdr.tossChar()
@@ -189,6 +205,8 @@ func Tokenize(str string) ([]string, error) {
 		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndSingleQuote, loc: res.err_loc}
 	case tokenizeResultCodeMissingEndDoubleQuote:
 		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndDoubleQuote, loc: res.err_loc}
+	case tokenizeResultCodeMissingEscapedCharacter:
+		return nil, TokenizeError{code: TokenizeErrorCodeMissingEscapedCharacter, loc: res.err_loc}
 	default:
 		return nil, TokenizeError{code: TokenizeErrorCodeGeneral, loc: res.err_loc}
 	}
@@ -204,6 +222,7 @@ const (
 	TokenizeErrorCodeGeneral TokenizeErrorCode = iota
 	TokenizeErrorCodeMissingEndSingleQuote
 	TokenizeErrorCodeMissingEndDoubleQuote
+	TokenizeErrorCodeMissingEscapedCharacter
 )
 
 func (e TokenizeError) Error() string {
@@ -212,6 +231,8 @@ func (e TokenizeError) Error() string {
 		return fmt.Sprintf("unterminated single-quote: at %d", e.loc)
 	case TokenizeErrorCodeMissingEndDoubleQuote:
 		return fmt.Sprintf("unterminated double-quote: at %d", e.loc)
+	case TokenizeErrorCodeMissingEscapedCharacter:
+		return fmt.Sprintf("missing escaped character: at %d", e.loc)
 	default:
 		return fmt.Sprintf("unknown TokenizeError code: .code=%d .loc=%d", e.code, e.loc)
 	}

app/tokenize/tokenize_test.go

@@ -43,6 +43,21 @@ func Test_tokenize(t *testing.T) {
 		{"foo \" space bar \"", tokenizeResult{tokens: []string{"foo", " space bar "}}},
 		{"foo \"Joe's lunch\"", tokenizeResult{tokens: []string{"foo", "Joe's lunch"}}},
 
+		// lone backslash
+		{"\\", tokenizeResult{tokens: []string{}, code: tokenizeResultCodeMissingEscapedCharacter}},
+		{"\\\\", tokenizeResult{tokens: []string{"\\"}}},
+		{"\\a", tokenizeResult{tokens: []string{"a"}}},
+		{"\\?", tokenizeResult{tokens: []string{"?"}}},
+		{"\\$", tokenizeResult{tokens: []string{"$"}}},
+		{"\\ ", tokenizeResult{tokens: []string{" "}}},
+		{"\\\t", tokenizeResult{tokens: []string{"\t"}}},
+		{"\\\n", tokenizeResult{tokens: []string{"\n"}}},
+		{"fo\\o", tokenizeResult{tokens: []string{"foo"}}},
+		{"fo\\\\o", tokenizeResult{tokens: []string{"fo\\o"}}},
+		{"foo \\ bar", tokenizeResult{tokens: []string{"foo", " bar"}}},
+		{"foo \\\\ bar", tokenizeResult{tokens: []string{"foo", "\\", "bar"}}},
+		{"foo \\'bar\\' baz", tokenizeResult{tokens: []string{"foo", "'bar'", "baz"}}},
+
 		//
 	}
 	for _, tc := range test_cases {