Add `ok` check to takeChar()

app/tokenize/tokenize.go

@@ -26,10 +26,13 @@ func (self *reader) peekChar() (byte, bool) {
 	return self.data[self.pos], true
 }
 
-func (self *reader) takeChar() string {
+func (self *reader) takeChar() (string, bool) {
 	self.checkPos()
+	if self.pos == self.len() {
+		return "", false
+	}
 	self.pos += 1
-	return self.data[self.pos-1 : self.pos]
+	return self.data[self.pos-1 : self.pos], true
 }
 
 func (self *reader) takeUntil(needle string) (string, bool) {
@@ -185,7 +188,7 @@ func tokenize(str string) tokenizeResult {
 			if rdr.done() {
 				return tokenizeResult{code: tokenizeResultCodeMissingEscapedCharacter, err_loc: rdr.pos - 1}
 			}
-			new_chars := rdr.takeChar()
+			new_chars, _ := rdr.takeChar()
 			b.bufAppend(new_chars)
 		default:
 			b.bufAppendChar(this_char)