|
|
@@ -18,7 +18,7 @@ func (self *reader) checkPos() {
|
|
18
|
18
|
}
|
|
19
|
19
|
}
|
|
20
|
20
|
|
|
21
|
|
-func (self *reader) peekByte() (byte, bool) {
|
|
|
21
|
+func (self *reader) peekChar() (byte, bool) {
|
|
22
|
22
|
self.checkPos()
|
|
23
|
23
|
if self.done() {
|
|
24
|
24
|
return 0, false
|
|
|
@@ -150,11 +150,11 @@ func tokenize(str string) tokenizeResult {
|
|
150
|
150
|
rdr := reader{data: str}
|
|
151
|
151
|
b := make_builder()
|
|
152
|
152
|
for {
|
|
153
|
|
- this_byte, ok := rdr.peekByte()
|
|
|
153
|
+ this_char, ok := rdr.peekChar()
|
|
154
|
154
|
if !ok {
|
|
155
|
155
|
break
|
|
156
|
156
|
}
|
|
157
|
|
- switch this_byte {
|
|
|
157
|
+ switch this_char {
|
|
158
|
158
|
case ' ':
|
|
159
|
159
|
b.bufCommit()
|
|
160
|
160
|
rdr.tossUntilNeitherOf(" \t\n")
|
|
|
@@ -188,7 +188,7 @@ func tokenize(str string) tokenizeResult {
|
|
188
|
188
|
new_chars := rdr.takeChar()
|
|
189
|
189
|
b.bufAppend(new_chars)
|
|
190
|
190
|
default:
|
|
191
|
|
- b.bufAppendChar(this_byte)
|
|
|
191
|
+ b.bufAppendChar(this_char)
|
|
192
|
192
|
rdr.tossChar()
|
|
193
|
193
|
}
|
|
194
|
194
|
}
|