Selaa lähdekoodia

Implement #TG6

Alois Mahdal 3 päivää sitten
vanhempi
commit
3c676f9864
2 muutettua tiedostoa jossa 26 lisäystä ja 0 poistoa
  1. 16
    0
      app/tokenize/tokenize.go
  2. 10
    0
      app/tokenize/tokenize_test.go

+ 16
- 0
app/tokenize/tokenize.go Näytä tiedosto

126
 		return "tokenizeResultCodeOk"
126
 		return "tokenizeResultCodeOk"
127
 	case tokenizeResultCodeMissingEndSingleQuote:
127
 	case tokenizeResultCodeMissingEndSingleQuote:
128
 		return "tokenizeResultCodeMissingEndSingleQuote"
128
 		return "tokenizeResultCodeMissingEndSingleQuote"
129
+	case tokenizeResultCodeMissingEndDoubleQuote:
130
+		return "tokenizeResultCodeMissingEndDoubleQuote"
129
 	default:
131
 	default:
130
 		return fmt.Sprintf("unknown!tokenizeResultCode(%d)", self)
132
 		return fmt.Sprintf("unknown!tokenizeResultCode(%d)", self)
131
 	}
133
 	}
134
 const (
136
 const (
135
 	tokenizeResultCodeOk tokenizeResultCode = iota
137
 	tokenizeResultCodeOk tokenizeResultCode = iota
136
 	tokenizeResultCodeMissingEndSingleQuote
138
 	tokenizeResultCodeMissingEndSingleQuote
139
+	tokenizeResultCodeMissingEndDoubleQuote
137
 )
140
 )
138
 
141
 
139
 func tokenize(str string) tokenizeResult {
142
 func tokenize(str string) tokenizeResult {
162
 			}
165
 			}
163
 			b.bufAppend(new_chars)
166
 			b.bufAppend(new_chars)
164
 			rdr.tossChar() // the second `'`
167
 			rdr.tossChar() // the second `'`
168
+		case '"':
169
+			rdr.tossChar() // first `"`
170
+			new_chars, found := rdr.takeUntil("\"")
171
+			if !found {
172
+				return tokenizeResult{code: tokenizeResultCodeMissingEndDoubleQuote, err_loc: rdr.pos - 1}
173
+			}
174
+			b.bufAppend(new_chars)
175
+			rdr.tossChar() // the second `"`
165
 		default:
176
 		default:
166
 			b.bufAppendChar(this_byte)
177
 			b.bufAppendChar(this_byte)
167
 			rdr.tossChar()
178
 			rdr.tossChar()
178
 		return res.tokens, nil
189
 		return res.tokens, nil
179
 	case tokenizeResultCodeMissingEndSingleQuote:
190
 	case tokenizeResultCodeMissingEndSingleQuote:
180
 		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndSingleQuote, loc: res.err_loc}
191
 		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndSingleQuote, loc: res.err_loc}
192
+	case tokenizeResultCodeMissingEndDoubleQuote:
193
+		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndDoubleQuote, loc: res.err_loc}
181
 	default:
194
 	default:
182
 		return nil, TokenizeError{code: TokenizeErrorCodeGeneral, loc: res.err_loc}
195
 		return nil, TokenizeError{code: TokenizeErrorCodeGeneral, loc: res.err_loc}
183
 	}
196
 	}
192
 const (
205
 const (
193
 	TokenizeErrorCodeGeneral TokenizeErrorCode = iota
206
 	TokenizeErrorCodeGeneral TokenizeErrorCode = iota
194
 	TokenizeErrorCodeMissingEndSingleQuote
207
 	TokenizeErrorCodeMissingEndSingleQuote
208
+	TokenizeErrorCodeMissingEndDoubleQuote
195
 )
209
 )
196
 
210
 
197
 func (e TokenizeError) Error() string {
211
 func (e TokenizeError) Error() string {
198
 	switch e.code {
212
 	switch e.code {
199
 	case TokenizeErrorCodeMissingEndSingleQuote:
213
 	case TokenizeErrorCodeMissingEndSingleQuote:
200
 		return fmt.Sprintf("unterminated single-quote: at %d", e.loc)
214
 		return fmt.Sprintf("unterminated single-quote: at %d", e.loc)
215
+	case TokenizeErrorCodeMissingEndDoubleQuote:
216
+		return fmt.Sprintf("unterminated double-quote: at %d", e.loc)
201
 	default:
217
 	default:
202
 		return fmt.Sprintf("unknown TokenizeError code: .code=%d .loc=%d", e.code, e.loc)
218
 		return fmt.Sprintf("unknown TokenizeError code: .code=%d .loc=%d", e.code, e.loc)
203
 	}
219
 	}

+ 10
- 0
app/tokenize/tokenize_test.go Näytä tiedosto

33
 		{"foo ' space bar '", tokenizeResult{tokens: []string{"foo", " space bar "}}},
33
 		{"foo ' space bar '", tokenizeResult{tokens: []string{"foo", " space bar "}}},
34
 		{"foo 'John \"Spaceman\" Doe'", tokenizeResult{tokens: []string{"foo", "John \"Spaceman\" Doe"}}},
34
 		{"foo 'John \"Spaceman\" Doe'", tokenizeResult{tokens: []string{"foo", "John \"Spaceman\" Doe"}}},
35
 
35
 
36
+		// double quotes
37
+		{"\"", tokenizeResult{tokens: []string{}, code: tokenizeResultCodeMissingEndDoubleQuote}},
38
+		{"\"\"", tokenizeResult{tokens: []string{""}}},
39
+		{"fo\"\"o", tokenizeResult{tokens: []string{"foo"}}},
40
+		{"foo \"\" bar", tokenizeResult{tokens: []string{"foo", "", "bar"}}},
41
+		{"foo \"and\" bar", tokenizeResult{tokens: []string{"foo", "and", "bar"}}},
42
+		{"foo \"\\\t\n\" bar", tokenizeResult{tokens: []string{"foo", "\\\t\n", "bar"}}},
43
+		{"foo \" space bar \"", tokenizeResult{tokens: []string{"foo", " space bar "}}},
44
+		{"foo \"Joe's lunch\"", tokenizeResult{tokens: []string{"foo", "Joe's lunch"}}},
45
+
36
 		//
46
 		//
37
 	}
47
 	}
38
 	for _, tc := range test_cases {
48
 	for _, tc := range test_cases {