Implement #NI6

app/runnable/runnable.go

@@ -60,7 +60,11 @@ func (e ParseError) Error() string {
 }
 
 func Parse(ctx *core.Context, command_line string) (Runnable, error) {
-	tokens := tokenize.Tokenize(command_line)
+	tokens, err := tokenize.Tokenize(command_line)
+	if err != nil {
+		return nil, err
+	}
+	//fmt.Printf("Parse():tokens=%#v\n", tokens)
 	if len(tokens) == 0 { // ie. empty or whitespace-only input
 		return nil, ParseError{
 			code: ParseErrorCodeNothing,

app/tokenize/tokenize.go

@@ -1,8 +1,204 @@
 package tokenize
 
-import "strings"
+import (
+	"fmt"
+	"strings"
+)
 
-func Tokenize(str string) []string {
-	tokens := strings.Fields(str)
-	return tokens
+type reader struct {
+	data string
+	pos  uint
+}
+
+func (self *reader) len() uint {
+	return uint(len(self.data))
+}
+
+func (self *reader) checkPos() {
+	if self.pos > uint(len(self.data)) {
+		panic(fmt.Sprintf("invalid cursor state: pos=%d but len(data)=%d", self.pos, len(self.data)))
+	}
+}
+
+func (self *reader) peekByte() (byte, bool) {
+	self.checkPos()
+	if self.done() {
+		return 0, false
+	}
+	return self.data[self.pos], true
+}
+
+func (self *reader) takeUntil(needle string) (string, bool) {
+	self.checkPos()
+	idx := strings.Index(self.data[self.pos:], needle)
+	if idx == -1 {
+		return "", false
+	}
+	end := self.pos + uint(idx)
+	out := self.data[self.pos:end]
+	self.pos = end
+	return out, true
+}
+
+func (self *reader) tossUntilNeitherOf(needles string) bool {
+	self.checkPos()
+	match := func(b byte, ns string) bool {
+		for i := range len(ns) {
+			if b == ns[i] {
+				return true
+			}
+		}
+		return false
+	}
+	tossed := uint(0)
+	for {
+		idx := self.pos + tossed
+		if idx == self.len() {
+			break
+		}
+		if match(self.data[idx], needles) {
+			tossed += 1
+			continue
+		}
+		break
+	}
+	self.pos += tossed
+	return tossed > 0
+}
+
+func (self *reader) tossChar() {
+	self.checkPos()
+	self.pos += 1
+}
+
+func (self *reader) done() bool {
+	self.checkPos()
+	return self.pos == self.len()
+}
+
+type builder struct {
+	tokens    []string
+	buf_set   bool
+	buf_chars string
+}
+
+func make_builder() builder {
+	return builder{tokens: make([]string, 0), buf_set: false, buf_chars: ""}
+}
+
+func (self *builder) bufAppend(chars string) {
+	if self.buf_set {
+		self.buf_chars += chars
+	} else {
+		self.buf_chars = chars
+		self.buf_set = true
+	}
+}
+
+func (self *builder) bufAppendChar(char byte) {
+	if !self.buf_set {
+		self.buf_chars = ""
+		self.buf_set = true
+	}
+	self.buf_chars += string(char)
+}
+
+func (self *builder) bufCommit() {
+	if !self.buf_set {
+		return
+	}
+	self.tokens = append(self.tokens, self.buf_chars)
+	self.buf_chars = ""
+	self.buf_set = false
+}
+
+type tokenizeResult struct {
+	tokens  []string
+	code    tokenizeResultCode
+	err_loc uint
+}
+
+type tokenizeResultCode uint8
+
+func (self tokenizeResultCode) String() string {
+	switch self {
+	case tokenizeResultCodeOk:
+		return "tokenizeResultCodeOk"
+	case tokenizeResultCodeMissingEndSingleQuote:
+		return "tokenizeResultCodeMissingEndSingleQuote"
+	default:
+		return fmt.Sprintf("unknown!tokenizeResultCode(%d)", self)
+	}
+}
+
+const (
+	tokenizeResultCodeOk tokenizeResultCode = iota
+	tokenizeResultCodeMissingEndSingleQuote
+)
+
+func tokenize(str string) tokenizeResult {
+	rdr := reader{data: str}
+	b := make_builder()
+	for {
+		this_byte, ok := rdr.peekByte()
+		if !ok {
+			break
+		}
+		switch this_byte {
+		case ' ':
+			b.bufCommit()
+			rdr.tossUntilNeitherOf(" \t\n")
+		case '\t':
+			b.bufCommit()
+			rdr.tossUntilNeitherOf(" \t\n")
+		case '\n':
+			b.bufCommit()
+			rdr.tossUntilNeitherOf(" \t\n")
+		case '\'':
+			rdr.tossChar() // first `'`
+			new_chars, found := rdr.takeUntil("'")
+			if !found {
+				return tokenizeResult{code: tokenizeResultCodeMissingEndSingleQuote, err_loc: rdr.pos - 1}
+			}
+			b.bufAppend(new_chars)
+			rdr.tossChar() // the second `'`
+		default:
+			b.bufAppendChar(this_byte)
+			rdr.tossChar()
+		}
+	}
+	b.bufCommit()
+	return tokenizeResult{tokens: b.tokens}
+}
+
+func Tokenize(str string) ([]string, error) {
+	res := tokenize(str)
+	switch res.code {
+	case tokenizeResultCodeOk:
+		return res.tokens, nil
+	case tokenizeResultCodeMissingEndSingleQuote:
+		return nil, TokenizeError{code: TokenizeErrorCodeMissingEndSingleQuote, loc: res.err_loc}
+	default:
+		return nil, TokenizeError{code: TokenizeErrorCodeGeneral, loc: res.err_loc}
+	}
+}
+
+type TokenizeError struct {
+	code TokenizeErrorCode
+	loc  uint
+}
+type TokenizeErrorCode uint8
+
+const (
+	TokenizeErrorCodeGeneral TokenizeErrorCode = iota
+	TokenizeErrorCodeMissingEndSingleQuote
+)
+
+func (e TokenizeError) Error() string {
+	switch e.code {
+	case TokenizeErrorCodeMissingEndSingleQuote:
+		return fmt.Sprintf("unterminated single-quote: at %d", e.loc)
+	default:
+		return fmt.Sprintf("unknown TokenizeError code: .code=%d .loc=%d", e.code, e.loc)
+	}
 }

app/tokenize/tokenize_test.go

@@ -0,0 +1,104 @@
+package tokenize
+
+import "fmt"
+import "testing"
+
+func Test_tokenize(t *testing.T) {
+	var test_cases = []struct {
+		test_str    string
+		want_result tokenizeResult
+	}{
+
+		// emptiness
+		{"", tokenizeResult{tokens: []string{}}},
+
+		// unquoted whitespace
+		{" ", tokenizeResult{tokens: []string{}}},
+		{"\t", tokenizeResult{tokens: []string{}}},
+		{"\n", tokenizeResult{tokens: []string{}}},
+		{" \t\n", tokenizeResult{tokens: []string{}}},
+		{" \tfoo", tokenizeResult{tokens: []string{"foo"}}},
+		{"foo ", tokenizeResult{tokens: []string{"foo"}}},
+		{"foo bar", tokenizeResult{tokens: []string{"foo", "bar"}}},
+		{"foo\nbar", tokenizeResult{tokens: []string{"foo", "bar"}}},
+		{"foo\tbar", tokenizeResult{tokens: []string{"foo", "bar"}}},
+
+		// single quotes
+		{"'", tokenizeResult{tokens: []string{}, code: tokenizeResultCodeMissingEndSingleQuote}},
+		{"''", tokenizeResult{tokens: []string{""}}},
+		{"fo''o", tokenizeResult{tokens: []string{"foo"}}},
+		{"foo '' bar", tokenizeResult{tokens: []string{"foo", "", "bar"}}},
+		{"foo 'and' bar", tokenizeResult{tokens: []string{"foo", "and", "bar"}}},
+		{"foo '\\\t\n' bar", tokenizeResult{tokens: []string{"foo", "\\\t\n", "bar"}}},
+		{"foo ' space bar '", tokenizeResult{tokens: []string{"foo", " space bar "}}},
+		{"foo 'John \"Spaceman\" Doe'", tokenizeResult{tokens: []string{"foo", "John \"Spaceman\" Doe"}}},
+
+		//
+	}
+	for _, tc := range test_cases {
+		t.Run(fmt.Sprintf("%q", tc.test_str), func(t *testing.T) {
+			have_result := tokenize(tc.test_str)
+			if have_result.code != tc.want_result.code {
+				t.Errorf("unexpected result .code: got %s, want %s in %v", have_result.code, tc.want_result.code, have_result)
+				return
+			}
+			if have_result.err_loc != tc.want_result.err_loc {
+				t.Errorf("unexpected result .err_loc: got %d, want %d in %v", have_result.err_loc, tc.want_result.err_loc, have_result)
+				return
+			}
+			if len(have_result.tokens) != len(tc.want_result.tokens) {
+				t.Errorf("unexpected number of result .tokens: got %d, want %d in %v", len(have_result.tokens), len(tc.want_result.tokens), have_result)
+				return
+			}
+			for i := range have_result.tokens {
+				if have_result.tokens[i] == tc.want_result.tokens[i] {
+					continue
+				}
+				t.Errorf("unexpected token in result .tokens[%d]: got %q, want %q in %v", i, have_result.tokens[i], tc.want_result.tokens[i], have_result)
+				return
+			}
+		})
+	}
+}
+
+func Test_reader_tossUntilNeitherOf(t *testing.T) {
+	var test_cases = []struct {
+		test_data     string
+		test_needles  string
+		test_startpos uint
+		want_endpos   uint
+		want_ok       bool
+	}{
+		{"", "", 0, 0, false},
+		{"", "x", 0, 0, false},
+		{"", "xy", 0, 0, false},
+		{"x", "", 0, 0, false},
+		{"x", "x", 0, 1, true},
+		{"x", "xy", 0, 1, true},
+		{"x", "yx", 0, 1, true},
+		{"xa", "x", 0, 1, true},
+		{"xa", "xy", 0, 1, true},
+		{"xa", "yx", 0, 1, true},
+		{"xya", "x", 0, 1, true},
+		{"xya", "xy", 0, 2, true},
+		{"xya", "yx", 0, 2, true},
+		{"xxya", "x", 1, 2, true},
+		{"xxya", "xy", 1, 3, true},
+		{"xxya", "yx", 1, 3, true},
+	}
+	for _, tc := range test_cases {
+		t.Run(fmt.Sprintf("%q[%d:]-%q", tc.test_data, tc.test_startpos, tc.test_needles), func(t *testing.T) {
+			test_reader := reader{data: tc.test_data}
+			test_reader.pos = tc.test_startpos
+			have_ok := test_reader.tossUntilNeitherOf(tc.test_needles)
+			if test_reader.pos != tc.want_endpos {
+				t.Errorf("unexpected position after toss: got %d, want %d", test_reader.pos, tc.want_endpos)
+				return
+			}
+			if have_ok != tc.want_ok {
+				t.Errorf("unexpected ok: got %v, want %v", have_ok, tc.want_ok)
+				return
+			}
+		})
+	}
+}