Fix delimiter regex to properly support caret (^)

Fix #2861

src/options_test.go

@@ -65,6 +65,19 @@ func TestDelimiterRegexRegex(t *testing.T) {
 	}
 }
 
+func TestDelimiterRegexRegexCaret(t *testing.T) {
+	delim := delimiterRegexp(`(^\s*|\s+)`)
+	tokens := Tokenize("foo  bar baz", delim)
+	if delim.str != nil ||
+		len(tokens) != 4 ||
+		tokens[0].text.ToString() != "" ||
+		tokens[1].text.ToString() != "foo  " ||
+		tokens[2].text.ToString() != "bar " ||
+		tokens[3].text.ToString() != "baz" {
+		t.Errorf("%s %d", tokens, len(tokens))
+	}
+}
+
 func TestSplitNth(t *testing.T) {
 	{
 		ranges := splitNth("..")

src/tokenizer.go

@@ -156,14 +156,14 @@ func Tokenize(text string, delimiter Delimiter) []Token {
 	// FIXME performance
 	var tokens []string
 	if delimiter.regex != nil {
-		for len(text) > 0 {
+		locs := delimiter.regex.FindAllStringIndex(text, -1)
-			loc := delimiter.regex.FindStringIndex(text)
+		begin := 0
-			if len(loc) < 2 {
+		for _, loc := range locs {
-				loc = []int{0, len(text)}
+			tokens = append(tokens, text[begin:loc[1]])
-			}
+			begin = loc[1]
-			last := util.Max(loc[1], 1)
+		}
-			tokens = append(tokens, text[:last])
+		if begin < len(text) {
-			text = text[last:]
+			tokens = append(tokens, text[begin:])
 		}
 	}
 	return withPrefixLengths(tokens, 0)