fixed parsing of quotes and of detecting rename statements

go/sql/parser.go

@@ -8,10 +8,12 @@ package sql
 import (
 	"regexp"
 	"strconv"
+	"strings"
 )
 
 var (
-	renameColumnRegexp = regexp.MustCompile(`(?i)change\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
+	stripQuotesRegexp  = regexp.MustCompile("('[^']*')")
+	renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
 )
 
 type Parser struct {
@@ -24,17 +26,54 @@ func NewParser() *Parser {
 	}
 }
 
-func (this *Parser) ParseAlterStatement(alterStatement string) (err error) {
+func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) {
-	allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterStatement, -1)
+	terminatingQuote := rune(0)
-	for _, submatch := range allStringSubmatch {
+	f := func(c rune) bool {
-		if unquoted, err := strconv.Unquote(submatch[2]); err == nil {
+		switch {
-			submatch[2] = unquoted
+		case c == terminatingQuote:
-		}
+			terminatingQuote = rune(0)
-		if unquoted, err := strconv.Unquote(submatch[3]); err == nil {
+			return false
-			submatch[3] = unquoted
+		case terminatingQuote != rune(0):
+			return false
+		case c == '\'':
+			terminatingQuote = c
+			return false
+		case c == '(':
+			terminatingQuote = ')'
+			return false
+		default:
+			return c == ','
 		}
+	}
 
-		this.columnRenameMap[submatch[2]] = submatch[3]
+	tokens = strings.FieldsFunc(alterStatement, f)
+	for i := range tokens {
+		tokens[i] = strings.TrimSpace(tokens[i])
+	}
+	return tokens, nil
+}
+
+func (this *Parser) stripQuotesFromAlterStatement(alterStatement string) (strippedStatement string) {
+	strippedStatement = alterStatement
+	strippedStatement = stripQuotesRegexp.ReplaceAllString(strippedStatement, "''")
+	return strippedStatement
+}
+
+func (this *Parser) ParseAlterStatement(alterStatement string) (err error) {
+	alterTokens, _ := this.tokenizeAlterStatement(alterStatement)
+	for _, alterToken := range alterTokens {
+		alterToken = this.stripQuotesFromAlterStatement(alterToken)
+		allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1)
+		for _, submatch := range allStringSubmatch {
+			if unquoted, err := strconv.Unquote(submatch[2]); err == nil {
+				submatch[2] = unquoted
+			}
+			if unquoted, err := strconv.Unquote(submatch[3]); err == nil {
+				submatch[3] = unquoted
+			}
+
+			this.columnRenameMap[submatch[2]] = submatch[3]
+		}
 	}
 	return nil
 }

go/sql/parser_test.go

@@ -6,6 +6,7 @@
 package sql
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/outbrain/golib/log"
@@ -66,3 +67,53 @@ func TestParseAlterStatementNonTrivial(t *testing.T) {
 		test.S(t).ExpectEquals(renames["f"], "fl")
 	}
 }
+
+func TestTokenizeAlterStatement(t *testing.T) {
+	parser := NewParser()
+	{
+		alterStatement := "add column t int"
+		tokens, _ := parser.tokenizeAlterStatement(alterStatement)
+		test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int"}))
+	}
+	{
+		alterStatement := "add column t int, change column i int"
+		tokens, _ := parser.tokenizeAlterStatement(alterStatement)
+		test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "change column i int"}))
+	}
+	{
+		alterStatement := "add column t int, change column i int 'some comment'"
+		tokens, _ := parser.tokenizeAlterStatement(alterStatement)
+		test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "change column i int 'some comment'"}))
+	}
+	{
+		alterStatement := "add column t int, change column i int 'some comment, with comma'"
+		tokens, _ := parser.tokenizeAlterStatement(alterStatement)
+		test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "change column i int 'some comment, with comma'"}))
+	}
+	{
+		alterStatement := "add column t int, add column e enum('a','b','c')"
+		tokens, _ := parser.tokenizeAlterStatement(alterStatement)
+		log.Errorf("%#v", tokens)
+		test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "add column e enum('a','b','c')"}))
+	}
+	{
+		alterStatement := "add column t int(11), add column e enum('a','b','c')"
+		tokens, _ := parser.tokenizeAlterStatement(alterStatement)
+		log.Errorf("%#v", tokens)
+		test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int(11)", "add column e enum('a','b','c')"}))
+	}
+}
+
+func TestStripQuotesFromAlterStatement(t *testing.T) {
+	parser := NewParser()
+	{
+		alterStatement := "add column e enum('a','b','c')"
+		strippedStatement := parser.stripQuotesFromAlterStatement(alterStatement)
+		test.S(t).ExpectEquals(strippedStatement, "add column e enum('','','')")
+	}
+	{
+		alterStatement := "change column i int 'some comment, with comma'"
+		strippedStatement := parser.stripQuotesFromAlterStatement(alterStatement)
+		test.S(t).ExpectEquals(strippedStatement, "change column i int ''")
+	}
+}