Add preparser to cleanup the sql file before parser ingestion

Author: kefniark

internal/parser.go

@@ -55,6 +55,7 @@ func normalizeSql(sql string) string {
 }
 
 func ParseSchema(sql string) (*core.SQLSchema, error) {
+	sql = normalize(sql)
 	stmts, err := parser.Parse(sql)
 	if err != nil {
 		return nil, fmt.Errorf("schema parsing error: %w", err)

internal/preparser.go

@@ -0,0 +1,56 @@
+package internal
+
+import (
+	"regexp"
+	"slices"
+	"strings"
+)
+
+// The parser used has some limitation (based on cockroachDB syntax),
+// The preparser is there to normalize the input and avoid a set of known errors
+
+func normalize(sql string) string {
+	alter := []func(string) string{removeTrigger, removeCustomTypes, replaceSubType}
+	for _, f := range alter {
+		sql = f(sql)
+	}
+	return sql
+}
+
+var triggerRegexp = regexp.MustCompile(`CREATE TRIGGER[\s\S]*?END\s*?;`)
+
+func removeTrigger(sql string) string {
+	res := triggerRegexp.FindAllStringSubmatchIndex(sql, -1)
+	if len(res) == 0 {
+		return sql
+	}
+
+	slices.Reverse(res)
+
+	for _, ind := range res {
+		sql = sql[:ind[0]] + sql[ind[1]:]
+	}
+
+	return sql
+}
+
+var customTypeRegexp = regexp.MustCompile(`CREATE TYPE[\s\S]*?;`)
+
+func removeCustomTypes(sql string) string {
+	res := customTypeRegexp.FindAllStringSubmatchIndex(sql, -1)
+	if len(res) == 0 {
+		return sql
+	}
+
+	slices.Reverse(res)
+
+	for _, ind := range res {
+		sql = sql[:ind[0]] + sql[ind[1]:]
+	}
+
+	return sql
+}
+
+func replaceSubType(sql string) string {
+	return strings.ReplaceAll(sql, "BLOB SUB_TYPE TEXT", "bytea")
+}

internal/preparser_test.go

@@ -0,0 +1,66 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTrigger(t *testing.T) {
+	schema, err := ParseSchema(`
+		CREATE TABLE actor (
+  			actor_id numeric NOT NULL ,
+			first_name VARCHAR(45) NOT NULL,
+			last_name VARCHAR(45) NOT NULL,
+			last_update TIMESTAMP NOT NULL,
+			PRIMARY KEY  (actor_id)
+  		);
+
+		CREATE  INDEX idx_actor_last_name ON actor(last_name);
+ 
+		CREATE TRIGGER actor_trigger_ai AFTER INSERT ON actor
+		BEGIN
+			UPDATE actor SET last_update = DATETIME('NOW')  WHERE rowid = new.rowid;
+		END;
+	`)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 4, len(schema.Tables["actor"].Columns))
+}
+
+func TestEnums(t *testing.T) {
+	schema, err := ParseSchema(`
+		CREATE TABLE actor (
+  			actor_id numeric NOT NULL ,
+			first_name VARCHAR(45) NOT NULL,
+			last_name VARCHAR(45) NOT NULL,
+			last_update TIMESTAMP NOT NULL,
+			PRIMARY KEY  (actor_id)
+  		);
+
+		CREATE  INDEX idx_actor_last_name ON actor(last_name);
+ 
+		CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');
+	`)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 4, len(schema.Tables["actor"].Columns))
+}
+
+func TestBlob(t *testing.T) {
+	schema, err := ParseSchema(`
+		CREATE TABLE actor (
+  			actor_id numeric NOT NULL ,
+			description BLOB SUB_TYPE TEXT DEFAULT NULL,
+			first_name VARCHAR(45) NOT NULL,
+			last_name VARCHAR(45) NOT NULL,
+			last_update TIMESTAMP NOT NULL,
+			PRIMARY KEY  (actor_id)
+  		);
+
+		CREATE  INDEX idx_actor_last_name ON actor(last_name);
+	`)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 5, len(schema.Tables["actor"].Columns))
+}