Add function to match expressions in tokens

Author: maxmousee

expression.go

@@ -0,0 +1,44 @@
+package stringutils
+
+// Expression represents tokens to be matched in a list of tokens
+// they might have a specific word or not, in case the word is empty, any token of a given Type will be matched
+type Expression struct {
+	Type string // verb, keyword, separator, etc
+	Text string // words to be matched with a token (empty for any)
+}
+
+// FindExpressionsInTokens checks if a given Expression is present in a list of tokens
+func FindExpressionsInTokens(tokens []Token, expressions []Expression) bool {
+	if len(expressions) > len(tokens) {
+		return false
+	}
+	if len(expressions) > 1 {
+		return FindExpressionsInTokens(tokens[1:], expressions[1:])
+	} else if len(expressions) == 1 {
+		if len(tokens) > 1 {
+			return FindExpressionInTokens(tokens[1:], expressions[0])
+		} else {
+			return FindExpressionInTokens(tokens, expressions[0])
+		}
+	}
+	return false
+}
+
+// FindExpressionInTokens checks if a given Expression is present in a list of tokens
+func FindExpressionInTokens(tokens []Token, expression Expression) bool {
+	for _, token := range tokens {
+		if MatchToken(token, expression) {
+			return true
+		}
+	}
+	return false
+}
+
+// MatchToken checks if a given Expression matches a given Token
+func MatchToken(token Token, expression Expression) bool {
+	if len(expression.Text) == 0 {
+		return EqualsIgnoreCase(token.Type, expression.Type)
+	} else {
+		return EqualsIgnoreCase(token.Type, expression.Type) && EqualsIgnoreCase(token.Text, expression.Text)
+	}
+}

expression_test.go

@@ -0,0 +1,120 @@
+package stringutils
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestFindExpressionInTokens(t *testing.T) {
+	assertions := assert.New(t)
+
+	token := Token{
+		Type:     "keyword",
+		Position: 1,
+		Text:     "if",
+	}
+
+	expression := Expression{
+		Type: "keyword",
+		Text: "if",
+	}
+	assertions.True(FindExpressionInTokens([]Token{token}, expression))
+
+	token = Token{
+		Type:     "bracket",
+		Position: 1,
+		Text:     "(",
+	}
+
+	expression = Expression{
+		Type: "keyword",
+		Text: "if",
+	}
+	assertions.False(FindExpressionInTokens([]Token{token}, expression))
+}
+
+func TestFindExpressionsInTokens(t *testing.T) {
+	assertions := assert.New(t)
+
+	token := Token{
+		Type:     "keyword",
+		Position: 1,
+		Text:     "if",
+	}
+
+	expression := Expression{
+		Type: "keyword",
+		Text: "if",
+	}
+	assertions.True(FindExpressionsInTokens([]Token{token}, []Expression{expression}))
+
+	token2 := Token{
+		Type:     "bracket",
+		Position: 2,
+		Text:     "(",
+	}
+
+	expression2 := Expression{
+		Type: "bracket",
+		Text: "(",
+	}
+	assertions.True(FindExpressionsInTokens([]Token{token, token2}, []Expression{expression, expression2}))
+
+	expression2 = Expression{
+		Type: "keyword",
+		Text: "for",
+	}
+	assertions.False(FindExpressionsInTokens([]Token{token, token2}, []Expression{expression, expression2}))
+
+	token = Token{
+		Type:     "bracket",
+		Position: 1,
+		Text:     "(",
+	}
+
+	expression = Expression{
+		Type: "keyword",
+		Text: "if",
+	}
+	assertions.False(FindExpressionsInTokens([]Token{token}, []Expression{expression}))
+
+	token2 = Token{
+		Type:     "bracket",
+		Position: 2,
+		Text:     ")",
+	}
+
+	assertions.False(FindExpressionsInTokens([]Token{token, token2}, []Expression{expression}))
+
+	assertions.False(FindExpressionsInTokens([]Token{token}, []Expression{expression, expression2}))
+
+	assertions.False(FindExpressionsInTokens([]Token{token}, []Expression{}))
+}
+
+func TestMatchToken(t *testing.T) {
+	assertions := assert.New(t)
+
+	token := Token{
+		Type:     "keyword",
+		Position: 1,
+		Text:     "if",
+	}
+
+	expression := Expression{
+		Type: "keyword",
+		Text: "if",
+	}
+	assertions.True(MatchToken(token, expression))
+
+	expression = Expression{
+		Type: "keyword",
+		Text: "IF",
+	}
+	assertions.True(MatchToken(token, expression))
+
+	expression = Expression{
+		Type: "keyword",
+		Text: "",
+	}
+	assertions.True(MatchToken(token, expression))
+}

lexer.go

@@ -27,19 +27,13 @@ func TokenizeWord(word string, position int, tokenTypes []TokenType) Token {
 // If no match is found, it returns a token type with "" as type
 func LookupType(word string, tokenTypes []TokenType) TokenType {
 	for _, aToken := range tokenTypes {
-		if aToken.CaseSensitive {
-			if EqualsAny(word, aToken.Words) {
-				return aToken
-			}
-		} else {
-			if EqualsAnyIgnoreCase(word, aToken.Words) {
-				return aToken
-			}
+		if EqualsAnyIgnoreCase(word, aToken.Words) {
+			return aToken
 		}
 	}
+
 	return TokenType{
-		Type:          "",
-		Words:         []string{""},
-		CaseSensitive: false,
+		Type:  "",
+		Words: []string{""},
 	}
 }

lexer_test.go

@@ -10,9 +10,8 @@ func TestTokenizeCaseSensitive(t *testing.T) {
 
 	var tokenTypes []TokenType
 	tokenTypes = append(tokenTypes, TokenType{
-		Type:          "keyword",
-		Words:         []string{"if", "for"},
-		CaseSensitive: true,
+		Type:  "keyword",
+		Words: []string{"if", "for"},
 	})
 	result := Tokenize("if", tokenTypes)
 	assertions.Equal("if", result[0].Text)
@@ -25,9 +24,8 @@ func TestTokenizeCaseInsensitive(t *testing.T) {
 
 	var tokenTypes []TokenType
 	tokenTypes = append(tokenTypes, TokenType{
-		Type:          "keyword",
-		Words:         []string{"if", "for"},
-		CaseSensitive: false,
+		Type:  "keyword",
+		Words: []string{"if", "for"},
 	})
 	result := Tokenize("IF", tokenTypes)
 	assertions.Equal("IF", result[0].Text)
@@ -40,9 +38,8 @@ func TestTokenizeWord(t *testing.T) {
 
 	var tokenTypes []TokenType
 	tokenTypes = append(tokenTypes, TokenType{
-		Type:          "keyword",
-		Words:         []string{"if", "for"},
-		CaseSensitive: true,
+		Type:  "keyword",
+		Words: []string{"if", "for"},
 	})
 	result := TokenizeWord("if", 0, tokenTypes)
 	assertions.Equal("if", result.Text)
@@ -55,29 +52,25 @@ func TestLookupType(t *testing.T) {
 
 	var tokenTypes []TokenType
 	tokenTypes = append(tokenTypes, TokenType{
-		Type:          "keyword",
-		Words:         []string{"if", "for"},
-		CaseSensitive: false,
+		Type:  "keyword",
+		Words: []string{"if", "for"},
 	})
 
 	result := LookupType("for", tokenTypes)
 	assertions.Equal("keyword", result.Type)
 	assertions.Equal([]string{"if", "for"}, result.Words)
-	assertions.Equal(false, result.CaseSensitive)
 }
 
 func TestLookupTypeNotFound(t *testing.T) {
 	assertions := assert.New(t)
 
 	var tokenTypes []TokenType
 	tokenTypes = append(tokenTypes, TokenType{
-		Type:          "keyword",
-		Words:         []string{"if", "for"},
-		CaseSensitive: false,
+		Type:  "keyword",
+		Words: []string{"if", "for"},
 	})
 
 	result := LookupType("var", tokenTypes)
 	assertions.Equal("", result.Type)
 	assertions.Equal([]string{""}, result.Words)
-	assertions.Equal(false, result.CaseSensitive)
 }

token.go

@@ -7,10 +7,9 @@ type Token struct {
 	Text     string // text
 }
 
-//TokenType represents a category of words, with a collection of tokens and whether or not they should be matched as
+// TokenType represents a category of words, with a collection of tokens and whether or not they should be matched as
 // case sensitive
 type TokenType struct {
-	Type          string   // verb, keyword, separator, etc
-	Words         []string // words to be considered as a given token type
-	CaseSensitive bool     // whether or not this token type is case sensitive
+	Type  string   // verb, keyword, separator, etc
+	Words []string // words to be considered as a given token type
 }

token_test.go

@@ -9,11 +9,9 @@ func TestTokenType(t *testing.T) {
 	assertions := assert.New(t)
 
 	result := TokenType{
-		Type:          "keyword",
-		Words:         []string{"for", "if"},
-		CaseSensitive: true,
+		Type:  "keyword",
+		Words: []string{"for", "if"},
 	}
-	assertions.Equal(true, result.CaseSensitive)
 	assertions.Equal([]string{"for", "if"}, result.Words)
 	assertions.Equal("keyword", result.Type)
 }