Create token types

Author: maxmousee

lexer.go

@@ -0,0 +1,2 @@
+package stringutils
+

stringutils.go

@@ -179,7 +179,7 @@ func IsAlphabet(r rune) bool {
 }
 
 // WordSplit splits a string into words. Returns a slice of words.
-// If there is no word in a string, return nil.
+// If there is no word in a string, returns an empty string.
 //
 // Word is defined as a locale dependent string containing alphabetic characters,
 // which may also contain but not start with `'` and `-` characters.

token.go

@@ -0,0 +1,13 @@
+package stringutils
+
+type Token struct {
+	Type   				string          // verb, keyword, separator, etc
+	Position	    	int             // position of the word in string
+	Text				string			// text
+}
+
+type TokenType struct {
+	Type			string 				// verb, keyword, separator, etc
+	Words 			[]string			// words to be considered as a given token type
+	CaseSensitive	bool				// whether or not this token type is case sensitive
+}

token_test.go

@@ -0,0 +1,32 @@
+package stringutils
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestTokenType(t *testing.T) {
+	assertions := assert.New(t)
+
+	result := TokenType{
+		Type:          "keyword",
+		Words:         []string{"for", "if"},
+		CaseSensitive: true,
+	}
+	assertions.Equal(true, result.CaseSensitive)
+	assertions.Equal([]string{"for", "if"}, result.Words)
+	assertions.Equal("keyword", result.Type)
+}
+
+func TestToken(t *testing.T) {
+	assertions := assert.New(t)
+
+	result := Token{
+		Type:       	"keyword",
+		Position:   	1,
+		Text:			"if",
+	}
+	assertions.Equal("keyword", result.Type)
+	assertions.Equal("if", result.Text)
+	assertions.Equal(1, result.Position)
+}