init commit, basic tokenizing and indexing logic without any ranking of results on a simple wikipedia dump

1 files changed, 38 insertions, 0 deletions

diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go
new file mode 100644
index 0000000..2bb8f96
--- /dev/null
+++ b/analyzer/analyzer.go
@@ -0,0 +1,38 @@
+package analyzer
+
+import (
+	"strings"
+	"unicode"
+)
+
+var stopWords = map[string]struct{}{
+	"the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {},
+	"is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {},
+	"at": {}, "to": {}, "from": {}, "by": {}, "of": {},
+}
+
+func ProcessText(text string) []string {
+	text = strings.ToLower(text)
+	text = SanitizeText(text)
+	tokens := strings.Fields(text)
+	var cleanTokens []string
+	for _, token := range tokens {
+		_, exists := stopWords[token]
+		if !exists {
+			cleanTokens = append(cleanTokens, token)
+		}
+	}
+
+	return cleanTokens
+}
+
+func SanitizeText(s string) string {
+	var builder strings.Builder
+
+	for _, char := range s {
+		if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) {
+			builder.WriteRune(char)
+		}
+	}
+	return builder.String()
+}