From c91c3afd8700138f8eb8a25a4ae422c2c9e46cea Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 28 Jul 2026 17:37:06 +0200 Subject: refactored entire code base to be in the package core so i can later implement interfaces to interact with the engine --- core/analyzer/analyzer.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 core/analyzer/analyzer.go (limited to 'core/analyzer/analyzer.go') diff --git a/core/analyzer/analyzer.go b/core/analyzer/analyzer.go new file mode 100644 index 0000000..6a02541 --- /dev/null +++ b/core/analyzer/analyzer.go @@ -0,0 +1,38 @@ +package analyzer + +import ( + "strings" + "unicode" +) + +var stopWords = map[string]struct{}{ + "the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {}, + "is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {}, + "at": {}, "to": {}, "from": {}, "by": {}, "of": {}, +} + +func ProcessText(text string) []string { + text = strings.ToLower(text) + text = SanitizeText(text) + tokens := strings.Fields(text) + var cleanTokens []string + for _, token := range tokens { + _, exists := stopWords[token] + if !exists { + cleanTokens = append(cleanTokens, token) + } + } + + return cleanTokens +} + +func SanitizeText(s string) string { + var builder strings.Builder + + for _, char := range s { + if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) { + builder.WriteRune(char) + } + } + return builder.String() +} -- cgit v1.2.3