summaryrefslogtreecommitdiff
path: root/analyzer
diff options
context:
space:
mode:
Diffstat (limited to 'analyzer')
-rw-r--r--analyzer/analyzer.go38
1 files changed, 0 insertions, 38 deletions
diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go
deleted file mode 100644
index 2bb8f96..0000000
--- a/analyzer/analyzer.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package analyzer
-
-import (
- "strings"
- "unicode"
-)
-
-var stopWords = map[string]struct{}{
- "the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {},
- "is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {},
- "at": {}, "to": {}, "from": {}, "by": {}, "of": {},
-}
-
-func ProcessText(text string) []string {
- text = strings.ToLower(text)
- text = SanitizeText(text)
- tokens := strings.Fields(text)
- var cleanTokens []string
- for _, token := range tokens {
- _, exists := stopWords[token]
- if !exists {
- cleanTokens = append(cleanTokens, token)
- }
- }
-
- return cleanTokens
-}
-
-func SanitizeText(s string) string {
- var builder strings.Builder
-
- for _, char := range s {
- if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) {
- builder.WriteRune(char)
- }
- }
- return builder.String()
-}