diff options
Diffstat (limited to 'analyzer')
| -rw-r--r-- | analyzer/analyzer.go | 38 |
1 files changed, 0 insertions, 38 deletions
diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go deleted file mode 100644 index 2bb8f96..0000000 --- a/analyzer/analyzer.go +++ /dev/null @@ -1,38 +0,0 @@ -package analyzer - -import ( - "strings" - "unicode" -) - -var stopWords = map[string]struct{}{ - "the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {}, - "is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {}, - "at": {}, "to": {}, "from": {}, "by": {}, "of": {}, -} - -func ProcessText(text string) []string { - text = strings.ToLower(text) - text = SanitizeText(text) - tokens := strings.Fields(text) - var cleanTokens []string - for _, token := range tokens { - _, exists := stopWords[token] - if !exists { - cleanTokens = append(cleanTokens, token) - } - } - - return cleanTokens -} - -func SanitizeText(s string) string { - var builder strings.Builder - - for _, char := range s { - if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) { - builder.WriteRune(char) - } - } - return builder.String() -} |
