1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
|
package analyzer
import (
"strings"
"unicode"
)
var stopWords = map[string]struct{}{
"the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {},
"is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {},
"at": {}, "to": {}, "from": {}, "by": {}, "of": {},
}
func ProcessText(text string) []string {
text = strings.ToLower(text)
text = SanitizeText(text)
tokens := strings.Fields(text)
var cleanTokens []string
for _, token := range tokens {
_, exists := stopWords[token]
if !exists {
cleanTokens = append(cleanTokens, token)
}
}
return cleanTokens
}
func SanitizeText(s string) string {
var builder strings.Builder
for _, char := range s {
if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) {
builder.WriteRune(char)
}
}
return builder.String()
}
|
