From 3038483c570af7b923dda39e857b9cab8f89e5dd Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 27 Jul 2026 23:55:14 +0200 Subject: init commit, basic tokenizing and indexing logic without any ranking of results on a simple wikipedia dump --- analyzer/analyzer.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 analyzer/analyzer.go (limited to 'analyzer') diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go new file mode 100644 index 0000000..2bb8f96 --- /dev/null +++ b/analyzer/analyzer.go @@ -0,0 +1,38 @@ +package analyzer + +import ( + "strings" + "unicode" +) + +var stopWords = map[string]struct{}{ + "the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {}, + "is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {}, + "at": {}, "to": {}, "from": {}, "by": {}, "of": {}, +} + +func ProcessText(text string) []string { + text = strings.ToLower(text) + text = SanitizeText(text) + tokens := strings.Fields(text) + var cleanTokens []string + for _, token := range tokens { + _, exists := stopWords[token] + if !exists { + cleanTokens = append(cleanTokens, token) + } + } + + return cleanTokens +} + +func SanitizeText(s string) string { + var builder strings.Builder + + for _, char := range s { + if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) { + builder.WriteRune(char) + } + } + return builder.String() +} -- cgit v1.2.3