package analyzer

import (
	"strings"
	"unicode"
)

var stopWords = map[string]struct{}{
	"the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {},
	"is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {},
	"at": {}, "to": {}, "from": {}, "by": {}, "of": {},
}

func ProcessText(text string) []string {
	text = strings.ToLower(text)
	text = SanitizeText(text)
	tokens := strings.Fields(text)
	var cleanTokens []string
	for _, token := range tokens {
		_, exists := stopWords[token]
		if !exists {
			cleanTokens = append(cleanTokens, token)
		}
	}

	return cleanTokens
}

func SanitizeText(s string) string {
	var builder strings.Builder

	for _, char := range s {
		if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) {
			builder.WriteRune(char)
		}
	}
	return builder.String()
}