init commit, basic tokenizing and indexing logic without any ranking of results on a simple wikipedia dump

1 files changed, 32 insertions, 0 deletions

diff --git a/index/index.go b/index/index.go
new file mode 100644
index 0000000..7456213
--- /dev/null
+++ b/index/index.go
@@ -0,0 +1,32 @@
+package index
+
+import "sync"
+
+type InvertedIndex struct {
+	Mu       sync.RWMutex
+	Data     map[string][]int
+	DocNames map[int]string
+}
+
+func New() *InvertedIndex {
+	return &InvertedIndex{
+		Data:     make(map[string][]int),
+		DocNames: make(map[int]string),
+	}
+}
+
+func (idx *InvertedIndex) Add(docID int, title string, tokens []string) {
+	idx.Mu.Lock()
+	defer idx.Mu.Unlock()
+
+	idx.DocNames[docID] = title
+	seen := make(map[string]struct{})
+
+	for _, token := range tokens {
+		if _, exists := seen[token]; !exists {
+			idx.Data[token] = append(idx.Data[token], docID)
+
+			seen[token] = struct{}{}
+		}
+	}
+}