init commit, basic tokenizing and indexing logic without any ranking of results on a simple wikipedia dump

11 files changed, 267 insertions, 0 deletions

diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..7fecb8f
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Ignored default folder with query files
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
diff --git a/.idea/awesomeProject.iml b/.idea/awesomeProject.iml
new file mode 100644
index 0000000..7c421e3
--- /dev/null
+++ b/.idea/awesomeProject.iml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="GoModuleSettings" enabled="true" />
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/encodings.xml b/.idea/encodings.xml
new file mode 100644
index 0000000..df87cf9
--- /dev/null
+++ b/.idea/encodings.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" addBOMForNewFiles="with BOM under Windows, with no BOM otherwise" />
+</project>
\ No newline at end of file
diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml
new file mode 100644
index 0000000..644cdf0
--- /dev/null
+++ b/.idea/go.imports.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GoImports">
+    <option name="excludedPackages">
+      <array>
+        <option value="golang.org/x/net/context" />
+      </array>
+    </option>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..cc47053
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/awesomeProject.iml" filepath="$PROJECT_DIR$/.idea/awesomeProject.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go
new file mode 100644
index 0000000..2bb8f96
--- /dev/null
+++ b/analyzer/analyzer.go
@@ -0,0 +1,38 @@
+package analyzer
+
+import (
+	"strings"
+	"unicode"
+)
+
+var stopWords = map[string]struct{}{
+	"the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {},
+	"is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {},
+	"at": {}, "to": {}, "from": {}, "by": {}, "of": {},
+}
+
+func ProcessText(text string) []string {
+	text = strings.ToLower(text)
+	text = SanitizeText(text)
+	tokens := strings.Fields(text)
+	var cleanTokens []string
+	for _, token := range tokens {
+		_, exists := stopWords[token]
+		if !exists {
+			cleanTokens = append(cleanTokens, token)
+		}
+	}
+
+	return cleanTokens
+}
+
+func SanitizeText(s string) string {
+	var builder strings.Builder
+
+	for _, char := range s {
+		if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) {
+			builder.WriteRune(char)
+		}
+	}
+	return builder.String()
+}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..3463cc3
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,3 @@
+module searchEngine
+
+go 1.26
diff --git a/index/index.go b/index/index.go
new file mode 100644
index 0000000..7456213
--- /dev/null
+++ b/index/index.go
@@ -0,0 +1,32 @@
+package index
+
+import "sync"
+
+type InvertedIndex struct {
+	Mu       sync.RWMutex
+	Data     map[string][]int
+	DocNames map[int]string
+}
+
+func New() *InvertedIndex {
+	return &InvertedIndex{
+		Data:     make(map[string][]int),
+		DocNames: make(map[int]string),
+	}
+}
+
+func (idx *InvertedIndex) Add(docID int, title string, tokens []string) {
+	idx.Mu.Lock()
+	defer idx.Mu.Unlock()
+
+	idx.DocNames[docID] = title
+	seen := make(map[string]struct{})
+
+	for _, token := range tokens {
+		if _, exists := seen[token]; !exists {
+			idx.Data[token] = append(idx.Data[token], docID)
+
+			seen[token] = struct{}{}
+		}
+	}
+}
diff --git a/ingest/ingest.go b/ingest/ingest.go
new file mode 100644
index 0000000..bffc417
--- /dev/null
+++ b/ingest/ingest.go
@@ -0,0 +1,48 @@
+package ingest
+
+import (
+	"encoding/xml"
+	"io"
+	"os"
+)
+
+type Page struct {
+	Title string `xml:"title"`
+	Text  string `xml:"revision>text"`
+	Id    int    `xml:"id"`
+}
+
+func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// 3. Attach the streaming XML decoder
+	decoder := xml.NewDecoder(file)
+
+	for {
+		t, err := decoder.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		// Look for the opening <page> tag
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "page" {
+				var p Page
+				// Decode the entire page element into the struct
+				if err := decoder.DecodeElement(&p, &se); err == nil {
+					// Push to your worker pool channel
+					jobQueue <- p
+				}
+			}
+		}
+	}
+	return nil
+}
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..459da39
--- /dev/null
+++ b/main.go
@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"searchEngine/analyzer"
+	"searchEngine/index"
+	"searchEngine/ingest"
+	"searchEngine/search"
+	"sync"
+)
+
+func main() {
+	myIndex := index.New()
+	jobQueue := make(chan ingest.Page, 100)
+
+	var wg sync.WaitGroup
+
+	for range 4 {
+		wg.Go(func() {
+			for page := range jobQueue {
+				tokens := analyzer.ProcessText(page.Text)
+				myIndex.Add(page.Id, page.Title, tokens)
+			}
+		})
+	}
+
+	err := ingest.IngestWikiDump("C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml", jobQueue)
+	if err != nil {
+		panic(err)
+	}
+
+	close(jobQueue)
+	wg.Wait()
+	fmt.Println("Indexing complete! Running a test search...")
+
+	results := search.Search(myIndex, "computer science")
+
+	fmt.Printf("Found %d articles matching your query:\n", len(results))
+	for i, title := range results {
+		if i >= 10 {
+			fmt.Println("... and more.")
+			break
+		}
+		fmt.Printf("- %s\n", title)
+	}
+
+}
diff --git a/search/search.go b/search/search.go
new file mode 100644
index 0000000..ced94b7
--- /dev/null
+++ b/search/search.go
@@ -0,0 +1,58 @@
+package search
+
+import (
+	"searchEngine/analyzer"
+	"searchEngine/index"
+)
+
+func Search(idx *index.InvertedIndex, query string) []string {
+	tokenizedQuery := analyzer.ProcessText(query)
+
+	if len(tokenizedQuery) == 0 {
+		return nil
+	}
+
+	idx.Mu.RLock()
+	defer idx.Mu.RUnlock()
+
+	firstToken := tokenizedQuery[0]
+	baseIDs, exists := idx.Data[firstToken]
+	if !exists {
+		return nil
+	}
+
+	intersection := make([]int, len(baseIDs))
+	copy(intersection, baseIDs)
+
+	for i := 1; i < len(tokenizedQuery); i++ {
+		token := tokenizedQuery[i]
+
+		nextIDs, exists := idx.Data[token]
+		if !exists {
+			return nil
+		}
+
+		lookup := make(map[int]struct{})
+		for _, id := range nextIDs {
+			lookup[id] = struct{}{}
+		}
+
+		var filtered []int
+		for _, id := range intersection {
+			if _, found := lookup[id]; found {
+				filtered = append(filtered, id)
+			}
+		}
+
+		intersection = filtered
+
+	}
+	var results []string
+	for _, id := range intersection {
+		if title, exists := idx.DocNames[id]; exists {
+			results = append(results, title)
+		}
+	}
+
+	return results
+}