diff options
| -rw-r--r-- | .idea/.gitignore | 10 | ||||
| -rw-r--r-- | .idea/awesomeProject.iml | 9 | ||||
| -rw-r--r-- | .idea/encodings.xml | 4 | ||||
| -rw-r--r-- | .idea/go.imports.xml | 10 | ||||
| -rw-r--r-- | .idea/modules.xml | 8 | ||||
| -rw-r--r-- | analyzer/analyzer.go | 38 | ||||
| -rw-r--r-- | go.mod | 3 | ||||
| -rw-r--r-- | index/index.go | 32 | ||||
| -rw-r--r-- | ingest/ingest.go | 48 | ||||
| -rw-r--r-- | main.go | 47 | ||||
| -rw-r--r-- | search/search.go | 58 |
11 files changed, 267 insertions, 0 deletions
diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..7fecb8f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/awesomeProject.iml b/.idea/awesomeProject.iml new file mode 100644 index 0000000..7c421e3 --- /dev/null +++ b/.idea/awesomeProject.iml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="WEB_MODULE" version="4"> + <component name="GoModuleSettings" enabled="true" /> + <component name="NewModuleRootManager"> + <content url="file://$MODULE_DIR$" /> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> +</module>
\ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="Encoding" addBOMForNewFiles="with BOM under Windows, with no BOM otherwise" /> +</project>
\ No newline at end of file diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml new file mode 100644 index 0000000..644cdf0 --- /dev/null +++ b/.idea/go.imports.xml @@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="GoImports"> + <option name="excludedPackages"> + <array> + <option value="golang.org/x/net/context" /> + </array> + </option> + </component> +</project>
\ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..cc47053 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectModuleManager"> + <modules> + <module fileurl="file://$PROJECT_DIR$/.idea/awesomeProject.iml" filepath="$PROJECT_DIR$/.idea/awesomeProject.iml" /> + </modules> + </component> +</project>
\ No newline at end of file diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go new file mode 100644 index 0000000..2bb8f96 --- /dev/null +++ b/analyzer/analyzer.go @@ -0,0 +1,38 @@ +package analyzer + +import ( + "strings" + "unicode" +) + +var stopWords = map[string]struct{}{ + "the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {}, + "is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {}, + "at": {}, "to": {}, "from": {}, "by": {}, "of": {}, +} + +func ProcessText(text string) []string { + text = strings.ToLower(text) + text = SanitizeText(text) + tokens := strings.Fields(text) + var cleanTokens []string + for _, token := range tokens { + _, exists := stopWords[token] + if !exists { + cleanTokens = append(cleanTokens, token) + } + } + + return cleanTokens +} + +func SanitizeText(s string) string { + var builder strings.Builder + + for _, char := range s { + if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) { + builder.WriteRune(char) + } + } + return builder.String() +} @@ -0,0 +1,3 @@ +module searchEngine + +go 1.26 diff --git a/index/index.go b/index/index.go new file mode 100644 index 0000000..7456213 --- /dev/null +++ b/index/index.go @@ -0,0 +1,32 @@ +package index + +import "sync" + +type InvertedIndex struct { + Mu sync.RWMutex + Data map[string][]int + DocNames map[int]string +} + +func New() *InvertedIndex { + return &InvertedIndex{ + Data: make(map[string][]int), + DocNames: make(map[int]string), + } +} + +func (idx *InvertedIndex) Add(docID int, title string, tokens []string) { + idx.Mu.Lock() + defer idx.Mu.Unlock() + + idx.DocNames[docID] = title + seen := make(map[string]struct{}) + + for _, token := range tokens { + if _, exists := seen[token]; !exists { + idx.Data[token] = append(idx.Data[token], docID) + + seen[token] = struct{}{} + } + } +} diff --git a/ingest/ingest.go b/ingest/ingest.go new file mode 100644 index 0000000..bffc417 --- /dev/null +++ b/ingest/ingest.go @@ -0,0 +1,48 @@ +package ingest + +import ( + "encoding/xml" + "io" + "os" +) + +type Page struct { + Title string `xml:"title"` + Text string `xml:"revision>text"` + Id int `xml:"id"` +} + +func IngestWikiDump(filePath string, jobQueue chan<- Page) error { + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + // 3. Attach the streaming XML decoder + decoder := xml.NewDecoder(file) + + for { + t, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return err + } + + // Look for the opening <page> tag + switch se := t.(type) { + case xml.StartElement: + if se.Name.Local == "page" { + var p Page + // Decode the entire page element into the struct + if err := decoder.DecodeElement(&p, &se); err == nil { + // Push to your worker pool channel + jobQueue <- p + } + } + } + } + return nil +} @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "searchEngine/analyzer" + "searchEngine/index" + "searchEngine/ingest" + "searchEngine/search" + "sync" +) + +func main() { + myIndex := index.New() + jobQueue := make(chan ingest.Page, 100) + + var wg sync.WaitGroup + + for range 4 { + wg.Go(func() { + for page := range jobQueue { + tokens := analyzer.ProcessText(page.Text) + myIndex.Add(page.Id, page.Title, tokens) + } + }) + } + + err := ingest.IngestWikiDump("C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml", jobQueue) + if err != nil { + panic(err) + } + + close(jobQueue) + wg.Wait() + fmt.Println("Indexing complete! Running a test search...") + + results := search.Search(myIndex, "computer science") + + fmt.Printf("Found %d articles matching your query:\n", len(results)) + for i, title := range results { + if i >= 10 { + fmt.Println("... and more.") + break + } + fmt.Printf("- %s\n", title) + } + +} diff --git a/search/search.go b/search/search.go new file mode 100644 index 0000000..ced94b7 --- /dev/null +++ b/search/search.go @@ -0,0 +1,58 @@ +package search + +import ( + "searchEngine/analyzer" + "searchEngine/index" +) + +func Search(idx *index.InvertedIndex, query string) []string { + tokenizedQuery := analyzer.ProcessText(query) + + if len(tokenizedQuery) == 0 { + return nil + } + + idx.Mu.RLock() + defer idx.Mu.RUnlock() + + firstToken := tokenizedQuery[0] + baseIDs, exists := idx.Data[firstToken] + if !exists { + return nil + } + + intersection := make([]int, len(baseIDs)) + copy(intersection, baseIDs) + + for i := 1; i < len(tokenizedQuery); i++ { + token := tokenizedQuery[i] + + nextIDs, exists := idx.Data[token] + if !exists { + return nil + } + + lookup := make(map[int]struct{}) + for _, id := range nextIDs { + lookup[id] = struct{}{} + } + + var filtered []int + for _, id := range intersection { + if _, found := lookup[id]; found { + filtered = append(filtered, id) + } + } + + intersection = filtered + + } + var results []string + for _, id := range intersection { + if title, exists := idx.DocNames[id]; exists { + results = append(results, title) + } + } + + return results +} |