diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | .idea/.gitignore | 10 | ||||
| -rw-r--r-- | .idea/.name | 1 | ||||
| -rw-r--r-- | .idea/encodings.xml | 4 | ||||
| -rw-r--r-- | .idea/go.imports.xml | 10 | ||||
| -rw-r--r-- | .idea/modules.xml | 8 | ||||
| -rw-r--r-- | .idea/search-engine.iml | 9 | ||||
| -rw-r--r-- | .idea/vcs.xml | 6 | ||||
| -rw-r--r-- | core/analyzer/analyzer.go (renamed from analyzer/analyzer.go) | 2 | ||||
| -rw-r--r-- | core/engine.go | 56 | ||||
| -rw-r--r-- | core/index/index.go (renamed from index/index.go) | 5 | ||||
| -rw-r--r-- | core/ingest/ingest.go (renamed from ingest/ingest.go) | 6 | ||||
| -rw-r--r-- | core/scoring/field.go (renamed from scoring/field.go) | 4 | ||||
| -rw-r--r-- | core/scoring/idf.go (renamed from scoring/idf.go) | 6 | ||||
| -rw-r--r-- | core/search/search.go (renamed from search/search.go) | 6 | ||||
| -rw-r--r-- | main.go | 31 |
16 files changed, 125 insertions, 41 deletions
@@ -1 +1 @@ -.idea
\ No newline at end of file +
\ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..7fecb8f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..84f5cc1 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +search-engine
\ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="Encoding" addBOMForNewFiles="with BOM under Windows, with no BOM otherwise" /> +</project>
\ No newline at end of file diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml new file mode 100644 index 0000000..644cdf0 --- /dev/null +++ b/.idea/go.imports.xml @@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="GoImports"> + <option name="excludedPackages"> + <array> + <option value="golang.org/x/net/context" /> + </array> + </option> + </component> +</project>
\ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..ef17046 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectModuleManager"> + <modules> + <module fileurl="file://$PROJECT_DIR$/.idea/search-engine.iml" filepath="$PROJECT_DIR$/.idea/search-engine.iml" /> + </modules> + </component> +</project>
\ No newline at end of file diff --git a/.idea/search-engine.iml b/.idea/search-engine.iml new file mode 100644 index 0000000..7c421e3 --- /dev/null +++ b/.idea/search-engine.iml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="WEB_MODULE" version="4"> + <component name="GoModuleSettings" enabled="true" /> + <component name="NewModuleRootManager"> + <content url="file://$MODULE_DIR$" /> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> +</module>
\ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="VcsDirectoryMappings"> + <mapping directory="$PROJECT_DIR$" vcs="Git" /> + </component> +</project>
\ No newline at end of file diff --git a/analyzer/analyzer.go b/core/analyzer/analyzer.go index 2bb8f96..6a02541 100644 --- a/analyzer/analyzer.go +++ b/core/analyzer/analyzer.go @@ -1,4 +1,4 @@ -package analyzer +package analyzer import ( "strings" diff --git a/core/engine.go b/core/engine.go new file mode 100644 index 0000000..4b6b150 --- /dev/null +++ b/core/engine.go @@ -0,0 +1,56 @@ +package core + +import ( + "fmt" + "searchEngine/core/analyzer" + "searchEngine/core/index" + "searchEngine/core/ingest" + "searchEngine/core/search" + "sync" +) + +type Engine struct { + idx *index.InvertedIndex +} + +func NewEngine() *Engine { + return &Engine{ + idx: index.New(), + } +} + +func (e *Engine) Index() *index.InvertedIndex { + return e.idx +} + +func (e *Engine) IndexDump(filePath string, numWorkers int) error { + if numWorkers <= 0 { + numWorkers = 4 + } + + jobQueue := make(chan ingest.Page, 100) + var wg sync.WaitGroup + + for i := 0; i < numWorkers; i++ { + wg.Go(func() { + for page := range jobQueue { + tokens := analyzer.ProcessText(page.Text) + e.idx.Add(page.Id, page.Title, tokens) + } + }) + } + + err := ingest.IngestWikiDump(filePath, jobQueue) + close(jobQueue) + wg.Wait() + + if err != nil { + return fmt.Errorf("failed to ingest wiki dump: %w", err) + } + + return nil +} + +func (e *Engine) Search(query string) []string { + return search.Search(e.idx, query) +} diff --git a/index/index.go b/core/index/index.go index 60c3d9f..59b315c 100644 --- a/index/index.go +++ b/core/index/index.go @@ -1,7 +1,7 @@ -package index +package index import ( - "searchEngine/analyzer" + "searchEngine/core/analyzer" "sync" ) @@ -12,6 +12,7 @@ type Posting struct { BodyPositions []int TitlePositions []int } + type InvertedIndex struct { Mu sync.RWMutex Data map[string][]Posting diff --git a/ingest/ingest.go b/core/ingest/ingest.go index bffc417..d54ff9d 100644 --- a/ingest/ingest.go +++ b/core/ingest/ingest.go @@ -1,4 +1,4 @@ -package ingest +package ingest import ( "encoding/xml" @@ -19,7 +19,6 @@ func IngestWikiDump(filePath string, jobQueue chan<- Page) error { } defer file.Close() - // 3. Attach the streaming XML decoder decoder := xml.NewDecoder(file) for { @@ -31,14 +30,11 @@ func IngestWikiDump(filePath string, jobQueue chan<- Page) error { return err } - // Look for the opening <page> tag switch se := t.(type) { case xml.StartElement: if se.Name.Local == "page" { var p Page - // Decode the entire page element into the struct if err := decoder.DecodeElement(&p, &se); err == nil { - // Push to your worker pool channel jobQueue <- p } } diff --git a/scoring/field.go b/core/scoring/field.go index d84ab3b..89f6132 100644 --- a/scoring/field.go +++ b/core/scoring/field.go @@ -1,6 +1,6 @@ -package scoring +package scoring -import "searchEngine/index" +import "searchEngine/core/index" var TitleWeight = 5.0 var BodyWeight = 1.0 diff --git a/scoring/idf.go b/core/scoring/idf.go index 5c85ba3..f5aaecd 100644 --- a/scoring/idf.go +++ b/core/scoring/idf.go @@ -1,8 +1,8 @@ -package scoring +package scoring import ( "math" - "searchEngine/index" + "searchEngine/core/index" ) func CalculateIDF(totalDocs int, docFrequency int) float64 { @@ -11,7 +11,7 @@ func CalculateIDF(totalDocs int, docFrequency int) float64 { } return math.Log10(float64(totalDocs) / float64(docFrequency)) } -func ScoreTFIDF(p index.Posting, idf float64) float64 { +func ScoreTFIDF(p index.Posting, idf float64) float64 { return FieldScore(p) * idf } diff --git a/search/search.go b/core/search/search.go index 061c99c..e159d43 100644 --- a/search/search.go +++ b/core/search/search.go @@ -1,8 +1,8 @@ -package search +package search import ( - "searchEngine/analyzer" - "searchEngine/index" + "searchEngine/core/analyzer" + "searchEngine/core/index" "slices" ) @@ -1,39 +1,23 @@ -package main +package main import ( "fmt" - "searchEngine/analyzer" - "searchEngine/index" - "searchEngine/ingest" - "searchEngine/search" - "sync" + "searchEngine/core" ) func main() { - myIndex := index.New() - jobQueue := make(chan ingest.Page, 100) + engine := core.NewEngine() - var wg sync.WaitGroup - - for range 4 { - wg.Go(func() { - for page := range jobQueue { - tokens := analyzer.ProcessText(page.Text) - myIndex.Add(page.Id, page.Title, tokens) - } - }) - } - - err := ingest.IngestWikiDump("C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml", jobQueue) + dumpPath := "C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml" + fmt.Println("Ingesting and indexing wiki dump...") + err := engine.IndexDump(dumpPath, 4) if err != nil { panic(err) } - close(jobQueue) - wg.Wait() fmt.Println("Indexing complete! Running a test search...") - results := search.Search(myIndex, "league of legends") + results := engine.Search("league of legends") fmt.Printf("Found %d articles matching your query:\n", len(results)) for i, title := range results { @@ -43,5 +27,4 @@ func main() { } fmt.Printf("- %s\n", title) } - } |