summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--core/engine.go14
-rw-r--r--core/index/index.go42
-rw-r--r--core/ingest/ingest.go5
-rw-r--r--core/scoring/field.go2
-rw-r--r--core/search/search.go51
-rw-r--r--main.go65
-rw-r--r--wikipedia_index.gobbin0 -> 259 bytes
7 files changed, 127 insertions, 52 deletions
diff --git a/core/engine.go b/core/engine.go
index 4b6b150..6e929e2 100644
--- a/core/engine.go
+++ b/core/engine.go
@@ -19,6 +19,11 @@ func NewEngine() *Engine {
}
}
+func NewEngineWithIndex(idx *index.InvertedIndex) *Engine {
+ return &Engine{
+ idx: idx,
+ }
+}
func (e *Engine) Index() *index.InvertedIndex {
return e.idx
}
@@ -39,8 +44,13 @@ func (e *Engine) IndexDump(filePath string, numWorkers int) error {
}
})
}
-
- err := ingest.IngestWikiDump(filePath, jobQueue)
+ var processedCount int
+ err := ingest.WikiDump(filePath, jobQueue, func() {
+ processedCount++
+ if processedCount%1000 == 0 {
+ fmt.Printf("\rProcessed %d articles...", processedCount)
+ }
+ })
close(jobQueue)
wg.Wait()
diff --git a/core/index/index.go b/core/index/index.go
index 59b315c..2e669fc 100644
--- a/core/index/index.go
+++ b/core/index/index.go
@@ -1,20 +1,20 @@
package index
import (
+ "encoding/gob"
+ "os"
"searchEngine/core/analyzer"
"sync"
)
type Posting struct {
DocId int
- BodyCount int
- TitleCount int
BodyPositions []int
TitlePositions []int
}
type InvertedIndex struct {
- Mu sync.RWMutex
+ Mu sync.RWMutex `gob:"-"`
Data map[string][]Posting
DocNames map[int]string
}
@@ -30,23 +30,19 @@ func (idx *InvertedIndex) Add(docID int, title string, bodyTokens []string) {
titleTokens := analyzer.ProcessText(title)
type fieldStats struct {
- body int
bodyPositions []int
- title int
titlePositions []int
}
docStats := make(map[string]fieldStats)
for position, token := range titleTokens {
stats := docStats[token]
- stats.title++
stats.titlePositions = append(stats.titlePositions, position)
docStats[token] = stats
}
for position, token := range bodyTokens {
stats := docStats[token]
- stats.body++
stats.bodyPositions = append(stats.bodyPositions, position)
docStats[token] = stats
}
@@ -59,10 +55,38 @@ func (idx *InvertedIndex) Add(docID int, title string, bodyTokens []string) {
for token, stats := range docStats {
idx.Data[token] = append(idx.Data[token], Posting{
DocId: docID,
- BodyCount: stats.body,
- TitleCount: stats.title,
BodyPositions: stats.bodyPositions,
TitlePositions: stats.titlePositions,
})
}
}
+
+func (idx *InvertedIndex) Save(filepath string) error {
+ idx.Mu.RLock()
+ defer idx.Mu.RUnlock()
+
+ file, err := os.Create(filepath)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ encoder := gob.NewEncoder(file)
+ return encoder.Encode(idx)
+}
+
+func Load(filepath string) (*InvertedIndex, error) {
+ file, err := os.Open(filepath)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ idx := &InvertedIndex{}
+ decoder := gob.NewDecoder(file)
+ err = decoder.Decode(idx)
+ if err != nil {
+ return nil, err
+ }
+ return idx, nil
+}
diff --git a/core/ingest/ingest.go b/core/ingest/ingest.go
index d54ff9d..ad80fe5 100644
--- a/core/ingest/ingest.go
+++ b/core/ingest/ingest.go
@@ -12,7 +12,7 @@ type Page struct {
Id int `xml:"id"`
}
-func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
+func WikiDump(filePath string, jobQueue chan<- Page, onProgress func()) error {
file, err := os.Open(filePath)
if err != nil {
return err
@@ -36,6 +36,9 @@ func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
var p Page
if err := decoder.DecodeElement(&p, &se); err == nil {
jobQueue <- p
+ if onProgress != nil {
+ onProgress()
+ }
}
}
}
diff --git a/core/scoring/field.go b/core/scoring/field.go
index 89f6132..80aaffe 100644
--- a/core/scoring/field.go
+++ b/core/scoring/field.go
@@ -6,5 +6,5 @@ var TitleWeight = 5.0
var BodyWeight = 1.0
func FieldScore(p index.Posting) float64 {
- return (float64(p.TitleCount) * TitleWeight) + (float64(p.BodyCount) * BodyWeight)
+ return (float64(len(p.TitlePositions)) * TitleWeight) + (float64(len(p.BodyPositions)) * BodyWeight)
}
diff --git a/core/search/search.go b/core/search/search.go
index e159d43..721574c 100644
--- a/core/search/search.go
+++ b/core/search/search.go
@@ -1,14 +1,16 @@
package search
import (
+ "cmp"
"searchEngine/core/analyzer"
"searchEngine/core/index"
+ "searchEngine/core/scoring"
"slices"
)
type Result struct {
DocID int
- Score int
+ Score float64
}
func Search(idx *index.InvertedIndex, query string) []string {
@@ -20,45 +22,44 @@ func Search(idx *index.InvertedIndex, query string) []string {
idx.Mu.RLock()
defer idx.Mu.RUnlock()
- firstToken := tokenizedQuery[0]
- basePostings, exists := idx.Data[firstToken]
- if !exists {
- return nil
- }
+ totalDocs := len(idx.DocNames)
+ docMatchCounts := make(map[int]int)
+ scores := make(map[int]float64)
- scores := make(map[int]int)
- for _, p := range basePostings {
- scores[p.DocId] = p.BodyCount
+ uniqueMap := make(map[string]bool)
+ var uniqueTokens []string
+ for _, token := range tokenizedQuery {
+ if !uniqueMap[token] {
+ uniqueMap[token] = true
+ uniqueTokens = append(uniqueTokens, token)
+ }
}
- for i := 1; i < len(tokenizedQuery); i++ {
- token := tokenizedQuery[i]
- nextPostings, exists := idx.Data[token]
+ for _, token := range tokenizedQuery {
+ postings, exists := idx.Data[token]
if !exists {
return nil
}
- lookup := make(map[int]int)
- for _, p := range nextPostings {
- lookup[p.DocId] = p.BodyCount
- }
+ docFrequency := len(postings)
+ idf := scoring.CalculateIDF(totalDocs, docFrequency)
- newScores := make(map[int]int)
- for docID, currentScore := range scores {
- if nextCount, found := lookup[docID]; found {
- newScores[docID] = currentScore + nextCount
- }
+ for _, p := range postings {
+ docMatchCounts[p.DocId]++
+ termScore := scoring.ScoreTFIDF(p, idf)
+ scores[p.DocId] += termScore
}
- scores = newScores
}
var results []Result
- for docID, score := range scores {
- results = append(results, Result{DocID: docID, Score: score})
+ for docID, count := range docMatchCounts {
+ if count == len(uniqueTokens) {
+ results = append(results, Result{DocID: docID, Score: scores[docID]})
+ }
}
slices.SortFunc(results, func(a, b Result) int {
- return b.Score - a.Score
+ return cmp.Compare(b.Score, a.Score)
})
var titles []string
diff --git a/main.go b/main.go
index c2de95e..7e9eaf2 100644
--- a/main.go
+++ b/main.go
@@ -2,29 +2,66 @@ package main
import (
"fmt"
+ "os"
"searchEngine/core"
+ "searchEngine/core/index"
)
func main() {
- engine := core.NewEngine()
+ indexPath := "wikipedia_index.gob"
+ var engine *core.Engine
- dumpPath := "C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml"
- fmt.Println("Ingesting and indexing wiki dump...")
- err := engine.IndexDump(dumpPath, 4)
- if err != nil {
- panic(err)
+ if _, err := os.Stat(indexPath); err == nil {
+ fmt.Println("Loading pre-built index from disk...")
+ idx, loadErr := index.Load(indexPath)
+ if loadErr != nil {
+ fmt.Printf("Warning: Cache file is corrupted (%v). Re-indexing...\n", loadErr)
+ os.Remove(indexPath)
+ engine = core.NewEngine()
+ dumpPath := "C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml"
+
+ fmt.Println("Ingesting and indexing wiki dump...")
+ err := engine.IndexDump(dumpPath, 4)
+ if err != nil {
+ panic(err)
+ }
+
+ fmt.Println("\nIndexing complete! Saving index to disk for future runs...")
+ err = engine.Index().Save(indexPath)
+ if err != nil {
+ fmt.Printf("Warning: Failed to save index cache: %v\n", err)
+ }
+ } else {
+ engine = core.NewEngineWithIndex(idx)
+ fmt.Println("Index loaded successfully!")
+
+ }
}
- fmt.Println("Indexing complete! Running a test search...")
+ fmt.Println("Running test searches...")
+
+ searches := []string{
+ "albert einstein",
+ "second world war",
+ "solar system",
+ "rome juliet shakespeare",
+ "cold war united states soviet union",
+ "quantum mechanics",
+ "capital city of france",
+ " ALBERT EINSTEIN ",
+ }
- results := engine.Search("league of legends")
+ for _, query := range searches {
+ fmt.Printf("\nsearching %s:\n", query)
+ results := engine.Search(query)
- fmt.Printf("Found %d articles matching your query:\n", len(results))
- for i, title := range results {
- if i >= 10 {
- fmt.Println("... and more.")
- break
+ fmt.Printf("Found %d articles matching your query:\n", len(results))
+ for i, title := range results {
+ if i >= 10 {
+ fmt.Println("... and more.")
+ break
+ }
+ fmt.Printf("- %s\n", title)
}
- fmt.Printf("- %s\n", title)
}
}
diff --git a/wikipedia_index.gob b/wikipedia_index.gob
new file mode 100644
index 0000000..dd392c9
--- /dev/null
+++ b/wikipedia_index.gob
Binary files differ
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-05 02:21:37 +0000