summaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
Diffstat (limited to 'core')
-rw-r--r--core/engine.go14
-rw-r--r--core/index/index.go42
-rw-r--r--core/ingest/ingest.go5
-rw-r--r--core/scoring/field.go2
-rw-r--r--core/search/search.go51
5 files changed, 76 insertions, 38 deletions
diff --git a/core/engine.go b/core/engine.go
index 4b6b150..6e929e2 100644
--- a/core/engine.go
+++ b/core/engine.go
@@ -19,6 +19,11 @@ func NewEngine() *Engine {
}
}
+func NewEngineWithIndex(idx *index.InvertedIndex) *Engine {
+ return &Engine{
+ idx: idx,
+ }
+}
func (e *Engine) Index() *index.InvertedIndex {
return e.idx
}
@@ -39,8 +44,13 @@ func (e *Engine) IndexDump(filePath string, numWorkers int) error {
}
})
}
-
- err := ingest.IngestWikiDump(filePath, jobQueue)
+ var processedCount int
+ err := ingest.WikiDump(filePath, jobQueue, func() {
+ processedCount++
+ if processedCount%1000 == 0 {
+ fmt.Printf("\rProcessed %d articles...", processedCount)
+ }
+ })
close(jobQueue)
wg.Wait()
diff --git a/core/index/index.go b/core/index/index.go
index 59b315c..2e669fc 100644
--- a/core/index/index.go
+++ b/core/index/index.go
@@ -1,20 +1,20 @@
package index
import (
+ "encoding/gob"
+ "os"
"searchEngine/core/analyzer"
"sync"
)
type Posting struct {
DocId int
- BodyCount int
- TitleCount int
BodyPositions []int
TitlePositions []int
}
type InvertedIndex struct {
- Mu sync.RWMutex
+ Mu sync.RWMutex `gob:"-"`
Data map[string][]Posting
DocNames map[int]string
}
@@ -30,23 +30,19 @@ func (idx *InvertedIndex) Add(docID int, title string, bodyTokens []string) {
titleTokens := analyzer.ProcessText(title)
type fieldStats struct {
- body int
bodyPositions []int
- title int
titlePositions []int
}
docStats := make(map[string]fieldStats)
for position, token := range titleTokens {
stats := docStats[token]
- stats.title++
stats.titlePositions = append(stats.titlePositions, position)
docStats[token] = stats
}
for position, token := range bodyTokens {
stats := docStats[token]
- stats.body++
stats.bodyPositions = append(stats.bodyPositions, position)
docStats[token] = stats
}
@@ -59,10 +55,38 @@ func (idx *InvertedIndex) Add(docID int, title string, bodyTokens []string) {
for token, stats := range docStats {
idx.Data[token] = append(idx.Data[token], Posting{
DocId: docID,
- BodyCount: stats.body,
- TitleCount: stats.title,
BodyPositions: stats.bodyPositions,
TitlePositions: stats.titlePositions,
})
}
}
+
+func (idx *InvertedIndex) Save(filepath string) error {
+ idx.Mu.RLock()
+ defer idx.Mu.RUnlock()
+
+ file, err := os.Create(filepath)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ encoder := gob.NewEncoder(file)
+ return encoder.Encode(idx)
+}
+
+func Load(filepath string) (*InvertedIndex, error) {
+ file, err := os.Open(filepath)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ idx := &InvertedIndex{}
+ decoder := gob.NewDecoder(file)
+ err = decoder.Decode(idx)
+ if err != nil {
+ return nil, err
+ }
+ return idx, nil
+}
diff --git a/core/ingest/ingest.go b/core/ingest/ingest.go
index d54ff9d..ad80fe5 100644
--- a/core/ingest/ingest.go
+++ b/core/ingest/ingest.go
@@ -12,7 +12,7 @@ type Page struct {
Id int `xml:"id"`
}
-func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
+func WikiDump(filePath string, jobQueue chan<- Page, onProgress func()) error {
file, err := os.Open(filePath)
if err != nil {
return err
@@ -36,6 +36,9 @@ func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
var p Page
if err := decoder.DecodeElement(&p, &se); err == nil {
jobQueue <- p
+ if onProgress != nil {
+ onProgress()
+ }
}
}
}
diff --git a/core/scoring/field.go b/core/scoring/field.go
index 89f6132..80aaffe 100644
--- a/core/scoring/field.go
+++ b/core/scoring/field.go
@@ -6,5 +6,5 @@ var TitleWeight = 5.0
var BodyWeight = 1.0
func FieldScore(p index.Posting) float64 {
- return (float64(p.TitleCount) * TitleWeight) + (float64(p.BodyCount) * BodyWeight)
+ return (float64(len(p.TitlePositions)) * TitleWeight) + (float64(len(p.BodyPositions)) * BodyWeight)
}
diff --git a/core/search/search.go b/core/search/search.go
index e159d43..721574c 100644
--- a/core/search/search.go
+++ b/core/search/search.go
@@ -1,14 +1,16 @@
package search
import (
+ "cmp"
"searchEngine/core/analyzer"
"searchEngine/core/index"
+ "searchEngine/core/scoring"
"slices"
)
type Result struct {
DocID int
- Score int
+ Score float64
}
func Search(idx *index.InvertedIndex, query string) []string {
@@ -20,45 +22,44 @@ func Search(idx *index.InvertedIndex, query string) []string {
idx.Mu.RLock()
defer idx.Mu.RUnlock()
- firstToken := tokenizedQuery[0]
- basePostings, exists := idx.Data[firstToken]
- if !exists {
- return nil
- }
+ totalDocs := len(idx.DocNames)
+ docMatchCounts := make(map[int]int)
+ scores := make(map[int]float64)
- scores := make(map[int]int)
- for _, p := range basePostings {
- scores[p.DocId] = p.BodyCount
+ uniqueMap := make(map[string]bool)
+ var uniqueTokens []string
+ for _, token := range tokenizedQuery {
+ if !uniqueMap[token] {
+ uniqueMap[token] = true
+ uniqueTokens = append(uniqueTokens, token)
+ }
}
- for i := 1; i < len(tokenizedQuery); i++ {
- token := tokenizedQuery[i]
- nextPostings, exists := idx.Data[token]
+ for _, token := range tokenizedQuery {
+ postings, exists := idx.Data[token]
if !exists {
return nil
}
- lookup := make(map[int]int)
- for _, p := range nextPostings {
- lookup[p.DocId] = p.BodyCount
- }
+ docFrequency := len(postings)
+ idf := scoring.CalculateIDF(totalDocs, docFrequency)
- newScores := make(map[int]int)
- for docID, currentScore := range scores {
- if nextCount, found := lookup[docID]; found {
- newScores[docID] = currentScore + nextCount
- }
+ for _, p := range postings {
+ docMatchCounts[p.DocId]++
+ termScore := scoring.ScoreTFIDF(p, idf)
+ scores[p.DocId] += termScore
}
- scores = newScores
}
var results []Result
- for docID, score := range scores {
- results = append(results, Result{DocID: docID, Score: score})
+ for docID, count := range docMatchCounts {
+ if count == len(uniqueTokens) {
+ results = append(results, Result{DocID: docID, Score: scores[docID]})
+ }
}
slices.SortFunc(results, func(a, b Result) int {
- return b.Score - a.Score
+ return cmp.Compare(b.Score, a.Score)
})
var titles []string
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-05 04:25:42 +0000