blob: 6e929e2bccb19de279599f865e0f922dd70cff6a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
package core
import (
"fmt"
"searchEngine/core/analyzer"
"searchEngine/core/index"
"searchEngine/core/ingest"
"searchEngine/core/search"
"sync"
)
type Engine struct {
idx *index.InvertedIndex
}
func NewEngine() *Engine {
return &Engine{
idx: index.New(),
}
}
func NewEngineWithIndex(idx *index.InvertedIndex) *Engine {
return &Engine{
idx: idx,
}
}
func (e *Engine) Index() *index.InvertedIndex {
return e.idx
}
func (e *Engine) IndexDump(filePath string, numWorkers int) error {
if numWorkers <= 0 {
numWorkers = 4
}
jobQueue := make(chan ingest.Page, 100)
var wg sync.WaitGroup
for i := 0; i < numWorkers; i++ {
wg.Go(func() {
for page := range jobQueue {
tokens := analyzer.ProcessText(page.Text)
e.idx.Add(page.Id, page.Title, tokens)
}
})
}
var processedCount int
err := ingest.WikiDump(filePath, jobQueue, func() {
processedCount++
if processedCount%1000 == 0 {
fmt.Printf("\rProcessed %d articles...", processedCount)
}
})
close(jobQueue)
wg.Wait()
if err != nil {
return fmt.Errorf("failed to ingest wiki dump: %w", err)
}
return nil
}
func (e *Engine) Search(query string) []string {
return search.Search(e.idx, query)
}
|
