blob: 4b6b15074451b78d93bb5d471605832d2bd6cb43 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
|
package core
import (
"fmt"
"searchEngine/core/analyzer"
"searchEngine/core/index"
"searchEngine/core/ingest"
"searchEngine/core/search"
"sync"
)
type Engine struct {
idx *index.InvertedIndex
}
func NewEngine() *Engine {
return &Engine{
idx: index.New(),
}
}
func (e *Engine) Index() *index.InvertedIndex {
return e.idx
}
func (e *Engine) IndexDump(filePath string, numWorkers int) error {
if numWorkers <= 0 {
numWorkers = 4
}
jobQueue := make(chan ingest.Page, 100)
var wg sync.WaitGroup
for i := 0; i < numWorkers; i++ {
wg.Go(func() {
for page := range jobQueue {
tokens := analyzer.ProcessText(page.Text)
e.idx.Add(page.Id, page.Title, tokens)
}
})
}
err := ingest.IngestWikiDump(filePath, jobQueue)
close(jobQueue)
wg.Wait()
if err != nil {
return fmt.Errorf("failed to ingest wiki dump: %w", err)
}
return nil
}
func (e *Engine) Search(query string) []string {
return search.Search(e.idx, query)
}
|
