From c91c3afd8700138f8eb8a25a4ae422c2c9e46cea Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 28 Jul 2026 17:37:06 +0200 Subject: refactored entire code base to be in the package core so i can later implement interfaces to interact with the engine --- .gitignore | 2 +- .idea/.gitignore | 10 +++++++ .idea/.name | 1 + .idea/encodings.xml | 4 +++ .idea/go.imports.xml | 10 +++++++ .idea/modules.xml | 8 ++++++ .idea/search-engine.iml | 9 ++++++ .idea/vcs.xml | 6 ++++ analyzer/analyzer.go | 38 ------------------------- core/analyzer/analyzer.go | 38 +++++++++++++++++++++++++ core/engine.go | 56 ++++++++++++++++++++++++++++++++++++ core/index/index.go | 68 ++++++++++++++++++++++++++++++++++++++++++++ core/ingest/ingest.go | 44 +++++++++++++++++++++++++++++ core/scoring/field.go | 10 +++++++ core/scoring/idf.go | 17 +++++++++++ core/search/search.go | 72 +++++++++++++++++++++++++++++++++++++++++++++++ index/index.go | 67 ------------------------------------------- ingest/ingest.go | 48 ------------------------------- main.go | 31 +++++--------------- scoring/field.go | 10 ------- scoring/idf.go | 17 ----------- search/search.go | 72 ----------------------------------------------- 22 files changed, 361 insertions(+), 277 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/.name create mode 100644 .idea/encodings.xml create mode 100644 .idea/go.imports.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/search-engine.iml create mode 100644 .idea/vcs.xml delete mode 100644 analyzer/analyzer.go create mode 100644 core/analyzer/analyzer.go create mode 100644 core/engine.go create mode 100644 core/index/index.go create mode 100644 core/ingest/ingest.go create mode 100644 core/scoring/field.go create mode 100644 core/scoring/idf.go create mode 100644 core/search/search.go delete mode 100644 index/index.go delete mode 100644 ingest/ingest.go delete mode 100644 scoring/field.go delete mode 100644 scoring/idf.go delete mode 100644 search/search.go diff --git a/.gitignore b/.gitignore index 874e31d..5f28270 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -.idea \ No newline at end of file + \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..7fecb8f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..84f5cc1 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +search-engine \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml new file mode 100644 index 0000000..644cdf0 --- /dev/null +++ b/.idea/go.imports.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..ef17046 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/search-engine.iml b/.idea/search-engine.iml new file mode 100644 index 0000000..7c421e3 --- /dev/null +++ b/.idea/search-engine.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go deleted file mode 100644 index 2bb8f96..0000000 --- a/analyzer/analyzer.go +++ /dev/null @@ -1,38 +0,0 @@ -package analyzer - -import ( - "strings" - "unicode" -) - -var stopWords = map[string]struct{}{ - "the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {}, - "is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {}, - "at": {}, "to": {}, "from": {}, "by": {}, "of": {}, -} - -func ProcessText(text string) []string { - text = strings.ToLower(text) - text = SanitizeText(text) - tokens := strings.Fields(text) - var cleanTokens []string - for _, token := range tokens { - _, exists := stopWords[token] - if !exists { - cleanTokens = append(cleanTokens, token) - } - } - - return cleanTokens -} - -func SanitizeText(s string) string { - var builder strings.Builder - - for _, char := range s { - if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) { - builder.WriteRune(char) - } - } - return builder.String() -} diff --git a/core/analyzer/analyzer.go b/core/analyzer/analyzer.go new file mode 100644 index 0000000..6a02541 --- /dev/null +++ b/core/analyzer/analyzer.go @@ -0,0 +1,38 @@ +package analyzer + +import ( + "strings" + "unicode" +) + +var stopWords = map[string]struct{}{ + "the": {}, "a": {}, "an": {}, "and": {}, "or": {}, "but": {}, + "is": {}, "are": {}, "was": {}, "were": {}, "in": {}, "on": {}, + "at": {}, "to": {}, "from": {}, "by": {}, "of": {}, +} + +func ProcessText(text string) []string { + text = strings.ToLower(text) + text = SanitizeText(text) + tokens := strings.Fields(text) + var cleanTokens []string + for _, token := range tokens { + _, exists := stopWords[token] + if !exists { + cleanTokens = append(cleanTokens, token) + } + } + + return cleanTokens +} + +func SanitizeText(s string) string { + var builder strings.Builder + + for _, char := range s { + if unicode.IsLetter(char) || unicode.IsNumber(char) || unicode.IsSpace(char) { + builder.WriteRune(char) + } + } + return builder.String() +} diff --git a/core/engine.go b/core/engine.go new file mode 100644 index 0000000..4b6b150 --- /dev/null +++ b/core/engine.go @@ -0,0 +1,56 @@ +package core + +import ( + "fmt" + "searchEngine/core/analyzer" + "searchEngine/core/index" + "searchEngine/core/ingest" + "searchEngine/core/search" + "sync" +) + +type Engine struct { + idx *index.InvertedIndex +} + +func NewEngine() *Engine { + return &Engine{ + idx: index.New(), + } +} + +func (e *Engine) Index() *index.InvertedIndex { + return e.idx +} + +func (e *Engine) IndexDump(filePath string, numWorkers int) error { + if numWorkers <= 0 { + numWorkers = 4 + } + + jobQueue := make(chan ingest.Page, 100) + var wg sync.WaitGroup + + for i := 0; i < numWorkers; i++ { + wg.Go(func() { + for page := range jobQueue { + tokens := analyzer.ProcessText(page.Text) + e.idx.Add(page.Id, page.Title, tokens) + } + }) + } + + err := ingest.IngestWikiDump(filePath, jobQueue) + close(jobQueue) + wg.Wait() + + if err != nil { + return fmt.Errorf("failed to ingest wiki dump: %w", err) + } + + return nil +} + +func (e *Engine) Search(query string) []string { + return search.Search(e.idx, query) +} diff --git a/core/index/index.go b/core/index/index.go new file mode 100644 index 0000000..59b315c --- /dev/null +++ b/core/index/index.go @@ -0,0 +1,68 @@ +package index + +import ( + "searchEngine/core/analyzer" + "sync" +) + +type Posting struct { + DocId int + BodyCount int + TitleCount int + BodyPositions []int + TitlePositions []int +} + +type InvertedIndex struct { + Mu sync.RWMutex + Data map[string][]Posting + DocNames map[int]string +} + +func New() *InvertedIndex { + return &InvertedIndex{ + Data: make(map[string][]Posting), + DocNames: make(map[int]string), + } +} + +func (idx *InvertedIndex) Add(docID int, title string, bodyTokens []string) { + titleTokens := analyzer.ProcessText(title) + + type fieldStats struct { + body int + bodyPositions []int + title int + titlePositions []int + } + docStats := make(map[string]fieldStats) + + for position, token := range titleTokens { + stats := docStats[token] + stats.title++ + stats.titlePositions = append(stats.titlePositions, position) + docStats[token] = stats + } + + for position, token := range bodyTokens { + stats := docStats[token] + stats.body++ + stats.bodyPositions = append(stats.bodyPositions, position) + docStats[token] = stats + } + + idx.Mu.Lock() + defer idx.Mu.Unlock() + + idx.DocNames[docID] = title + + for token, stats := range docStats { + idx.Data[token] = append(idx.Data[token], Posting{ + DocId: docID, + BodyCount: stats.body, + TitleCount: stats.title, + BodyPositions: stats.bodyPositions, + TitlePositions: stats.titlePositions, + }) + } +} diff --git a/core/ingest/ingest.go b/core/ingest/ingest.go new file mode 100644 index 0000000..d54ff9d --- /dev/null +++ b/core/ingest/ingest.go @@ -0,0 +1,44 @@ +package ingest + +import ( + "encoding/xml" + "io" + "os" +) + +type Page struct { + Title string `xml:"title"` + Text string `xml:"revision>text"` + Id int `xml:"id"` +} + +func IngestWikiDump(filePath string, jobQueue chan<- Page) error { + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + decoder := xml.NewDecoder(file) + + for { + t, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return err + } + + switch se := t.(type) { + case xml.StartElement: + if se.Name.Local == "page" { + var p Page + if err := decoder.DecodeElement(&p, &se); err == nil { + jobQueue <- p + } + } + } + } + return nil +} diff --git a/core/scoring/field.go b/core/scoring/field.go new file mode 100644 index 0000000..89f6132 --- /dev/null +++ b/core/scoring/field.go @@ -0,0 +1,10 @@ +package scoring + +import "searchEngine/core/index" + +var TitleWeight = 5.0 +var BodyWeight = 1.0 + +func FieldScore(p index.Posting) float64 { + return (float64(p.TitleCount) * TitleWeight) + (float64(p.BodyCount) * BodyWeight) +} diff --git a/core/scoring/idf.go b/core/scoring/idf.go new file mode 100644 index 0000000..f5aaecd --- /dev/null +++ b/core/scoring/idf.go @@ -0,0 +1,17 @@ +package scoring + +import ( + "math" + "searchEngine/core/index" +) + +func CalculateIDF(totalDocs int, docFrequency int) float64 { + if docFrequency == 0 { + return 0.0 + } + return math.Log10(float64(totalDocs) / float64(docFrequency)) +} + +func ScoreTFIDF(p index.Posting, idf float64) float64 { + return FieldScore(p) * idf +} diff --git a/core/search/search.go b/core/search/search.go new file mode 100644 index 0000000..e159d43 --- /dev/null +++ b/core/search/search.go @@ -0,0 +1,72 @@ +package search + +import ( + "searchEngine/core/analyzer" + "searchEngine/core/index" + "slices" +) + +type Result struct { + DocID int + Score int +} + +func Search(idx *index.InvertedIndex, query string) []string { + tokenizedQuery := analyzer.ProcessText(query) + if len(tokenizedQuery) == 0 { + return nil + } + + idx.Mu.RLock() + defer idx.Mu.RUnlock() + + firstToken := tokenizedQuery[0] + basePostings, exists := idx.Data[firstToken] + if !exists { + return nil + } + + scores := make(map[int]int) + for _, p := range basePostings { + scores[p.DocId] = p.BodyCount + } + + for i := 1; i < len(tokenizedQuery); i++ { + token := tokenizedQuery[i] + nextPostings, exists := idx.Data[token] + if !exists { + return nil + } + + lookup := make(map[int]int) + for _, p := range nextPostings { + lookup[p.DocId] = p.BodyCount + } + + newScores := make(map[int]int) + for docID, currentScore := range scores { + if nextCount, found := lookup[docID]; found { + newScores[docID] = currentScore + nextCount + } + } + scores = newScores + } + + var results []Result + for docID, score := range scores { + results = append(results, Result{DocID: docID, Score: score}) + } + + slices.SortFunc(results, func(a, b Result) int { + return b.Score - a.Score + }) + + var titles []string + for _, res := range results { + if title, exists := idx.DocNames[res.DocID]; exists { + titles = append(titles, title) + } + } + + return titles +} diff --git a/index/index.go b/index/index.go deleted file mode 100644 index 60c3d9f..0000000 --- a/index/index.go +++ /dev/null @@ -1,67 +0,0 @@ -package index - -import ( - "searchEngine/analyzer" - "sync" -) - -type Posting struct { - DocId int - BodyCount int - TitleCount int - BodyPositions []int - TitlePositions []int -} -type InvertedIndex struct { - Mu sync.RWMutex - Data map[string][]Posting - DocNames map[int]string -} - -func New() *InvertedIndex { - return &InvertedIndex{ - Data: make(map[string][]Posting), - DocNames: make(map[int]string), - } -} - -func (idx *InvertedIndex) Add(docID int, title string, bodyTokens []string) { - titleTokens := analyzer.ProcessText(title) - - type fieldStats struct { - body int - bodyPositions []int - title int - titlePositions []int - } - docStats := make(map[string]fieldStats) - - for position, token := range titleTokens { - stats := docStats[token] - stats.title++ - stats.titlePositions = append(stats.titlePositions, position) - docStats[token] = stats - } - - for position, token := range bodyTokens { - stats := docStats[token] - stats.body++ - stats.bodyPositions = append(stats.bodyPositions, position) - docStats[token] = stats - } - - idx.Mu.Lock() - defer idx.Mu.Unlock() - - idx.DocNames[docID] = title - - for token, stats := range docStats { - idx.Data[token] = append(idx.Data[token], Posting{ - DocId: docID, - BodyCount: stats.body, - TitleCount: stats.title, - BodyPositions: stats.bodyPositions, - TitlePositions: stats.titlePositions, - }) - } -} diff --git a/ingest/ingest.go b/ingest/ingest.go deleted file mode 100644 index bffc417..0000000 --- a/ingest/ingest.go +++ /dev/null @@ -1,48 +0,0 @@ -package ingest - -import ( - "encoding/xml" - "io" - "os" -) - -type Page struct { - Title string `xml:"title"` - Text string `xml:"revision>text"` - Id int `xml:"id"` -} - -func IngestWikiDump(filePath string, jobQueue chan<- Page) error { - file, err := os.Open(filePath) - if err != nil { - return err - } - defer file.Close() - - // 3. Attach the streaming XML decoder - decoder := xml.NewDecoder(file) - - for { - t, err := decoder.Token() - if err == io.EOF { - break - } - if err != nil { - return err - } - - // Look for the opening tag - switch se := t.(type) { - case xml.StartElement: - if se.Name.Local == "page" { - var p Page - // Decode the entire page element into the struct - if err := decoder.DecodeElement(&p, &se); err == nil { - // Push to your worker pool channel - jobQueue <- p - } - } - } - } - return nil -} diff --git a/main.go b/main.go index 8580668..c2de95e 100644 --- a/main.go +++ b/main.go @@ -1,39 +1,23 @@ -package main +package main import ( "fmt" - "searchEngine/analyzer" - "searchEngine/index" - "searchEngine/ingest" - "searchEngine/search" - "sync" + "searchEngine/core" ) func main() { - myIndex := index.New() - jobQueue := make(chan ingest.Page, 100) + engine := core.NewEngine() - var wg sync.WaitGroup - - for range 4 { - wg.Go(func() { - for page := range jobQueue { - tokens := analyzer.ProcessText(page.Text) - myIndex.Add(page.Id, page.Title, tokens) - } - }) - } - - err := ingest.IngestWikiDump("C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml", jobQueue) + dumpPath := "C:/Users/bambi15/Documents/wikipedia/simplewiki-2026-07-01-p1p1273731/simplewiki-2026-07-01-p1p1273731.xml" + fmt.Println("Ingesting and indexing wiki dump...") + err := engine.IndexDump(dumpPath, 4) if err != nil { panic(err) } - close(jobQueue) - wg.Wait() fmt.Println("Indexing complete! Running a test search...") - results := search.Search(myIndex, "league of legends") + results := engine.Search("league of legends") fmt.Printf("Found %d articles matching your query:\n", len(results)) for i, title := range results { @@ -43,5 +27,4 @@ func main() { } fmt.Printf("- %s\n", title) } - } diff --git a/scoring/field.go b/scoring/field.go deleted file mode 100644 index d84ab3b..0000000 --- a/scoring/field.go +++ /dev/null @@ -1,10 +0,0 @@ -package scoring - -import "searchEngine/index" - -var TitleWeight = 5.0 -var BodyWeight = 1.0 - -func FieldScore(p index.Posting) float64 { - return (float64(p.TitleCount) * TitleWeight) + (float64(p.BodyCount) * BodyWeight) -} diff --git a/scoring/idf.go b/scoring/idf.go deleted file mode 100644 index 5c85ba3..0000000 --- a/scoring/idf.go +++ /dev/null @@ -1,17 +0,0 @@ -package scoring - -import ( - "math" - "searchEngine/index" -) - -func CalculateIDF(totalDocs int, docFrequency int) float64 { - if docFrequency == 0 { - return 0.0 - } - return math.Log10(float64(totalDocs) / float64(docFrequency)) -} -func ScoreTFIDF(p index.Posting, idf float64) float64 { - - return FieldScore(p) * idf -} diff --git a/search/search.go b/search/search.go deleted file mode 100644 index 061c99c..0000000 --- a/search/search.go +++ /dev/null @@ -1,72 +0,0 @@ -package search - -import ( - "searchEngine/analyzer" - "searchEngine/index" - "slices" -) - -type Result struct { - DocID int - Score int -} - -func Search(idx *index.InvertedIndex, query string) []string { - tokenizedQuery := analyzer.ProcessText(query) - if len(tokenizedQuery) == 0 { - return nil - } - - idx.Mu.RLock() - defer idx.Mu.RUnlock() - - firstToken := tokenizedQuery[0] - basePostings, exists := idx.Data[firstToken] - if !exists { - return nil - } - - scores := make(map[int]int) - for _, p := range basePostings { - scores[p.DocId] = p.BodyCount - } - - for i := 1; i < len(tokenizedQuery); i++ { - token := tokenizedQuery[i] - nextPostings, exists := idx.Data[token] - if !exists { - return nil - } - - lookup := make(map[int]int) - for _, p := range nextPostings { - lookup[p.DocId] = p.BodyCount - } - - newScores := make(map[int]int) - for docID, currentScore := range scores { - if nextCount, found := lookup[docID]; found { - newScores[docID] = currentScore + nextCount - } - } - scores = newScores - } - - var results []Result - for docID, score := range scores { - results = append(results, Result{DocID: docID, Score: score}) - } - - slices.SortFunc(results, func(a, b Result) int { - return b.Score - a.Score - }) - - var titles []string - for _, res := range results { - if title, exists := idx.DocNames[res.DocID]; exists { - titles = append(titles, title) - } - } - - return titles -} -- cgit v1.2.3