init commit, basic tokenizing and indexing logic without any ranking of results on a simple wikipedia dump

1 files changed, 48 insertions, 0 deletions

diff --git a/ingest/ingest.go b/ingest/ingest.go
new file mode 100644
index 0000000..bffc417
--- /dev/null
+++ b/ingest/ingest.go
@@ -0,0 +1,48 @@
+package ingest
+
+import (
+	"encoding/xml"
+	"io"
+	"os"
+)
+
+type Page struct {
+	Title string `xml:"title"`
+	Text  string `xml:"revision>text"`
+	Id    int    `xml:"id"`
+}
+
+func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// 3. Attach the streaming XML decoder
+	decoder := xml.NewDecoder(file)
+
+	for {
+		t, err := decoder.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		// Look for the opening <page> tag
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "page" {
+				var p Page
+				// Decode the entire page element into the struct
+				if err := decoder.DecodeElement(&p, &se); err == nil {
+					// Push to your worker pool channel
+					jobQueue <- p
+				}
+			}
+		}
+	}
+	return nil
+}