From 3038483c570af7b923dda39e857b9cab8f89e5dd Mon Sep 17 00:00:00 2001
From: alex <alexander.vrabel@proton.me>
Date: Mon, 27 Jul 2026 23:55:14 +0200
Subject: init commit, basic tokenizing and indexing logic without any ranking
 of results on a simple wikipedia dump

---
 ingest/ingest.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 ingest/ingest.go

(limited to 'ingest')

diff --git a/ingest/ingest.go b/ingest/ingest.go
new file mode 100644
index 0000000..bffc417
--- /dev/null
+++ b/ingest/ingest.go
@@ -0,0 +1,48 @@
+package ingest
+
+import (
+	"encoding/xml"
+	"io"
+	"os"
+)
+
+type Page struct {
+	Title string `xml:"title"`
+	Text  string `xml:"revision>text"`
+	Id    int    `xml:"id"`
+}
+
+func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// 3. Attach the streaming XML decoder
+	decoder := xml.NewDecoder(file)
+
+	for {
+		t, err := decoder.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		// Look for the opening <page> tag
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "page" {
+				var p Page
+				// Decode the entire page element into the struct
+				if err := decoder.DecodeElement(&p, &se); err == nil {
+					// Push to your worker pool channel
+					jobQueue <- p
+				}
+			}
+		}
+	}
+	return nil
+}
-- 
cgit v1.2.3