refactored entire code base to be in the package core so i can later implement interfaces to interact with the engine

1 files changed, 44 insertions, 0 deletions

diff --git a/core/ingest/ingest.go b/core/ingest/ingest.go
new file mode 100644
index 0000000..d54ff9d
--- /dev/null
+++ b/core/ingest/ingest.go
@@ -0,0 +1,44 @@
+package ingest
+
+import (
+	"encoding/xml"
+	"io"
+	"os"
+)
+
+type Page struct {
+	Title string `xml:"title"`
+	Text  string `xml:"revision>text"`
+	Id    int    `xml:"id"`
+}
+
+func IngestWikiDump(filePath string, jobQueue chan<- Page) error {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	decoder := xml.NewDecoder(file)
+
+	for {
+		t, err := decoder.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		switch se := t.(type) {
+		case xml.StartElement:
+			if se.Name.Local == "page" {
+				var p Page
+				if err := decoder.DecodeElement(&p, &se); err == nil {
+					jobQueue <- p
+				}
+			}
+		}
+	}
+	return nil
+}