From c91c3afd8700138f8eb8a25a4ae422c2c9e46cea Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 28 Jul 2026 17:37:06 +0200 Subject: refactored entire code base to be in the package core so i can later implement interfaces to interact with the engine --- core/ingest/ingest.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 core/ingest/ingest.go (limited to 'core/ingest/ingest.go') diff --git a/core/ingest/ingest.go b/core/ingest/ingest.go new file mode 100644 index 0000000..d54ff9d --- /dev/null +++ b/core/ingest/ingest.go @@ -0,0 +1,44 @@ +package ingest + +import ( + "encoding/xml" + "io" + "os" +) + +type Page struct { + Title string `xml:"title"` + Text string `xml:"revision>text"` + Id int `xml:"id"` +} + +func IngestWikiDump(filePath string, jobQueue chan<- Page) error { + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + decoder := xml.NewDecoder(file) + + for { + t, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return err + } + + switch se := t.(type) { + case xml.StartElement: + if se.Name.Local == "page" { + var p Page + if err := decoder.DecodeElement(&p, &se); err == nil { + jobQueue <- p + } + } + } + } + return nil +} -- cgit v1.2.3