From 04b2f5344e2342cee0d584a9b9cbdfd33a46aeb8 Mon Sep 17 00:00:00 2001 From: ashledombos Date: Tue, 30 Jun 2026 21:57:11 +0000 Subject: [PATCH 1/3] http: add Metalink 4 (RFC 5854) renderer + client geo override Serve a Metalink 4.0 document (.meta4 / application/metalink4+xml) listing the candidate mirrors for a file, ordered by preference, with the source file hashes. This enables client-side mirror selection and failover (dnf/librepo, aria2, ...) instead of the single-target 302. - New MetalinkRenderer producing the RFC 5854 XML (generator, published, file/size/hash, url with location + priority). - Triggered by the ?meta4 query param or an Accept: application/metalink4+xml header (new METALINK request type). - Hash types follow the IANA registry (sha-256/sha-1/md5); only the hashes actually computed for the file are emitted. - Client geolocation override (mainly for testing on private IPs): ?country / ?continent feed the restriction and primary-country logic, ?lat / ?lon set the client coordinates consumed by the distance-based ranking. Refs #49 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01QhiB9FMKecvFyETJRYY8Nb --- http/context.go | 10 +++++ http/http.go | 25 +++++++++++ http/pagerenderer.go | 102 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/http/context.go b/http/context.go index 5b923bc7..da5568f3 100644 --- a/http/context.go +++ b/http/context.go @@ -23,6 +23,7 @@ const ( FILESTATS MIRRORSTATS CHECKSUM + METALINK UNDEFINED SecureOption = iota WITHTLS @@ -40,6 +41,7 @@ type Context struct { isMirrorStats bool isFileStats bool isChecksum bool + isMetalink bool isPretty bool secureOption SecureOption } @@ -60,6 +62,9 @@ func NewContext(w http.ResponseWriter, r *http.Request, t Templates) *Context { } else if c.paramBool("md5") || c.paramBool("sha1") || c.paramBool("sha256") { c.typ = CHECKSUM c.isChecksum = true + } else if c.paramBool("meta4") || strings.Contains(strings.ToLower(r.Header.Get("Accept")), "application/metalink4+xml") { + c.typ = METALINK + c.isMetalink = true } else { c.typ = STANDARD } @@ -129,6 +134,11 @@ func (c *Context) IsChecksum() bool { return c.isChecksum } +// IsMetalink returns true if a Metalink 4 document has been requested +func (c *Context) IsMetalink() bool { + return c.isMetalink +} + // IsPretty returns true if the pretty json has been requested func (c *Context) IsPretty() bool { return c.isPretty diff --git a/http/http.go b/http/http.go index 4b970c63..e76dac62 100644 --- a/http/http.go +++ b/http/http.go @@ -210,6 +210,8 @@ func (h *HTTP) requestDispatcher(w http.ResponseWriter, r *http.Request) { switch ctx.Type() { case MIRRORLIST: fallthrough + case METALINK: + fallthrough case STANDARD: h.mirrorHandler(w, r, ctx) case MIRRORSTATS: @@ -262,6 +264,27 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex clientInfo := h.geoip.GetRecord(remoteIP) //TODO return a pointer? + // Allow the client to override its detected geolocation. This is mainly + // useful for testing (e.g. private IPs that can't be geolocated, as on the + // preprod) and to let a client request mirrors for a specific location. + // - country/continent drive the country/continent restriction and the + // primary-country tie-break in the selection engine. + // - lat/lon set the client coordinates, which is what the distance-based + // ranking actually consumes (a country code alone does not relocate the + // client geographically). + if country := ctx.QueryParam("country"); country != "" { + clientInfo.CountryCode = strings.ToUpper(country) + } + if continent := ctx.QueryParam("continent"); continent != "" { + clientInfo.ContinentCode = strings.ToUpper(continent) + } + if v, err := strconv.ParseFloat(ctx.QueryParam("lat"), 32); err == nil { + clientInfo.Latitude = float32(v) + } + if v, err := strconv.ParseFloat(ctx.QueryParam("lon"), 32); err == nil { + clientInfo.Longitude = float32(v) + } + mlist, excluded, err := h.engine.Selection(ctx, h.cache, &fileInfo, clientInfo) /* Handle errors */ @@ -318,6 +341,8 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex if ctx.IsMirrorlist() { resultRenderer = &MirrorListRenderer{} + } else if ctx.IsMetalink() { + resultRenderer = &MetalinkRenderer{} } else { switch GetConfig().OutputMode { case "json": diff --git a/http/pagerenderer.go b/http/pagerenderer.go index 44f3f05f..31e9666c 100644 --- a/http/pagerenderer.go +++ b/http/pagerenderer.go @@ -6,14 +6,17 @@ package http import ( "bytes" "encoding/json" + "encoding/xml" "errors" "fmt" "net/http" "sort" "strconv" "strings" + "time" . "github.com/etix/mirrorbits/config" + "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/mirrors" ) @@ -100,6 +103,105 @@ func (w *RedirectRenderer) Write(ctx *Context, results *mirrors.Results) (status return http.StatusNotFound, nil } +// Metalink 4.0 (RFC 5854) document structures. The XML namespace on the root +// element produces the required xmlns="urn:ietf:params:xml:ns:metalink". +type metalink struct { + XMLName xml.Name `xml:"urn:ietf:params:xml:ns:metalink metalink"` + Generator string `xml:"generator,omitempty"` + Published string `xml:"published,omitempty"` + Files []metalinkFile `xml:"file"` +} + +type metalinkFile struct { + Name string `xml:"name,attr"` + Size int64 `xml:"size,omitempty"` + Hashes []metalinkHash `xml:"hash,omitempty"` + URLs []metalinkURL `xml:"url"` +} + +type metalinkHash struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` +} + +type metalinkURL struct { + Location string `xml:"location,attr,omitempty"` + Priority int `xml:"priority,attr,omitempty"` + Value string `xml:",chardata"` +} + +// MetalinkRenderer renders a Metalink 4.0 (RFC 5854) document listing the +// candidate mirrors for the requested file, ordered by preference, so that the +// client (dnf/librepo, aria2, ...) can perform the final mirror selection and +// failover itself instead of being redirected to a single mirror. +type MetalinkRenderer struct{} + +// Type returns the type of renderer +func (w *MetalinkRenderer) Type() string { + return "METALINK" +} + +// Write is used to write the result to the ResponseWriter +func (w *MetalinkRenderer) Write(ctx *Context, results *mirrors.Results) (statusCode int, err error) { + if len(results.MirrorList) == 0 { + // No mirror returned for this request + http.NotFound(ctx.ResponseWriter(), ctx.Request()) + return http.StatusNotFound, nil + } + + // The Metalink "name" must be a relative path (no leading slash) + path := strings.TrimPrefix(results.FileInfo.Path, "/") + + file := metalinkFile{ + Name: path, + Size: results.FileInfo.Size, + } + + // Source file hashes, identical across all mirrors. Hash type names follow + // the IANA registry as required by RFC 5854. + if results.FileInfo.Sha256 != "" { + file.Hashes = append(file.Hashes, metalinkHash{Type: "sha-256", Value: results.FileInfo.Sha256}) + } + if results.FileInfo.Sha1 != "" { + file.Hashes = append(file.Hashes, metalinkHash{Type: "sha-1", Value: results.FileInfo.Sha1}) + } + if results.FileInfo.Md5 != "" { + file.Hashes = append(file.Hashes, metalinkHash{Type: "md5", Value: results.FileInfo.Md5}) + } + + // Candidate mirrors, already ordered by preference by the selection engine. + // priority 1 is the most preferred (RFC 5854 ยง4.1.6). + for i, m := range results.MirrorList { + var location string + if len(m.CountryFields) > 0 { + location = strings.ToLower(m.CountryFields[0]) + } + file.URLs = append(file.URLs, metalinkURL{ + Location: location, + Priority: i + 1, + Value: m.AbsoluteURL + path, + }) + } + + doc := metalink{ + Generator: "mirrorbits/" + core.VERSION, + Published: time.Now().UTC().Format(time.RFC3339), + Files: []metalinkFile{file}, + } + + output, err := xml.MarshalIndent(doc, "", " ") + if err != nil { + return http.StatusInternalServerError, err + } + + ctx.ResponseWriter().Header().Set("Content-Type", "application/metalink4+xml; charset=utf-8") + ctx.ResponseWriter().Header().Set("Content-Length", strconv.Itoa(len(xml.Header)+len(output))) + ctx.ResponseWriter().Write([]byte(xml.Header)) + ctx.ResponseWriter().Write(output) + + return http.StatusOK, nil +} + // MirrorListRenderer is used to render the mirrorlist page using the HTML templates type MirrorListRenderer struct{} From dc9fb53251271754c387f2b0f1912ac43500e59d Mon Sep 17 00:00:00 2001 From: ashledombos Date: Tue, 30 Jun 2026 22:24:39 +0000 Subject: [PATCH 2/3] http: serve the full candidate list for metalink requests Metalink documents are meant to let the client fail over across mirrors, so don't trim the candidate list down to 5 (as done for the redirect/json path) and don't randomize the order: treat metalink like mirrorlist, i.e. return the full list deterministically ordered by score. The redirect and JSON paths are unchanged. Refs #49 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01QhiB9FMKecvFyETJRYY8Nb --- http/selection.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/http/selection.go b/http/selection.go index ad312e55..d5354890 100644 --- a/http/selection.go +++ b/http/selection.go @@ -57,8 +57,10 @@ func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *f mlist[i], mlist[j] = mlist[j], mlist[i] } - // Shortcut - if !ctx.IsMirrorlist() { + // Shortcut: the redirect/json path only needs a handful of mirrors, + // but mirrorlist and metalink want the full candidate list so the + // client can fail over across all of them. + if !ctx.IsMirrorlist() && !ctx.IsMetalink() { // Reduce the number of mirrors to process mlist = mlist[:utils.Min(5, len(mlist))] } @@ -121,7 +123,7 @@ func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *f if selected > 1 { - if ctx.IsMirrorlist() { + if ctx.IsMirrorlist() || ctx.IsMetalink() { // Don't reorder the results, just set the percentage for i := 0; i < selected; i++ { id := mlist[i].ID From a9f1c6052e172e7e8f523591d5c058bde9a93630 Mon Sep 17 00:00:00 2001 From: ashledombos Date: Tue, 30 Jun 2026 22:47:49 +0000 Subject: [PATCH 3/3] http: add Metalink 3.0 renderer for dnf, and use basename for file name Real-world testing showed dnf/librepo cannot consume the RFC 5854 (v4) document: librepo only parses Metalink 3.0 (metalinker.org), i.e. /// with hashes wrapped in and dashless hash type names. This is the format Fedora's MirrorManager actually serves. - New Metalink3Renderer (triggered by ?metalink or Accept: application/metalink+xml) producing the v3 layout dnf expects. The v4 renderer (?meta4) is kept for aria2/wget2 and standard compliance. - Fix the to the basename (e.g. "repomd.xml") instead of the full path: librepo matches it by exact string equality, so a full path was silently rejected. Applies to both v3 and v4. - Treat metalink (v3 and v4) like mirrorlist in the selection engine so the full candidate list is returned. Validated end to end: `dnf makecache` via metalink= succeeds (exit 0, "Metadata cache created") on both Fedora dnf5 and OpenMandriva (cooker, dnf5 5.2.16), fetching repomd.xml straight from a mirror. Refs #49 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01QhiB9FMKecvFyETJRYY8Nb --- http/context.go | 13 ++++- http/http.go | 2 + http/pagerenderer.go | 123 ++++++++++++++++++++++++++++++++++++++++++- http/selection.go | 4 +- 4 files changed, 137 insertions(+), 5 deletions(-) diff --git a/http/context.go b/http/context.go index da5568f3..b57730ee 100644 --- a/http/context.go +++ b/http/context.go @@ -42,6 +42,7 @@ type Context struct { isFileStats bool isChecksum bool isMetalink bool + isMetalink3 bool isPretty bool secureOption SecureOption } @@ -65,6 +66,10 @@ func NewContext(w http.ResponseWriter, r *http.Request, t Templates) *Context { } else if c.paramBool("meta4") || strings.Contains(strings.ToLower(r.Header.Get("Accept")), "application/metalink4+xml") { c.typ = METALINK c.isMetalink = true + } else if c.paramBool("metalink") || strings.Contains(strings.ToLower(r.Header.Get("Accept")), "application/metalink+xml") { + // Metalink 3.0 (metalinker.org), the format consumed by dnf/librepo + c.typ = METALINK + c.isMetalink3 = true } else { c.typ = STANDARD } @@ -134,11 +139,17 @@ func (c *Context) IsChecksum() bool { return c.isChecksum } -// IsMetalink returns true if a Metalink 4 document has been requested +// IsMetalink returns true if a Metalink 4 (RFC 5854) document has been requested func (c *Context) IsMetalink() bool { return c.isMetalink } +// IsMetalink3 returns true if a Metalink 3.0 (metalinker.org) document has been +// requested. This is the format consumed by dnf/librepo. +func (c *Context) IsMetalink3() bool { + return c.isMetalink3 +} + // IsPretty returns true if the pretty json has been requested func (c *Context) IsPretty() bool { return c.isPretty diff --git a/http/http.go b/http/http.go index e76dac62..a739d04d 100644 --- a/http/http.go +++ b/http/http.go @@ -341,6 +341,8 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex if ctx.IsMirrorlist() { resultRenderer = &MirrorListRenderer{} + } else if ctx.IsMetalink3() { + resultRenderer = &Metalink3Renderer{} } else if ctx.IsMetalink() { resultRenderer = &MetalinkRenderer{} } else { diff --git a/http/pagerenderer.go b/http/pagerenderer.go index 31e9666c..50187c5a 100644 --- a/http/pagerenderer.go +++ b/http/pagerenderer.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "sort" "strconv" "strings" @@ -149,11 +150,17 @@ func (w *MetalinkRenderer) Write(ctx *Context, results *mirrors.Results) (status return http.StatusNotFound, nil } - // The Metalink "name" must be a relative path (no leading slash) + // Path of the file relative to the repository root (no leading slash), + // used to build the per-mirror URLs. path := strings.TrimPrefix(results.FileInfo.Path, "/") file := metalinkFile{ - Name: path, + // The "name" is the file basename, not the full path. librepo (dnf) + // matches it by exact string equality against the name it expects + // (e.g. "repomd.xml"), so a full path here would be ignored and the + // metalink rejected. This also matches what Fedora's MirrorManager + // emits and what aria2 uses as the output filename. + Name: filepath.Base(path), Size: results.FileInfo.Size, } @@ -202,6 +209,118 @@ func (w *MetalinkRenderer) Write(ctx *Context, results *mirrors.Results) (status return http.StatusOK, nil } +// Metalink 3.0 (metalinker.org) document structures. This is the legacy format +// consumed by dnf/librepo, which does NOT understand the RFC 5854 (v4) layout: +// it expects /// with a "preference" attribute and +// hashes wrapped in . This is what Fedora's MirrorManager serves. +type metalink3 struct { + XMLName xml.Name `xml:"http://www.metalinker.org/ metalink"` + Version string `xml:"version,attr"` + Type string `xml:"type,attr,omitempty"` + Generator string `xml:"generator,attr,omitempty"` + Files []metalink3File `xml:"files>file"` +} + +type metalink3File struct { + Name string `xml:"name,attr"` + Size int64 `xml:"size,omitempty"` + Hashes []metalink3Hash `xml:"verification>hash,omitempty"` + URLs []metalink3URL `xml:"resources>url"` +} + +type metalink3Hash struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` +} + +type metalink3URL struct { + Protocol string `xml:"protocol,attr,omitempty"` + Type string `xml:"type,attr,omitempty"` + Location string `xml:"location,attr,omitempty"` + Preference int `xml:"preference,attr,omitempty"` + Value string `xml:",chardata"` +} + +// Metalink3Renderer renders a Metalink 3.0 document. This is the format dnf +// expects in a repo's metalink= directive (the url= entries point at each +// mirror's repomd.xml; the client derives the per-mirror baseurl from them and +// performs the final selection/failover itself). +type Metalink3Renderer struct{} + +// Type returns the type of renderer +func (w *Metalink3Renderer) Type() string { + return "METALINK3" +} + +// Write is used to write the result to the ResponseWriter +func (w *Metalink3Renderer) Write(ctx *Context, results *mirrors.Results) (statusCode int, err error) { + if len(results.MirrorList) == 0 { + http.NotFound(ctx.ResponseWriter(), ctx.Request()) + return http.StatusNotFound, nil + } + + path := strings.TrimPrefix(results.FileInfo.Path, "/") + + file := metalink3File{ + // Basename only: librepo matches it by exact equality (e.g. "repomd.xml") + Name: filepath.Base(path), + Size: results.FileInfo.Size, + } + + // Metalink 3.0 uses dashless hash type names inside . + if results.FileInfo.Sha256 != "" { + file.Hashes = append(file.Hashes, metalink3Hash{Type: "sha256", Value: results.FileInfo.Sha256}) + } + if results.FileInfo.Sha1 != "" { + file.Hashes = append(file.Hashes, metalink3Hash{Type: "sha1", Value: results.FileInfo.Sha1}) + } + if results.FileInfo.Md5 != "" { + file.Hashes = append(file.Hashes, metalink3Hash{Type: "md5", Value: results.FileInfo.Md5}) + } + + // Candidate mirrors. In Metalink 3.0 "preference" is 0-100, higher is more + // preferred (the opposite of v4's "priority"), so map the rank accordingly. + for i, m := range results.MirrorList { + var location string + if len(m.CountryFields) > 0 { + location = strings.ToLower(m.CountryFields[0]) + } + proto := "http" + if strings.HasPrefix(m.AbsoluteURL, "https://") { + proto = "https" + } + preference := 100 - i + if preference < 1 { + preference = 1 + } + file.URLs = append(file.URLs, metalink3URL{ + Protocol: proto, + Type: proto, + Location: location, + Preference: preference, + Value: m.AbsoluteURL + path, + }) + } + + doc := metalink3{ + Version: "3.0", + Generator: "mirrorbits/" + core.VERSION, + Files: []metalink3File{file}, + } + + output, err := xml.MarshalIndent(doc, "", " ") + if err != nil { + return http.StatusInternalServerError, err + } + + ctx.ResponseWriter().Header().Set("Content-Type", "application/metalink+xml; charset=utf-8") + ctx.ResponseWriter().Header().Set("Content-Length", strconv.Itoa(len(xml.Header)+len(output))) + ctx.ResponseWriter().Write([]byte(xml.Header)) + ctx.ResponseWriter().Write(output) + + return http.StatusOK, nil +} + // MirrorListRenderer is used to render the mirrorlist page using the HTML templates type MirrorListRenderer struct{} diff --git a/http/selection.go b/http/selection.go index d5354890..3a7c5786 100644 --- a/http/selection.go +++ b/http/selection.go @@ -60,7 +60,7 @@ func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *f // Shortcut: the redirect/json path only needs a handful of mirrors, // but mirrorlist and metalink want the full candidate list so the // client can fail over across all of them. - if !ctx.IsMirrorlist() && !ctx.IsMetalink() { + if !ctx.IsMirrorlist() && !ctx.IsMetalink() && !ctx.IsMetalink3() { // Reduce the number of mirrors to process mlist = mlist[:utils.Min(5, len(mlist))] } @@ -123,7 +123,7 @@ func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *f if selected > 1 { - if ctx.IsMirrorlist() || ctx.IsMetalink() { + if ctx.IsMirrorlist() || ctx.IsMetalink() || ctx.IsMetalink3() { // Don't reorder the results, just set the percentage for i := 0; i < selected; i++ { id := mlist[i].ID