Metalink support: v3 (dnf) + v4 (aria2), with client geo override

http/context.go

@@ -23,6 +23,7 @@ const (
 	FILESTATS
 	MIRRORSTATS
 	CHECKSUM
+	METALINK
 
 	UNDEFINED SecureOption = iota
 	WITHTLS
@@ -40,6 +41,8 @@ type Context struct {
 	isMirrorStats bool
 	isFileStats   bool
 	isChecksum    bool
+	isMetalink    bool
+	isMetalink3   bool
 	isPretty      bool
 	secureOption  SecureOption
 }
@@ -60,6 +63,13 @@ func NewContext(w http.ResponseWriter, r *http.Request, t Templates) *Context {
 	} else if c.paramBool("md5") || c.paramBool("sha1") || c.paramBool("sha256") {
 		c.typ = CHECKSUM
 		c.isChecksum = true
+	} else if c.paramBool("meta4") || strings.Contains(strings.ToLower(r.Header.Get("Accept")), "application/metalink4+xml") {
+		c.typ = METALINK
+		c.isMetalink = true
+	} else if c.paramBool("metalink") || strings.Contains(strings.ToLower(r.Header.Get("Accept")), "application/metalink+xml") {
+		// Metalink 3.0 (metalinker.org), the format consumed by dnf/librepo
+		c.typ = METALINK
+		c.isMetalink3 = true
 	} else {
 		c.typ = STANDARD
 	}
@@ -129,6 +139,17 @@ func (c *Context) IsChecksum() bool {
 	return c.isChecksum
 }
 
+// IsMetalink returns true if a Metalink 4 (RFC 5854) document has been requested
+func (c *Context) IsMetalink() bool {
+	return c.isMetalink
+}
+
+// IsMetalink3 returns true if a Metalink 3.0 (metalinker.org) document has been
+// requested. This is the format consumed by dnf/librepo.
+func (c *Context) IsMetalink3() bool {
+	return c.isMetalink3
+}
+
 // IsPretty returns true if the pretty json has been requested
 func (c *Context) IsPretty() bool {
 	return c.isPretty

http/http.go

@@ -210,6 +210,8 @@ func (h *HTTP) requestDispatcher(w http.ResponseWriter, r *http.Request) {
 	switch ctx.Type() {
 	case MIRRORLIST:
 		fallthrough
+	case METALINK:
+		fallthrough
 	case STANDARD:
 		h.mirrorHandler(w, r, ctx)
 	case MIRRORSTATS:
@@ -262,6 +264,27 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex
 
 	clientInfo := h.geoip.GetRecord(remoteIP) //TODO return a pointer?
 
+	// Allow the client to override its detected geolocation. This is mainly
+	// useful for testing (e.g. private IPs that can't be geolocated, as on the
+	// preprod) and to let a client request mirrors for a specific location.
+	// - country/continent drive the country/continent restriction and the
+	//   primary-country tie-break in the selection engine.
+	// - lat/lon set the client coordinates, which is what the distance-based
+	//   ranking actually consumes (a country code alone does not relocate the
+	//   client geographically).
+	if country := ctx.QueryParam("country"); country != "" {
+		clientInfo.CountryCode = strings.ToUpper(country)
+	}
+	if continent := ctx.QueryParam("continent"); continent != "" {
+		clientInfo.ContinentCode = strings.ToUpper(continent)
+	}
+	if v, err := strconv.ParseFloat(ctx.QueryParam("lat"), 32); err == nil {
+		clientInfo.Latitude = float32(v)
+	}
+	if v, err := strconv.ParseFloat(ctx.QueryParam("lon"), 32); err == nil {
+		clientInfo.Longitude = float32(v)
+	}
+
 	mlist, excluded, err := h.engine.Selection(ctx, h.cache, &fileInfo, clientInfo)
 
 	/* Handle errors */
@@ -318,6 +341,10 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex
 
 	if ctx.IsMirrorlist() {
 		resultRenderer = &MirrorListRenderer{}
+	} else if ctx.IsMetalink3() {
+		resultRenderer = &Metalink3Renderer{}
+	} else if ctx.IsMetalink() {
+		resultRenderer = &MetalinkRenderer{}
 	} else {
 		switch GetConfig().OutputMode {
 		case "json":

http/pagerenderer.go

@@ -6,14 +6,18 @@ package http
 import (
 	"bytes"
 	"encoding/json"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	. "github.com/etix/mirrorbits/config"
+	"github.com/etix/mirrorbits/core"
 	"github.com/etix/mirrorbits/mirrors"
 )
 
@@ -100,6 +104,223 @@ func (w *RedirectRenderer) Write(ctx *Context, results *mirrors.Results) (status
 	return http.StatusNotFound, nil
 }
 
+// Metalink 4.0 (RFC 5854) document structures. The XML namespace on the root
+// element produces the required xmlns="urn:ietf:params:xml:ns:metalink".
+type metalink struct {
+	XMLName   xml.Name       `xml:"urn:ietf:params:xml:ns:metalink metalink"`
+	Generator string         `xml:"generator,omitempty"`
+	Published string         `xml:"published,omitempty"`
+	Files     []metalinkFile `xml:"file"`
+}
+
+type metalinkFile struct {
+	Name   string         `xml:"name,attr"`
+	Size   int64          `xml:"size,omitempty"`
+	Hashes []metalinkHash `xml:"hash,omitempty"`
+	URLs   []metalinkURL  `xml:"url"`
+}
+
+type metalinkHash struct {
+	Type  string `xml:"type,attr"`
+	Value string `xml:",chardata"`
+}
+
+type metalinkURL struct {
+	Location string `xml:"location,attr,omitempty"`
+	Priority int    `xml:"priority,attr,omitempty"`
+	Value    string `xml:",chardata"`
+}
+
+// MetalinkRenderer renders a Metalink 4.0 (RFC 5854) document listing the
+// candidate mirrors for the requested file, ordered by preference, so that the
+// client (dnf/librepo, aria2, ...) can perform the final mirror selection and
+// failover itself instead of being redirected to a single mirror.
+type MetalinkRenderer struct{}
+
+// Type returns the type of renderer
+func (w *MetalinkRenderer) Type() string {
+	return "METALINK"
+}
+
+// Write is used to write the result to the ResponseWriter
+func (w *MetalinkRenderer) Write(ctx *Context, results *mirrors.Results) (statusCode int, err error) {
+	if len(results.MirrorList) == 0 {
+		// No mirror returned for this request
+		http.NotFound(ctx.ResponseWriter(), ctx.Request())
+		return http.StatusNotFound, nil
+	}
+
+	// Path of the file relative to the repository root (no leading slash),
+	// used to build the per-mirror URLs.
+	path := strings.TrimPrefix(results.FileInfo.Path, "/")
+
+	file := metalinkFile{
+		// The "name" is the file basename, not the full path. librepo (dnf)
+		// matches it by exact string equality against the name it expects
+		// (e.g. "repomd.xml"), so a full path here would be ignored and the
+		// metalink rejected. This also matches what Fedora's MirrorManager
+		// emits and what aria2 uses as the output filename.
+		Name: filepath.Base(path),
+		Size: results.FileInfo.Size,
+	}
+
+	// Source file hashes, identical across all mirrors. Hash type names follow
+	// the IANA registry as required by RFC 5854.
+	if results.FileInfo.Sha256 != "" {
+		file.Hashes = append(file.Hashes, metalinkHash{Type: "sha-256", Value: results.FileInfo.Sha256})
+	}
+	if results.FileInfo.Sha1 != "" {
+		file.Hashes = append(file.Hashes, metalinkHash{Type: "sha-1", Value: results.FileInfo.Sha1})
+	}
+	if results.FileInfo.Md5 != "" {
+		file.Hashes = append(file.Hashes, metalinkHash{Type: "md5", Value: results.FileInfo.Md5})
+	}
+
+	// Candidate mirrors, already ordered by preference by the selection engine.
+	// priority 1 is the most preferred (RFC 5854 §4.1.6).
+	for i, m := range results.MirrorList {
+		var location string
+		if len(m.CountryFields) > 0 {
+			location = strings.ToLower(m.CountryFields[0])
+		}
+		file.URLs = append(file.URLs, metalinkURL{
+			Location: location,
+			Priority: i + 1,
+			Value:    m.AbsoluteURL + path,
+		})
+	}
+
+	doc := metalink{
+		Generator: "mirrorbits/" + core.VERSION,
+		Published: time.Now().UTC().Format(time.RFC3339),
+		Files:     []metalinkFile{file},
+	}
+
+	output, err := xml.MarshalIndent(doc, "", "  ")
+	if err != nil {
+		return http.StatusInternalServerError, err
+	}
+
+	ctx.ResponseWriter().Header().Set("Content-Type", "application/metalink4+xml; charset=utf-8")
+	ctx.ResponseWriter().Header().Set("Content-Length", strconv.Itoa(len(xml.Header)+len(output)))
+	ctx.ResponseWriter().Write([]byte(xml.Header))
+	ctx.ResponseWriter().Write(output)
+
+	return http.StatusOK, nil
+}
+
+// Metalink 3.0 (metalinker.org) document structures. This is the legacy format
+// consumed by dnf/librepo, which does NOT understand the RFC 5854 (v4) layout:
+// it expects <files>/<file>/<resources>/<url> with a "preference" attribute and
+// hashes wrapped in <verification>. This is what Fedora's MirrorManager serves.
+type metalink3 struct {
+	XMLName   xml.Name        `xml:"http://www.metalinker.org/ metalink"`
+	Version   string          `xml:"version,attr"`
+	Type      string          `xml:"type,attr,omitempty"`
+	Generator string          `xml:"generator,attr,omitempty"`
+	Files     []metalink3File `xml:"files>file"`
+}
+
+type metalink3File struct {
+	Name   string          `xml:"name,attr"`
+	Size   int64           `xml:"size,omitempty"`
+	Hashes []metalink3Hash `xml:"verification>hash,omitempty"`
+	URLs   []metalink3URL  `xml:"resources>url"`
+}
+
+type metalink3Hash struct {
+	Type  string `xml:"type,attr"`
+	Value string `xml:",chardata"`
+}
+
+type metalink3URL struct {
+	Protocol   string `xml:"protocol,attr,omitempty"`
+	Type       string `xml:"type,attr,omitempty"`
+	Location   string `xml:"location,attr,omitempty"`
+	Preference int    `xml:"preference,attr,omitempty"`
+	Value      string `xml:",chardata"`
+}
+
+// Metalink3Renderer renders a Metalink 3.0 document. This is the format dnf
+// expects in a repo's metalink= directive (the url= entries point at each
+// mirror's repomd.xml; the client derives the per-mirror baseurl from them and
+// performs the final selection/failover itself).
+type Metalink3Renderer struct{}
+
+// Type returns the type of renderer
+func (w *Metalink3Renderer) Type() string {
+	return "METALINK3"
+}
+
+// Write is used to write the result to the ResponseWriter
+func (w *Metalink3Renderer) Write(ctx *Context, results *mirrors.Results) (statusCode int, err error) {
+	if len(results.MirrorList) == 0 {
+		http.NotFound(ctx.ResponseWriter(), ctx.Request())
+		return http.StatusNotFound, nil
+	}
+
+	path := strings.TrimPrefix(results.FileInfo.Path, "/")
+
+	file := metalink3File{
+		// Basename only: librepo matches it by exact equality (e.g. "repomd.xml")
+		Name: filepath.Base(path),
+		Size: results.FileInfo.Size,
+	}
+
+	// Metalink 3.0 uses dashless hash type names inside <verification>.
+	if results.FileInfo.Sha256 != "" {
+		file.Hashes = append(file.Hashes, metalink3Hash{Type: "sha256", Value: results.FileInfo.Sha256})
+	}
+	if results.FileInfo.Sha1 != "" {
+		file.Hashes = append(file.Hashes, metalink3Hash{Type: "sha1", Value: results.FileInfo.Sha1})
+	}
+	if results.FileInfo.Md5 != "" {
+		file.Hashes = append(file.Hashes, metalink3Hash{Type: "md5", Value: results.FileInfo.Md5})
+	}
+
+	// Candidate mirrors. In Metalink 3.0 "preference" is 0-100, higher is more
+	// preferred (the opposite of v4's "priority"), so map the rank accordingly.
+	for i, m := range results.MirrorList {
+		var location string
+		if len(m.CountryFields) > 0 {
+			location = strings.ToLower(m.CountryFields[0])
+		}
+		proto := "http"
+		if strings.HasPrefix(m.AbsoluteURL, "https://") {
+			proto = "https"
+		}
+		preference := 100 - i
+		if preference < 1 {
+			preference = 1
+		}
+		file.URLs = append(file.URLs, metalink3URL{
+			Protocol:   proto,
+			Type:       proto,
+			Location:   location,
+			Preference: preference,
+			Value:      m.AbsoluteURL + path,
+		})
+	}
+
+	doc := metalink3{
+		Version:   "3.0",
+		Generator: "mirrorbits/" + core.VERSION,
+		Files:     []metalink3File{file},
+	}
+
+	output, err := xml.MarshalIndent(doc, "", "  ")
+	if err != nil {
+		return http.StatusInternalServerError, err
+	}
+
+	ctx.ResponseWriter().Header().Set("Content-Type", "application/metalink+xml; charset=utf-8")
+	ctx.ResponseWriter().Header().Set("Content-Length", strconv.Itoa(len(xml.Header)+len(output)))
+	ctx.ResponseWriter().Write([]byte(xml.Header))
+	ctx.ResponseWriter().Write(output)
+
+	return http.StatusOK, nil
+}
+
 // MirrorListRenderer is used to render the mirrorlist page using the HTML templates
 type MirrorListRenderer struct{}
 

http/selection.go

@@ -57,8 +57,10 @@ func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *f
 			mlist[i], mlist[j] = mlist[j], mlist[i]
 		}
 
-		// Shortcut
-		if !ctx.IsMirrorlist() {
+		// Shortcut: the redirect/json path only needs a handful of mirrors,
+		// but mirrorlist and metalink want the full candidate list so the
+		// client can fail over across all of them.
+		if !ctx.IsMirrorlist() && !ctx.IsMetalink() && !ctx.IsMetalink3() {
 			// Reduce the number of mirrors to process
 			mlist = mlist[:utils.Min(5, len(mlist))]
 		}
@@ -121,7 +123,7 @@ func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *f
 
 	if selected > 1 {
 
-		if ctx.IsMirrorlist() {
+		if ctx.IsMirrorlist() || ctx.IsMetalink() || ctx.IsMetalink3() {
 			// Don't reorder the results, just set the percentage
 			for i := 0; i < selected; i++ {
 				id := mlist[i].ID