Restore robots.txt and public homepage sitemap

src/msg/transports/http_routes.py

@@ -6,6 +6,7 @@
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from enum import Enum, StrEnum
+from html import escape
 from importlib.resources import files
 from urllib.parse import quote, unquote_to_bytes, urlencode, urlsplit
 
@@ -1783,6 +1784,29 @@ async def dispatch(request: Request):
             if path == '/-' or path.startswith('/-/'):
                 raise Failure('not_found')
             require(request.method in {'GET', 'HEAD'}, 'method_not_allowed')
+            if path in {'/robots.txt', '/sitemap.xml'}:
+                require(not request.url.query, 'unknown_query_parameter')
+                origin = service.settings.service_url.rstrip('/')
+                if path == '/robots.txt':
+                    payload = (
+                        f'User-agent: *\nAllow: /\nDisallow: /-/\nSitemap: {origin}/sitemap.xml\n'
+                    ).encode()
+                    media_type = 'text/plain'
+                else:
+                    # Only the public homepage is unconditional. Resource paths,
+                    # including rules, can change their ACL or be removed.
+                    payload = (
+                        '<?xml version="1.0" encoding="UTF-8"?>\n'
+                        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
+                        f'  <url><loc>{escape(origin + "/")}</loc></url>\n'
+                        '</urlset>\n'
+                    ).encode()
+                    media_type = 'application/xml'
+                return Response(
+                    b'' if request.method == 'HEAD' else payload,
+                    media_type=media_type,
+                    headers={**BASE_HEADERS, 'Content-Length': str(len(payload))},
+                )
             if path == '/favicon.png':
                 headers = {**BASE_HEADERS, 'Content-Length': str(len(HOME_FAVICON))}
                 return Response(

tests/test_search_engine_discovery.py

@@ -0,0 +1,44 @@
+"""Crawler discovery advertises only unconditional public URLs."""
+
+from xml.etree import ElementTree as ET
+
+import httpx
+import pytest
+
+from msg.transports.http import create_app
+
+
+@pytest.mark.asyncio
+async def test_crawler_discovery_is_public_read_only_and_uses_configured_origin(installed):
+    app, _ = installed
+    origin = app.settings.service_url.rstrip('/')
+    async with httpx.AsyncClient(
+        transport=httpx.ASGITransport(app=create_app(app)), base_url=origin
+    ) as http:
+        robots = await http.get('/robots.txt')
+        assert robots.status_code == 200
+        assert robots.headers['content-type'].startswith('text/plain')
+        assert 'User-agent: *\nAllow: /\nDisallow: /-/\n' in robots.text
+        assert f'Sitemap: {origin}/sitemap.xml\n' in robots.text
+
+        sitemap = await http.get('/sitemap.xml')
+        assert sitemap.status_code == 200
+        assert sitemap.headers['content-type'].startswith('application/xml')
+        document = ET.fromstring(sitemap.content)
+        namespace = {'s': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
+        locations = [item.text for item in document.findall('s:url/s:loc', namespace)]
+        # No resource names, account paths, operation URLs or private metadata.
+        assert locations == [origin + '/']
+        assert (await http.get(locations[0])).status_code == 200
+
+        for path, response in (('/robots.txt', robots), ('/sitemap.xml', sitemap)):
+            head = await http.head(path)
+            assert head.status_code == 200 and head.content == b''
+            assert head.headers['content-length'] == str(len(response.content))
+            assert head.headers['content-type'] == response.headers['content-type']
+            assert response.headers['cache-control'] == 'no-store'
+            assert response.headers['x-content-type-options'] == 'nosniff'
+            assert 'sandbox' in response.headers['content-security-policy']
+            assert (await http.post(path, content=b'overwrite')).status_code == 405
+            assert (await http.get(path + '?extra=1')).status_code == 400
+            assert (await http.get(path, headers={'Host': 'evil.example'})).status_code == 403