Skip to content

Commit ab51413

Browse files
committed
feat: serve PyPI and Simple JSON from the content app
Add PythonDistribution.content_handler_json so the content app returns the same PyPI JSON and PEP 691 Simple JSON as the /pypi/ API when Accept prefers JSON. Package artifacts stay binary, and Simple JSON uses the official media type so pip can consume it. "Assisted-By: Cursor"
1 parent 5b76249 commit ab51413

8 files changed

Lines changed: 500 additions & 9 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added `PythonDistribution.content_handler_json` so the content app can serve PyPI JSON and PEP 691 Simple JSON when the client prefers `application/json`. Requires a pulpcore that includes `Distribution.content_handler_json` (issue 7887).

‎docs/user/guides/host.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,22 @@ pip install --trusted-host localhost shelf-reader
103103

104104
See the [pip docs](https://pip.pypa.io/en/stable/topics/configuration) for more details.
105105

106+
## JSON from the content app
107+
108+
When pulpcore's content app is asked for JSON (`Accept: application/json`), Python distributions
109+
serve the same PyPI JSON and PEP 691 Simple JSON already available from the `/pypi/` API. Clients
110+
do not need the `/json` URL suffix or the PyPI Simple media types:
111+
112+
```bash
113+
http "${CONTENT_ORIGIN}${CONTENT_PATH_PREFIX}foo/simple/" Accept:application/json
114+
http "${CONTENT_ORIGIN}${CONTENT_PATH_PREFIX}foo/simple/shelf-reader/" Accept:application/json
115+
http "${CONTENT_ORIGIN}${CONTENT_PATH_PREFIX}foo/pypi/shelf-reader/" Accept:application/json
116+
```
117+
118+
The distribution root still returns pulpcore's generic file listing. Package artifacts remain binary
119+
regardless of `Accept`. This requires a pulpcore that includes `Distribution.content_handler_json`
120+
(pulpcore issue 7887).
121+
106122

107123
## Migrating off Publications
108124

‎pulp_python/app/models.py‎

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from django.contrib.postgres.fields import ArrayField
99
from django.core.exceptions import ObjectDoesNotExist
1010
from django.db import models
11+
from django.db.models import F, FilteredRelation, Q
1112
from django_lifecycle import (
1213
BEFORE_SAVE,
1314
hook,
@@ -36,10 +37,14 @@
3637
from .utils import (
3738
PYPI_LAST_SERIAL,
3839
PYPI_SERIAL_CONSTANT,
40+
PYPI_SIMPLE_V1_JSON,
3941
artifact_to_metadata_artifact,
4042
artifact_to_python_content_data,
43+
build_content_url,
4144
canonicalize_name,
4245
python_content_to_json,
46+
write_simple_detail_json,
47+
write_simple_index_json,
4348
)
4449

4550
log = getLogger(__name__)
@@ -126,7 +131,7 @@ def content_handler(self, path):
126131
pk__in=self.publication.repository_version.content, name_normalized=normalized
127132
)
128133
# TODO Change this value to the Repo's serial value when implemented
129-
headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
134+
headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"}
130135
if not settings.DOMAIN_ENABLED:
131136
domain = None
132137
json_body = python_content_to_json(
@@ -137,6 +142,111 @@ def content_handler(self, path):
137142

138143
return None
139144

145+
def content_handler_json(self, path):
146+
"""
147+
Handler to serve a JSON representation of the content at ``path`` for this Distribution.
148+
149+
Called by pulpcore's content app when a client's ``Accept`` header prefers JSON,
150+
instead of pulpcore's generic recursive file listing. This reuses the same
151+
serialization already used by the ``pypi/*/json`` and ``pypi/simple`` PyPI APIs, so
152+
requests made directly against the content app (e.g. by a UI or ``pip``-compatible
153+
tool) get the same structured data without needing to know those separate URL
154+
conventions.
155+
156+
Args:
157+
path (str): The path being requested
158+
Returns:
159+
None if there is no JSON representation to serve at path. Otherwise a
160+
JSON-serializable dict, or an aiohttp.web.Response for full header control.
161+
"""
162+
path = PurePath(path)
163+
parts = path.parts
164+
if not parts:
165+
# Distro root: let pulpcore's generic recursive listing handle it.
166+
return None
167+
168+
_, repo_version, _ = self.get_repository_publication_and_version()
169+
if repo_version is None:
170+
return None
171+
content = PythonPackageContent.objects.filter(pk__in=repo_version.content)
172+
domain = get_domain() if settings.DOMAIN_ENABLED else None
173+
174+
if parts[0] == "pypi" and 2 <= len(parts) <= 3 and parts[-1] != "json":
175+
# e.g. pypi/<name>/ or pypi/<name>/<version>/ (the *-suffixed /json paths are
176+
# already handled by content_handler, unconditionally on Accept).
177+
name = parts[1]
178+
version = parts[2] if len(parts) == 3 else None
179+
normalized = canonicalize_name(name)
180+
package_content = content.filter(name_normalized=normalized)
181+
headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"}
182+
yank_markers = dict(
183+
PackageYank.objects.filter(
184+
pk__in=repo_version.content, name_normalized=normalized
185+
).values_list("version", "yanked_reason")
186+
)
187+
json_body = python_content_to_json(
188+
self.base_path,
189+
package_content,
190+
version=version,
191+
domain=domain,
192+
repository_version=repo_version,
193+
yank_markers=yank_markers,
194+
)
195+
if json_body:
196+
return json_response(json_body, headers=headers)
197+
return None
198+
199+
if parts[0] == "simple":
200+
if len(parts) == 1:
201+
names = (
202+
content.order_by("name_normalized")
203+
.values_list("name", flat=True)
204+
.distinct("name_normalized")
205+
)
206+
headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"}
207+
return json_response(
208+
write_simple_index_json(list(names)),
209+
headers=headers,
210+
content_type=PYPI_SIMPLE_V1_JSON,
211+
)
212+
213+
if len(parts) != 2:
214+
return None
215+
216+
normalized = canonicalize_name(parts[1])
217+
packages = content.filter(name_normalized=normalized).annotate(
218+
active_membership=FilteredRelation(
219+
"version_memberships",
220+
condition=Q(
221+
version_memberships__repository=repo_version.repository,
222+
version_memberships__version_removed=None,
223+
),
224+
),
225+
repo_added_time=F("active_membership__pulp_created"),
226+
)
227+
if not packages.exists():
228+
return None
229+
releases = [
230+
{
231+
"filename": p.filename,
232+
"sha256": p.sha256,
233+
"metadata_sha256": p.metadata_sha256,
234+
"requires_python": p.requires_python,
235+
"size": p.size,
236+
"upload_time": p.repo_added_time or p.pulp_created,
237+
"version": p.version,
238+
"url": build_content_url(self.base_path, p.filename, domain=domain),
239+
}
240+
for p in packages
241+
]
242+
return json_response(
243+
write_simple_detail_json(normalized, releases),
244+
headers={PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT), "Vary": "Accept"},
245+
content_type=PYPI_SIMPLE_V1_JSON,
246+
)
247+
248+
return None
249+
140250
class Meta:
141251
default_related_name = "%(app_label)s_%(model_name)s"
142252
permissions = [

‎pulp_python/app/utils.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,20 @@ def python_content_to_urls(contents, base_path, domain=None):
506506
return [python_content_to_download_info(content, base_path, domain) for content in contents]
507507

508508

509+
def build_content_url(base_path, filename, domain=None):
510+
"""
511+
Builds the absolute content-app URL for a filename served under a distribution's base_path.
512+
"""
513+
origin = settings.CONTENT_ORIGIN or settings.PYPI_API_HOSTNAME or ""
514+
origin = origin.strip("/")
515+
prefix = settings.CONTENT_PATH_PREFIX.strip("/")
516+
base_path = base_path.strip("/")
517+
components = [origin, prefix, base_path, filename]
518+
if domain:
519+
components.insert(2, domain.name)
520+
return "/".join(components)
521+
522+
509523
def python_content_to_download_info(content, base_path, domain=None):
510524
"""
511525
Takes in a PythonPackageContent and base path of the distribution to create a dictionary of
@@ -524,14 +538,7 @@ def find_artifact():
524538
relative_path__endswith=".metadata"
525539
).first()
526540
artifact = find_artifact()
527-
origin = settings.CONTENT_ORIGIN or settings.PYPI_API_HOSTNAME or ""
528-
origin = origin.strip("/")
529-
prefix = settings.CONTENT_PATH_PREFIX.strip("/")
530-
base_path = base_path.strip("/")
531-
components = [origin, prefix, base_path, content.filename]
532-
if domain:
533-
components.insert(2, domain.name)
534-
url = "/".join(components)
541+
url = build_content_url(base_path, content.filename, domain=domain)
535542
md5 = artifact.md5 if artifact and artifact.md5 else ""
536543
return {
537544
"comment_text": "",
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
"""Content-app JSON via Accept: application/json (PythonDistribution.content_handler_json).
2+
3+
These tests hit the content app, not the DRF ``/pypi/`` API. They require pulpcore
4+
Phase 1 (issue 7887): ``Distribution.content_handler_json`` plus Handler Accept
5+
negotiation. When CI pulpcore is unpatched, skip — unit tests still cover the
6+
handler return values.
7+
"""
8+
9+
from importlib.util import find_spec
10+
from pathlib import Path
11+
from urllib.parse import urljoin
12+
13+
import pytest
14+
import requests
15+
16+
from pulp_python.tests.functional.constants import (
17+
PYPI_SIMPLE_V1_JSON,
18+
PYTHON_WHEEL_FILENAME,
19+
PYTHON_XS_PROJECT_SPECIFIER,
20+
)
21+
22+
JSON_ACCEPT = {"Accept": "application/json"}
23+
24+
25+
def _pulpcore_content_app_json_enabled():
26+
"""True when pulpcore will invoke Distribution.content_handler_json on JSON Accept.
27+
28+
Inspect source files instead of importing Django models, which are not ready
29+
during pytest collection. Use find_spec so editable installs are resolved.
30+
"""
31+
handler = find_spec("pulpcore.content.handler")
32+
if not handler or not handler.origin:
33+
return False
34+
handler_path = Path(handler.origin)
35+
publication_path = handler_path.resolve().parents[1] / "app" / "models" / "publication.py"
36+
try:
37+
handler_src = handler_path.read_text()
38+
publication_src = publication_path.read_text()
39+
except OSError:
40+
return False
41+
return "def negotiate_json" in handler_src and "def content_handler_json" in publication_src
42+
43+
44+
pytestmark = pytest.mark.skipif(
45+
not _pulpcore_content_app_json_enabled(),
46+
reason=(
47+
"Requires pulpcore content-app Accept negotiation (Handler.negotiate_json and "
48+
"Distribution.content_handler_json from pulpcore issue 7887). Unit tests cover "
49+
"PythonDistribution.content_handler_json without that pulpcore change."
50+
),
51+
)
52+
53+
54+
def _content_url(pulp_content_url, distro, rel=""):
55+
return urljoin(pulp_content_url, f"{distro.base_path}/{rel}")
56+
57+
58+
@pytest.mark.parallel
59+
def test_content_app_simple_index_and_detail_json(
60+
python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
61+
):
62+
"""``.../simple/`` and ``.../simple/<name>/`` return PEP 691 Simple JSON."""
63+
remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER)
64+
repo = python_repo_with_sync(remote)
65+
distro = python_distribution_factory(repository=repo)
66+
67+
index = requests.get(_content_url(pulp_content_url, distro, "simple/"), headers=JSON_ACCEPT)
68+
assert index.status_code == 200
69+
assert PYPI_SIMPLE_V1_JSON in index.headers["Content-Type"]
70+
index_body = index.json()
71+
assert index_body["meta"]["api-version"] == "1.1"
72+
names = [project["name"] for project in index_body["projects"]]
73+
assert any("shelf" in name.lower() for name in names)
74+
75+
detail = requests.get(
76+
_content_url(pulp_content_url, distro, "simple/shelf-reader/"), headers=JSON_ACCEPT
77+
)
78+
assert detail.status_code == 200
79+
assert PYPI_SIMPLE_V1_JSON in detail.headers["Content-Type"]
80+
detail_body = detail.json()
81+
assert detail_body["name"] == "shelf-reader"
82+
assert detail_body["files"]
83+
assert detail_body["versions"]
84+
filenames = [f["filename"] for f in detail_body["files"]]
85+
assert PYTHON_WHEEL_FILENAME in filenames or any("shelf" in f for f in filenames)
86+
87+
88+
@pytest.mark.parallel
89+
def test_content_app_pypi_json_without_json_suffix(
90+
python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
91+
):
92+
"""``.../pypi/<name>/`` returns PyPI JSON without the ``/json`` URL convention."""
93+
remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER)
94+
repo = python_repo_with_sync(remote)
95+
distro = python_distribution_factory(repository=repo)
96+
97+
response = requests.get(
98+
_content_url(pulp_content_url, distro, "pypi/shelf-reader/"), headers=JSON_ACCEPT
99+
)
100+
assert response.status_code == 200
101+
assert "application/json" in response.headers["Content-Type"]
102+
body = response.json()
103+
assert "info" in body
104+
assert "releases" in body
105+
assert "urls" in body
106+
assert "shelf" in body["info"]["name"].lower()
107+
108+
109+
@pytest.mark.parallel
110+
def test_content_app_distro_root_is_generic_listing(
111+
python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
112+
):
113+
"""Distribution root still uses pulpcore's generic JSON listing, not PyPI/Simple JSON."""
114+
remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER)
115+
repo = python_repo_with_sync(remote)
116+
distro = python_distribution_factory(repository=repo)
117+
118+
response = requests.get(_content_url(pulp_content_url, distro), headers=JSON_ACCEPT)
119+
assert response.status_code == 200
120+
assert "application/json" in response.headers["Content-Type"]
121+
body = response.json()
122+
assert "packages" in body
123+
assert "projects" not in body
124+
assert "info" not in body
125+
126+
127+
@pytest.mark.parallel
128+
def test_content_app_artifact_stays_binary(
129+
python_remote_factory, python_repo_with_sync, python_distribution_factory, pulp_content_url
130+
):
131+
"""Concrete package artifacts stay binary even when Accept prefers JSON."""
132+
remote = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER, policy="immediate")
133+
repo = python_repo_with_sync(remote)
134+
distro = python_distribution_factory(repository=repo)
135+
136+
response = requests.get(
137+
_content_url(pulp_content_url, distro, PYTHON_WHEEL_FILENAME), headers=JSON_ACCEPT
138+
)
139+
assert response.status_code == 200
140+
assert "application/json" not in response.headers.get("Content-Type", "")
141+
assert response.content[:2] == b"PK" # zip/wheel local file header

‎pulp_python/tests/unit/conftest.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import os
2+
3+
import django
4+
5+
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pulpcore.app.settings")
6+
7+
8+
def pytest_configure(config):
9+
django.setup()

0 commit comments

Comments
 (0)