Skip to content

Commit 58c4b0e

Browse files
committed
Address package catalog review feedback
Exclude versions yanked in the selected repository version before pagination. Report the latest repository membership creation time for each package version, including re-additions, and let DRF format timestamps consistently. Remove the unrelated Simple API ordering change, assert exact permission statuses, and explain the repository-version domain guard. Cover historical membership times, cross-repository additions, and yank filtering through the API. Ref #1358. Generated-By: Codex
1 parent 227b70b commit 58c4b0e

7 files changed

Lines changed: 119 additions & 29 deletions

File tree

‎CHANGES/1358.feature‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
Added repository and repository-version endpoints that list Python packages and version update times with limit/offset pagination.
1+
Added paginated package catalogs for repositories and repository versions, excluding yanked versions and reporting when files were added to the repository.

‎docs/user/guides/catalog.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,12 @@ A wheel and an sdist of the same version share one entry. Different version
2626
strings, such as `1.0.0` and `1.0.0+test.1`, produce separate entries.
2727

2828
Each version's `last_updated` is the latest `pulp_created` timestamp among its
29-
files in the selected repository version. This records when Pulp created the
30-
content, not the upstream release date.
29+
repository membership records in the selected repository version. This records
30+
when a file was added to the repository. Removing and re-adding a file updates
31+
this time. Historical repository versions retain the earlier addition time.
32+
33+
Package versions with a yank marker in the selected repository version are excluded.
34+
Packages with no remaining versions are excluded from both the results and `count`.
3135

3236
```json
3337
{

‎pulp_python/app/catalog.py‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,39 @@
22

33
from django import VERSION as DJANGO_VERSION
44
from django.contrib.postgres.aggregates import JSONBAgg
5-
from django.db.models import Case, IntegerField, Value, When
5+
from django.db.models import Case, Exists, F, IntegerField, OuterRef, Q, Value, When
6+
from django.utils.dateparse import parse_datetime
67
from packaging.version import InvalidVersion, Version
78

8-
from pulp_python.app.models import PythonPackageContent
9+
from pulp_python.app.models import PackageYank, PythonPackageContent
10+
11+
12+
def package_catalog_content(repository_version):
13+
"""Select unyanked files and their addition times in the repository version."""
14+
content = PythonPackageContent.objects.all()
15+
if repository_version is None:
16+
return content.annotate(repository_added=F("pulp_created")).none()
17+
yanks = repository_version.get_content(PackageYank.objects.all()).filter(
18+
name_normalized=OuterRef("name_normalized"), version=OuterRef("version")
19+
)
20+
return (
21+
content.filter(
22+
Q(version_memberships__version_removed__isnull=True)
23+
| Q(version_memberships__version_removed__number__gt=repository_version.number),
24+
version_memberships__repository_id=repository_version.repository_id,
25+
version_memberships__version_added__number__lte=repository_version.number,
26+
)
27+
.annotate(repository_added=F("version_memberships__pulp_created"))
28+
.filter(~Exists(yanks))
29+
)
930

1031

1132
def _version_sort_key(version):
1233
"""Sort invalid version strings after valid PEP 440 versions."""
1334
try:
14-
return (True, Version(version))
35+
return True, Version(version)
1536
except InvalidVersion:
16-
return (False, version)
37+
return False, version
1738

1839

1940
class PackageVersions(JSONBAgg):
@@ -27,9 +48,9 @@ def package_catalog_queryset(content, ordering=("name_normalized",)):
2748
"""Collect version timestamps and metadata IDs for each normalized package name.
2849
2950
Wheels and sdists with the same version share an aggregate entry.
30-
Each timestamp is the newest file's `pulp_created`. Use sdist metadata when available.
51+
Each timestamp is the latest file addition to the repository. Use sdist metadata when available.
3152
"""
32-
# Aggregate ordering selects the newest timestamp and prefers sdist metadata.
53+
# Aggregate ordering selects the latest addition time and prefers sdist metadata.
3354
ordering_argument = "order_by" if DJANGO_VERSION >= (5, 2) else "ordering"
3455
file_ordering = {
3556
ordering_argument: (
@@ -49,8 +70,8 @@ def package_catalog_queryset(content, ordering=("name_normalized",)):
4970
.annotate(
5071
versions=PackageVersions(
5172
"version",
52-
"pulp_created",
53-
**{ordering_argument: ("pulp_created",)},
73+
"repository_added",
74+
**{ordering_argument: ("repository_added",)},
5475
),
5576
metadata_ids=PackageVersions(
5677
"version",
@@ -69,7 +90,7 @@ def assemble_package_index(package_rows):
6990
for row in package_rows:
7091
row["versions"] = sorted(
7192
(
72-
{"version": version, "last_updated": updated}
93+
{"version": version, "last_updated": parse_datetime(updated)}
7394
for version, updated in row["versions"].items()
7495
),
7596
key=lambda unit: (_version_sort_key(unit["version"]), unit["version"]),

‎pulp_python/app/pypi/views.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ def list(self, request, path):
370370
if self.should_redirect(repo_version=repo_version):
371371
return redirect(urljoin(self.base_content_url, f"{path}/simple/"))
372372
names = (
373-
content.order_by("name_normalized", "name")
373+
content.order_by("name_normalized")
374374
.values_list("name", flat=True)
375375
.distinct("name_normalized")
376376
.iterator()

‎pulp_python/app/serializers.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -648,8 +648,8 @@ class PythonRepositoryPackageVersionSerializer(serializers.Serializer):
648648
version = serializers.CharField(help_text=_("The package's version number."))
649649
last_updated = serializers.DateTimeField(
650650
help_text=_(
651-
"The latest `pulp_created` timestamp for this version's files "
652-
"in the selected repository version."
651+
"The latest time a file of this package version was added to the repository, "
652+
"as of the selected repository version."
653653
),
654654
)
655655

‎pulp_python/app/viewsets.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
from pulp_python.app import models as python_models
3535
from pulp_python.app import serializers as python_serializers
3636
from pulp_python.app import tasks
37-
from pulp_python.app.catalog import package_catalog_queryset
37+
from pulp_python.app.catalog import package_catalog_content, package_catalog_queryset
3838

3939

4040
class PythonRepositoryPackageFilter(FilterSet):
@@ -90,13 +90,13 @@ def packages(self, request, **kwargs):
9090
"""List packages from the repository version identified by the resource URL."""
9191
resource = self.get_object()
9292
if isinstance(resource, RepositoryVersion):
93+
# Core filters models with a pulp_domain field; versions inherit it via repository.
9394
if resource.repository.pulp_domain_id != get_domain_pk():
9495
raise Http404
9596
version = resource
9697
else:
9798
version = resource.latest_version()
98-
content = python_models.PythonPackageContent.objects.all()
99-
content = version.get_content(content) if version is not None else content.none()
99+
content = package_catalog_content(version)
100100
queryset = self.filter_queryset(content)
101101
page = self.paginate_queryset(queryset)
102102
serializer = self.get_serializer(page if page is not None else queryset, many=True)

‎pulp_python/tests/functional/api/test_catalog.py‎

Lines changed: 76 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22

33
import io
44
import tarfile
5+
from urllib.parse import urljoin
56

67
import pytest
8+
import requests
79

810
from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER
911

@@ -63,7 +65,9 @@ def test_package_list(python_repo_with_sync, python_remote_factory, catalog):
6365

6466

6567
@pytest.mark.parallel
66-
def test_package_list_snapshots(python_bindings, python_repo_with_sync, monitor_task):
68+
def test_package_list_snapshots(
69+
bindings_cfg, python_bindings, python_repo_with_sync, python_repo_factory, monitor_task
70+
):
6771
"""Metadata and update times use only files in the requested repository version."""
6872
repo = python_repo_with_sync()
6973
repositories = python_bindings.RepositoriesPythonApi
@@ -81,27 +85,51 @@ def test_package_list_snapshots(python_bindings, python_repo_with_sync, monitor_
8185
original = repositories.packages(repo.pulp_href)
8286
assert original.count == 1
8387
row = original.results[0]
88+
response = requests.get(
89+
urljoin(bindings_cfg.host, f"{repo.pulp_href}packages/"),
90+
auth=(bindings_cfg.username, bindings_cfg.password),
91+
)
92+
assert response.status_code == 200
93+
assert response.json()["results"][0]["versions"][0]["last_updated"].endswith("Z")
8494
assert row.description == sdist.description
85-
assert [(item.version, item.last_updated) for item in row.versions] == [
86-
("0.1", newest.pulp_created)
87-
]
95+
assert [item.version for item in row.versions] == ["0.1"]
96+
added = row.versions[0].last_updated
97+
assert added >= versions.read(repo.latest_version_href).pulp_created
8898

8999
monitor_task(
90100
repositories.modify(repo.pulp_href, {"remove_content_units": [newest.pulp_href]}).task
91101
)
92102
current = repositories.packages(repo.pulp_href).results[0]
93-
assert current.versions[0].last_updated == remaining.pulp_created
103+
assert remaining.pulp_created < current.versions[0].last_updated <= added
94104
assert current.description == remaining.description
105+
monitor_task(
106+
repositories.modify(repo.pulp_href, {"add_content_units": [newest.pulp_href]}).task
107+
)
108+
readded = repositories.packages(repo.pulp_href).results[0].versions[0].last_updated
109+
assert readded > added
110+
other = python_repo_factory()
111+
monitor_task(
112+
repositories.modify(other.pulp_href, {"add_content_units": [newest.pulp_href]}).task
113+
)
114+
assert repositories.packages(repo.pulp_href).results[0].versions[0].last_updated == readded
95115
assert versions.packages(repo.latest_version_href).to_dict() == original.to_dict()
96116
with pytest.raises(python_bindings.ApiException) as exc:
97117
versions.packages(f"{repo.pulp_href}versions/999/")
98118
assert exc.value.status == 404
99119

100120

101121
@pytest.mark.parallel
102-
def test_package_list_versions(python_content_factory, python_repo, python_bindings, tmp_path):
122+
def test_package_list_versions(
123+
python_content_factory,
124+
python_repo,
125+
python_bindings,
126+
python_distribution_factory,
127+
monitor_task,
128+
tmp_path,
129+
):
103130
"""PEP 440 ordering selects metadata and preserves separate rebuild versions."""
104131
content = {}
132+
added_after = {}
105133
for version in ("1.10", "1.9", "1.9+test.1"):
106134
filename = f"catalog-test-{version}.tar.gz"
107135
path = tmp_path / filename
@@ -113,19 +141,56 @@ def test_package_list_versions(python_content_factory, python_repo, python_bindi
113141
info.size = len(metadata)
114142
archive.addfile(info, io.BytesIO(metadata))
115143
content[version] = python_content_factory(filename, file=str(path), repository=python_repo)
144+
repo = python_bindings.RepositoriesPythonApi.read(python_repo.pulp_href)
145+
added_after[version] = python_bindings.RepositoriesPythonVersionsApi.read(
146+
repo.latest_version_href
147+
).pulp_created
116148

117149
data = python_bindings.RepositoriesPythonApi.packages(python_repo.pulp_href)
118150
assert data.count == 1
119151
row = data.results[0]
120152
assert row.description.strip() == "Version 1.10"
121-
assert [(item.version, item.last_updated) for item in row.versions] == [
122-
(version, content[version].pulp_created) for version in ("1.10", "1.9+test.1", "1.9")
123-
]
153+
assert [item.version for item in row.versions] == ["1.10", "1.9+test.1", "1.9"]
154+
for item in row.versions:
155+
assert item.last_updated >= added_after[item.version]
156+
assert item.last_updated > content[item.version].pulp_created
157+
distro = python_distribution_factory(repository=python_repo)
158+
monitor_task(
159+
python_bindings.PypiYankApi.yank(
160+
path=distro.base_path, yank={"name": "catalog-test", "version": "1.10"}
161+
).task
162+
)
163+
row = python_bindings.RepositoriesPythonApi.packages(python_repo.pulp_href).results[0]
164+
assert [item.version for item in row.versions] == ["1.9+test.1", "1.9"]
165+
assert row.description.strip() == "Version 1.9+test.1"
166+
167+
168+
@pytest.mark.parallel
169+
def test_package_list_yanked(
170+
python_bindings, python_repo_with_sync, python_distribution_factory, monitor_task, catalog
171+
):
172+
"""Yank markers exclude versions only from repository versions containing them."""
173+
repo = python_repo_with_sync()
174+
distro = python_distribution_factory(repository=repo)
175+
monitor_task(
176+
python_bindings.PypiYankApi.yank(
177+
path=distro.base_path, yank={"name": "shelf-reader", "version": "0.1"}
178+
).task
179+
)
180+
historical = python_bindings.RepositoriesPythonVersionsApi.packages(repo.latest_version_href)
181+
assert historical.count == 1
182+
latest = python_bindings.RepositoriesPythonApi.read(repo.pulp_href)
183+
excluded = catalog(latest)
184+
assert excluded.count == 0
185+
assert excluded.results == []
124186

125187

126188
@pytest.mark.parallel
189+
@pytest.mark.parametrize(
190+
"catalog, denied_status", [("repository", 404), ("version", 403)], indirect=["catalog"]
191+
)
127192
def test_package_list_permissions(
128-
python_bindings, python_repo, python_repo_factory, gen_user, catalog
193+
python_bindings, python_repo, python_repo_factory, gen_user, catalog, denied_status
129194
):
130195
"""Both actions require view permission on the parent repository."""
131196
other = python_repo_factory()
@@ -134,7 +199,7 @@ def test_package_list_permissions(
134199
assert catalog(python_repo).results == []
135200
with pytest.raises(python_bindings.ApiException) as exc:
136201
catalog(other)
137-
assert exc.value.status in (403, 404)
202+
assert exc.value.status == denied_status
138203
with gen_user(model_roles=["python.pythonrepository_viewer"]):
139204
assert catalog(other).results == []
140205

0 commit comments

Comments
 (0)