Skip to content

Commit 0a5906d

Browse files
gerrod3claude
andcommitted
Key pull-through entries per project and refresh them on remote edits
Entries were stored in one hash per remote that kept growing while the remote was in use, and editing the remote orphaned them while cached simple pages still linked to them. Entries are now grouped per project parsed from the filename and expire on their own. Changing the remote's url, includes or excludes clears its cached simple pages and entries, other edits keep them. A None EXPIRES_TTL no longer crashes and means the entries never expire. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent fd55068 commit 0a5906d

4 files changed

Lines changed: 130 additions & 21 deletions

File tree

‎pulp_python/app/models.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77
from django.conf import settings
88
from django.contrib.postgres.fields import ArrayField
99
from django.core.exceptions import ObjectDoesNotExist
10-
from django.db import models
10+
from django.db import models, transaction
1111
from django_lifecycle import (
12+
AFTER_UPDATE,
13+
BEFORE_DELETE,
1214
BEFORE_SAVE,
1315
hook,
1416
)
@@ -40,6 +42,7 @@
4042
artifact_to_metadata_artifact,
4143
artifact_to_python_content_data,
4244
canonicalize_name,
45+
clear_pull_through_entries,
4346
lookup_pull_through_entry,
4447
python_content_to_json,
4548
unsign_pull_through_token,
@@ -398,6 +401,23 @@ def get_downloader(self, remote_artifact=None, url=None, download_factory=None,
398401
remote_artifact=remote_artifact, url=url, download_factory=download_factory, **kwargs
399402
)
400403

404+
@hook(AFTER_UPDATE, when_any=["url", "includes", "excludes"], has_changed=True)
405+
def invalidate_pull_through_cache(self):
406+
"""Rebuild the pull-through simple pages and entries when the files they list change."""
407+
408+
def invalidate():
409+
self.invalidate_cache()
410+
clear_pull_through_entries(self.pk)
411+
412+
transaction.on_commit(invalidate)
413+
414+
@hook(BEFORE_DELETE)
415+
def clear_pull_through_cache(self):
416+
"""Drop the pull-through entries of a deleted remote."""
417+
# The pk is cleared on delete before the transaction commits
418+
remote_pk = self.pk
419+
transaction.on_commit(lambda: clear_pull_through_entries(remote_pk))
420+
401421
class Meta:
402422
default_related_name = "%(app_label)s_%(model_name)s"
403423
permissions = [

‎pulp_python/app/utils.py‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from django.db.utils import IntegrityError
1818
from jinja2 import Template
1919
from packaging.requirements import Requirement
20-
from packaging.utils import canonicalize_name
20+
from packaging.utils import canonicalize_name, parse_sdist_filename, parse_wheel_filename
2121
from packaging.version import InvalidVersion, parse
2222
from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage
2323

@@ -40,8 +40,9 @@
4040

4141
PULL_THROUGH_PARAM = "pt"
4242
PULL_THROUGH_SALT = "pulp_python.pull_through"
43-
# Outlive both the cached simple pages and their max-age in client caches
44-
PULL_THROUGH_TTL = settings.CACHE_SETTINGS["EXPIRES_TTL"] + 600
43+
_CACHE_TTL = settings.CACHE_SETTINGS["EXPIRES_TTL"]
44+
# Outlive both the cached simple pages and their max-age in client caches, None never expires
45+
PULL_THROUGH_TTL = None if _CACHE_TTL is None else _CACHE_TTL + 600
4546

4647
PYPI_SIMPLE_V1_HTML = "application/vnd.pypi.simple.v1+html"
4748
PYPI_SIMPLE_V1_JSON = "application/vnd.pypi.simple.v1+json"
@@ -809,9 +810,17 @@ def unsign_pull_through_token(remote, token, filename):
809810
return None
810811

811812

812-
def _pull_through_key(remote):
813-
# Editing the remote invalidates its entries
814-
return f"pulp_python:pull_through:{remote.pk}:{remote.pulp_last_updated.timestamp()}"
813+
def _pull_through_key(remote, filename):
814+
# Group entries per project so each expires on its own, falling back to the filename for
815+
# legacy names that can't be parsed
816+
try:
817+
if filename.endswith(".whl"):
818+
bucket = parse_wheel_filename(filename)[0]
819+
else:
820+
bucket = parse_sdist_filename(filename)[0]
821+
except ValueError:
822+
bucket = filename
823+
return f"pulp_python:pull_through:{remote.pk}:{bucket}"
815824

816825

817826
def store_pull_through_entries(remote, entries):
@@ -824,17 +833,35 @@ def store_pull_through_entries(remote, entries):
824833
return False
825834
if not entries:
826835
return True
827-
key = _pull_through_key(remote)
836+
buckets = defaultdict(dict)
837+
for filename, entry in entries.items():
838+
buckets[_pull_through_key(remote, filename)][filename] = json.dumps(entry)
828839
try:
829-
redis = SyncContentCache().redis
830-
redis.hset(key, mapping={f: json.dumps(e) for f, e in entries.items()})
831-
redis.expire(key, PULL_THROUGH_TTL)
840+
pipe = SyncContentCache().redis.pipeline(transaction=False)
841+
for key, mapping in buckets.items():
842+
pipe.hset(key, mapping=mapping)
843+
if PULL_THROUGH_TTL is not None:
844+
pipe.expire(key, PULL_THROUGH_TTL)
845+
pipe.execute()
832846
except Exception as e:
833847
log.warning("Failed to store pull-through entries: %s", e)
834848
return False
835849
return True
836850

837851

852+
def clear_pull_through_entries(remote_pk):
853+
"""Deletes all the stored pull-through entries of a remote."""
854+
if not settings.CACHE_ENABLED:
855+
return
856+
try:
857+
redis = SyncContentCache().redis
858+
keys = list(redis.scan_iter(match=f"pulp_python:pull_through:{remote_pk}:*", count=1000))
859+
if keys:
860+
redis.delete(*keys)
861+
except Exception as e:
862+
log.warning("Failed to clear pull-through entries: %s", e)
863+
864+
838865
def lookup_pull_through_entry(remote, filename):
839866
"""
840867
Returns the upstream (url, sha256) for a file stored by store_pull_through_entries.
@@ -844,10 +871,9 @@ def lookup_pull_through_entry(remote, filename):
844871
if not settings.CACHE_ENABLED:
845872
return None
846873
is_metadata = filename.endswith(".metadata")
874+
filename = filename.removesuffix(".metadata")
847875
try:
848-
raw = SyncContentCache().redis.hget(
849-
_pull_through_key(remote), filename.removesuffix(".metadata")
850-
)
876+
raw = SyncContentCache().redis.hget(_pull_through_key(remote, filename), filename)
851877
except Exception as e:
852878
log.warning("Failed to look up pull-through entry: %s", e)
853879
return None

‎pulp_python/tests/functional/api/test_full_mirror.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,43 @@ def test_pull_through_rejects_bad_links(
326326
assert sha256(r.content).hexdigest() == package1.digests["sha256"]
327327

328328

329+
@pytest.mark.parallel
330+
def test_pull_through_remote_update(
331+
python_bindings, python_remote_factory, python_distribution_factory, monitor_task, signed_links
332+
):
333+
"""Tests that changing what the remote lists refreshes its pull-through pages and entries."""
334+
if signed_links:
335+
pytest.skip("Pull-through entries are only stored when Redis is available")
336+
remote = python_remote_factory(includes=["shelf-reader"])
337+
distro = python_distribution_factory(remote=remote.pulp_href)
338+
339+
url = f"{distro.base_url}simple/shelf-reader/"
340+
assert requests.get(url).headers["X-PULP-CACHE"] == "MISS"
341+
r = requests.get(url)
342+
assert r.headers["X-PULP-CACHE"] == "HIT"
343+
package1, package2 = ProjectPage.from_response(r, "shelf-reader").packages
344+
345+
# Other edits keep the cached page and its links
346+
body = {"download_concurrency": 5}
347+
monitor_task(python_bindings.RemotesPythonApi.partial_update(remote.pulp_href, body).task)
348+
assert requests.get(url).headers["X-PULP-CACHE"] == "HIT"
349+
r = requests.get(package1.url)
350+
assert r.status_code == 200
351+
assert sha256(r.content).hexdigest() == package1.digests["sha256"]
352+
353+
# Files listed by the old url need the simple page to be fetched again
354+
body = {"url": PYPI_URL}
355+
monitor_task(python_bindings.RemotesPythonApi.partial_update(remote.pulp_href, body).task)
356+
assert requests.get(package2.url).status_code == 404
357+
r = requests.get(url)
358+
assert r.headers["X-PULP-CACHE"] == "MISS"
359+
packages = ProjectPage.from_response(r, "shelf-reader").packages
360+
package2 = next(p for p in packages if p.filename == package2.filename)
361+
r = requests.get(package2.url)
362+
assert r.status_code == 200
363+
assert sha256(r.content).hexdigest() == package2.digests["sha256"]
364+
365+
329366
@pytest.mark.parallel
330367
def test_pull_through_metadata_with_repo(
331368
python_repo_factory,

‎pulp_python/tests/unit/test_pull_through.py‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
1-
from datetime import datetime, timedelta
21
from types import SimpleNamespace
32
from uuid import uuid4
43

54
import pytest
65
from django.conf import settings
76

7+
from pulpcore.plugin.cache import SyncContentCache
8+
9+
from pulp_python.app import utils
810
from pulp_python.app.utils import (
911
PULL_THROUGH_PARAM,
12+
clear_pull_through_entries,
1013
lookup_pull_through_entry,
1114
sign_pull_through_path,
1215
store_pull_through_entries,
@@ -66,22 +69,45 @@ def test_rejects_tampered_token():
6669

6770
@pytest.mark.skipif(not settings.CACHE_ENABLED, reason="CACHE_ENABLED is not set")
6871
def test_stored_entries():
69-
remote = SimpleNamespace(pk=uuid4(), pulp_last_updated=datetime.now())
72+
remote = SimpleNamespace(pk=uuid4())
73+
sdist = "shelf-reader-0.1.tar.gz"
74+
legacy = "pytz-2011k.tar.bz2"
7075
entry = {"url": URL, "sha256": "a" * 64, "metadata_sha256": None}
71-
assert store_pull_through_entries(remote, {FILENAME: entry})
76+
assert store_pull_through_entries(
77+
remote,
78+
{FILENAME: entry, sdist: {"url": f"{URL}/{sdist}"}, legacy: {"url": f"{URL}/{legacy}"}},
79+
)
7280

7381
assert lookup_pull_through_entry(remote, FILENAME) == (URL, "a" * 64)
7482
assert lookup_pull_through_entry(remote, f"{FILENAME}.metadata") is None
83+
assert lookup_pull_through_entry(remote, sdist) == (f"{URL}/{sdist}", None)
84+
assert lookup_pull_through_entry(remote, legacy) == (f"{URL}/{legacy}", None)
7585
assert lookup_pull_through_entry(remote, "evil-1.0-py3-none-any.whl") is None
86+
assert lookup_pull_through_entry(SimpleNamespace(pk=uuid4()), FILENAME) is None
7687

7788
entry["metadata_sha256"] = "b" * 64
7889
assert store_pull_through_entries(remote, {FILENAME: entry})
7990
assert lookup_pull_through_entry(remote, f"{FILENAME}.metadata") == (
8091
f"{URL}.metadata",
8192
"b" * 64,
8293
)
94+
assert lookup_pull_through_entry(remote, sdist) == (f"{URL}/{sdist}", None)
8395

84-
other = SimpleNamespace(pk=uuid4(), pulp_last_updated=remote.pulp_last_updated)
85-
assert lookup_pull_through_entry(other, FILENAME) is None
86-
edited = SimpleNamespace(pk=remote.pk, pulp_last_updated=datetime.now() + timedelta(1))
87-
assert lookup_pull_through_entry(edited, FILENAME) is None
96+
clear_pull_through_entries(remote.pk)
97+
assert lookup_pull_through_entry(remote, FILENAME) is None
98+
assert lookup_pull_through_entry(remote, legacy) is None
99+
100+
101+
@pytest.mark.skipif(not settings.CACHE_ENABLED, reason="CACHE_ENABLED is not set")
102+
@pytest.mark.parametrize("ttl", [1200, None])
103+
def test_stored_entries_ttl(monkeypatch, ttl):
104+
monkeypatch.setattr(utils, "PULL_THROUGH_TTL", ttl)
105+
remote = SimpleNamespace(pk=uuid4())
106+
assert store_pull_through_entries(remote, {FILENAME: {"url": URL}})
107+
108+
redis_ttl = SyncContentCache().redis.ttl(utils._pull_through_key(remote, FILENAME))
109+
if ttl is None:
110+
assert redis_ttl == -1
111+
else:
112+
assert 0 < redis_ttl <= ttl
113+
clear_pull_through_entries(remote.pk)

0 commit comments

Comments
 (0)