Skip to content

Commit 18e94be

Browse files
gerrod3claude
andcommitted
Bind signed pull-through links to the remote's listing and test without Redis
Signed tokens now depend on the remote's url, includes and excludes, so changing any of them invalidates links handed out before. Adds a Redis-disabled s3 runner so the signed mode runs in CI. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 0a5906d commit 18e94be

7 files changed

Lines changed: 102 additions & 25 deletions

File tree

‎.github/workflows/scripts/install.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,9 @@ fi
5757
# Needed for some functional tests
5858
cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
5959
cmd_prefix bash -c "usermod -a -G wheel pulp"
60+
61+
# In some scenarios we want to simulate a failed redis cache.
62+
if [[ " s3 " =~ " ${TEST} " ]]; then
63+
cmd_prefix bash -c "s6-rc -d change redis"
64+
echo "The Redis service was disabled for $TEST"
65+
fi

‎pulp_python/app/pypi/views.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,8 @@ def retrieve(self, request, path, package):
433433
normalized = canonicalize_name(package)
434434
releases = {}
435435
if self.distribution.remote:
436-
releases = self.pull_through_package_simple(normalized, path, self.distribution.remote)
436+
remote = self.distribution.remote.cast()
437+
releases = self.pull_through_package_simple(normalized, path, remote)
437438
elif self.should_redirect(repo_version=repo_ver):
438439
return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/"))
439440
if content is not None:

‎pulp_python/app/utils.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -779,14 +779,21 @@ async def aget_remote_simple_page(package, remote, max_retries=1):
779779
return page
780780

781781

782+
def _pull_through_signer(remote):
783+
# Changing what the remote lists invalidates its tokens, like it clears its stored entries
784+
listing = json.dumps([remote.url, remote.includes, remote.excludes])
785+
salt = f"{PULL_THROUGH_SALT}:{remote.pk}:{hashlib.sha256(listing.encode()).hexdigest()}"
786+
return signing.Signer(salt=salt)
787+
788+
782789
def sign_pull_through_path(remote, filename, url, sha256=None, metadata_sha256=None):
783790
"""Returns the content app path of a pull-through package, carrying its signed upstream url."""
784-
data = {"r": str(remote.pk), "f": filename, "u": url}
791+
data = {"f": filename, "u": url}
785792
if sha256:
786793
data["s"] = sha256
787794
if metadata_sha256:
788795
data["m"] = metadata_sha256
789-
token = signing.Signer(salt=PULL_THROUGH_SALT).sign_object(data, compress=True)
796+
token = _pull_through_signer(remote).sign_object(data, compress=True)
790797
return f"{filename}?{PULL_THROUGH_PARAM}={token}"
791798

792799

@@ -795,14 +802,13 @@ def unsign_pull_through_token(remote, token, filename):
795802
Returns the upstream (url, sha256) for a file of a signed pull-through token.
796803
797804
filename can be the signed package's filename or its PEP 658 metadata file. Returns None if
798-
the token is invalid, was signed for another remote or doesn't cover filename.
805+
the token is invalid, was signed for another remote or before its url, includes or excludes
806+
changed, or doesn't cover filename.
799807
"""
800808
try:
801-
data = signing.Signer(salt=PULL_THROUGH_SALT).unsign_object(token)
809+
data = _pull_through_signer(remote).unsign_object(token)
802810
except signing.BadSignature:
803811
return None
804-
if data.get("r") != str(remote.pk):
805-
return None
806812
if filename == data["f"]:
807813
return data["u"], data.get("s")
808814
if filename == f"{data['f']}.metadata" and data.get("m"):

‎pulp_python/tests/functional/api/test_full_mirror.py‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717

1818

1919
@pytest.fixture
20-
def signed_links(pulp_settings):
20+
def signed_links(redis_status):
2121
"""Pull-through links carry a signed token only when Redis is not available."""
22-
return not pulp_settings.CACHE_ENABLED
22+
return not redis_status
2323

2424

2525
def test_pull_through_install(
@@ -363,6 +363,36 @@ def test_pull_through_remote_update(
363363
assert sha256(r.content).hexdigest() == package2.digests["sha256"]
364364

365365

366+
@pytest.mark.parallel
367+
def test_pull_through_signed_links_remote_update(
368+
python_bindings, python_remote_factory, python_distribution_factory, monitor_task, signed_links
369+
):
370+
"""Tests that changing what the remote lists invalidates its signed pull-through links."""
371+
if not signed_links:
372+
pytest.skip("Pull-through links are only signed when Redis is not available")
373+
remote = python_remote_factory(includes=["shelf-reader"])
374+
distro = python_distribution_factory(remote=remote.pulp_href)
375+
376+
url = f"{distro.base_url}simple/shelf-reader/"
377+
package1, package2 = ProjectPage.from_response(requests.get(url), "shelf-reader").packages
378+
379+
# Other edits keep the links working
380+
body = {"download_concurrency": 5}
381+
monitor_task(python_bindings.RemotesPythonApi.partial_update(remote.pulp_href, body).task)
382+
r = requests.get(package1.url)
383+
assert r.status_code == 200
384+
assert sha256(r.content).hexdigest() == package1.digests["sha256"]
385+
386+
body = {"includes": ["shelf-reader", "aiohttp"]}
387+
monitor_task(python_bindings.RemotesPythonApi.partial_update(remote.pulp_href, body).task)
388+
assert requests.get(package2.url).status_code == 404
389+
packages = ProjectPage.from_response(requests.get(url), "shelf-reader").packages
390+
package2 = next(p for p in packages if p.filename == package2.filename)
391+
r = requests.get(package2.url)
392+
assert r.status_code == 200
393+
assert sha256(r.content).hexdigest() == package2.digests["sha256"]
394+
395+
366396
@pytest.mark.parallel
367397
def test_pull_through_metadata_with_repo(
368398
python_repo_factory,

‎pulp_python/tests/functional/api/test_simple_cache.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@
1313

1414

1515
@pytest.fixture
16-
def skip_without_cache(pulp_settings):
16+
def skip_without_cache(redis_status):
1717
"""
18-
Skip test if server-side caching is not enabled.
18+
Skip test if server-side caching is not enabled or Redis is unreachable.
1919
"""
20-
if not pulp_settings.CACHE_ENABLED:
21-
pytest.skip("CACHE_ENABLED is not set")
20+
if not redis_status:
21+
pytest.skip("Redis is not available")
2222

2323

2424
@pytest.fixture

‎pulp_python/tests/unit/test_pull_through.py‎

Lines changed: 44 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,26 @@
2020
URL = f"https://files.example.com/packages/ab/cd/{FILENAME}"
2121

2222

23+
def _remote(**kwargs):
24+
kwargs.setdefault("pk", uuid4())
25+
kwargs.setdefault("url", "https://pypi.org/")
26+
kwargs.setdefault("includes", [])
27+
kwargs.setdefault("excludes", [])
28+
return SimpleNamespace(**kwargs)
29+
30+
31+
def _redis_available():
32+
if not settings.CACHE_ENABLED:
33+
return False
34+
try:
35+
return SyncContentCache().redis.ping()
36+
except Exception:
37+
return False
38+
39+
40+
requires_redis = pytest.mark.skipif(not _redis_available(), reason="Redis is not available")
41+
42+
2343
def _token(remote, **kwargs):
2444
path = sign_pull_through_path(remote, FILENAME, URL, **kwargs)
2545
filename, token = path.split(f"?{PULL_THROUGH_PARAM}=")
@@ -28,7 +48,7 @@ def _token(remote, **kwargs):
2848

2949

3050
def test_round_trip():
31-
remote = SimpleNamespace(pk=uuid4())
51+
remote = _remote()
3252
token = _token(remote, sha256="a" * 64, metadata_sha256="b" * 64)
3353

3454
assert unsign_pull_through_token(remote, token, FILENAME) == (URL, "a" * 64)
@@ -39,37 +59,50 @@ def test_round_trip():
3959

4060

4161
def test_metadata_requires_signed_digest():
42-
remote = SimpleNamespace(pk=uuid4())
62+
remote = _remote()
4363
token = _token(remote, sha256="a" * 64)
4464

4565
assert unsign_pull_through_token(remote, token, f"{FILENAME}.metadata") is None
4666

4767

4868
def test_rejects_other_remote():
49-
token = _token(SimpleNamespace(pk=uuid4()), sha256="a" * 64)
69+
token = _token(_remote(), sha256="a" * 64)
5070

51-
assert unsign_pull_through_token(SimpleNamespace(pk=uuid4()), token, FILENAME) is None
71+
assert unsign_pull_through_token(_remote(), token, FILENAME) is None
72+
73+
74+
@pytest.mark.parametrize(
75+
"changes",
76+
[{"url": "https://example.com/"}, {"includes": ["shelf-reader"]}, {"excludes": ["django"]}],
77+
)
78+
def test_rejects_changed_remote(changes):
79+
remote = _remote()
80+
token = _token(remote, sha256="a" * 64)
81+
82+
assert (
83+
unsign_pull_through_token(_remote(**{**vars(remote), **changes}), token, FILENAME) is None
84+
)
5285

5386

5487
def test_rejects_other_filename():
55-
remote = SimpleNamespace(pk=uuid4())
88+
remote = _remote()
5689
token = _token(remote, sha256="a" * 64)
5790

5891
assert unsign_pull_through_token(remote, token, "evil-1.0-py3-none-any.whl") is None
5992

6093

6194
def test_rejects_tampered_token():
62-
remote = SimpleNamespace(pk=uuid4())
95+
remote = _remote()
6396
token = _token(remote, sha256="a" * 64)
6497
tampered = token[:-1] + ("A" if token[-1] != "A" else "B")
6598

6699
assert unsign_pull_through_token(remote, tampered, FILENAME) is None
67100
assert unsign_pull_through_token(remote, "not-a-token", FILENAME) is None
68101

69102

70-
@pytest.mark.skipif(not settings.CACHE_ENABLED, reason="CACHE_ENABLED is not set")
103+
@requires_redis
71104
def test_stored_entries():
72-
remote = SimpleNamespace(pk=uuid4())
105+
remote = _remote()
73106
sdist = "shelf-reader-0.1.tar.gz"
74107
legacy = "pytz-2011k.tar.bz2"
75108
entry = {"url": URL, "sha256": "a" * 64, "metadata_sha256": None}
@@ -83,7 +116,7 @@ def test_stored_entries():
83116
assert lookup_pull_through_entry(remote, sdist) == (f"{URL}/{sdist}", None)
84117
assert lookup_pull_through_entry(remote, legacy) == (f"{URL}/{legacy}", None)
85118
assert lookup_pull_through_entry(remote, "evil-1.0-py3-none-any.whl") is None
86-
assert lookup_pull_through_entry(SimpleNamespace(pk=uuid4()), FILENAME) is None
119+
assert lookup_pull_through_entry(_remote(), FILENAME) is None
87120

88121
entry["metadata_sha256"] = "b" * 64
89122
assert store_pull_through_entries(remote, {FILENAME: entry})
@@ -98,11 +131,11 @@ def test_stored_entries():
98131
assert lookup_pull_through_entry(remote, legacy) is None
99132

100133

101-
@pytest.mark.skipif(not settings.CACHE_ENABLED, reason="CACHE_ENABLED is not set")
134+
@requires_redis
102135
@pytest.mark.parametrize("ttl", [1200, None])
103136
def test_stored_entries_ttl(monkeypatch, ttl):
104137
monkeypatch.setattr(utils, "PULL_THROUGH_TTL", ttl)
105-
remote = SimpleNamespace(pk=uuid4())
138+
remote = _remote()
106139
assert store_pull_through_entries(remote, {FILENAME: {"url": URL}})
107140

108141
redis_ttl = SyncContentCache().redis.ttl(utils._pull_through_key(remote, FILENAME))

‎template_config.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ core_import_allowed: []
1818
deploy_client_to_pypi: true
1919
deploy_client_to_rubygems: true
2020
deploy_to_pypi: true
21-
disabled_redis_runners: []
21+
disabled_redis_runners:
22+
- "s3"
2223
docker_fixtures: false
2324
extra_files: []
2425
github_org: "pulp"

0 commit comments

Comments
 (0)