Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion CHANGES/1641.misc

This file was deleted.

1 change: 0 additions & 1 deletion CHANGES/2346.misc

This file was deleted.

3 changes: 0 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -186,9 +186,6 @@ RUN patch -p1 -d /usr/local/lib/pulp/lib/python${PYTHON_VERSION}/site-packages <
COPY images/assets/patches/0079-Log-Maven-path-index-fallback-reasons.patch /tmp/
RUN patch -p1 -d /usr/local/lib/pulp/lib/python${PYTHON_VERSION}/site-packages < /tmp/0079-Log-Maven-path-index-fallback-reasons.patch

COPY images/assets/patches/0080-Optimize-read-only-content-queries.patch /tmp/
RUN patch -p1 -d /usr/local/lib/pulp/lib/python${PYTHON_VERSION}/site-packages < /tmp/0080-Optimize-read-only-content-queries.patch

COPY images/assets/patches/0081-Fix-task-cancellation-during-graceful-shutdown.patch /tmp/
RUN patch -p1 -d /usr/local/lib/pulp/lib/python${PYTHON_VERSION}/site-packages < /tmp/0081-Fix-task-cancellation-during-graceful-shutdown.patch

Expand Down
101 changes: 0 additions & 101 deletions images/assets/patches/0080-Optimize-read-only-content-queries.patch

This file was deleted.

62 changes: 1 addition & 61 deletions pulp_service/pulp_service/app/experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,13 @@
import logging
import random
from time import perf_counter
from uuid import UUID

from django.conf import settings
from django.db import connections
from django.db.models import Exists, OuterRef, Q
from django.db.models import Q
from django_guid import get_guid

logger = logging.getLogger("pulp.experiment")
DIRECTORY_EXPERIMENT = "PULP-2505"
CONTENT_QUERY_EXPERIMENT = "PULP-2346"
CONTENT_QUERY_P_CANDIDATE = 0.5


def run_experiment(exp_id, control, candidate, *, p_candidate, context):
Expand Down Expand Up @@ -133,59 +129,3 @@ def directory_membership_dates(version, content_to_names, sources, *, serving_mo
p_candidate=probability,
context=context,
)


def _content_query_marker(variant, repository_id):
"""Return the PULP-2346 marker for a variant and repository."""
repository = UUID(str(repository_id))
return f"/* pexp={CONTENT_QUERY_EXPERIMENT} v={variant} */ /* repo={repository} */"


def mark_lazy_queryset(queryset, marker):
"""Embed an experiment marker in the SQL, so it also survives later evaluation (e.g. ``.values()``)."""
# The marker is built from constants and a UUID-validated repository id, never from user input.
return queryset.extra(where=[f"1 = 1 {marker}"]) # noqa: S610


def content_query_experiment(query, version, queryset, *, filters, limit=None):
"""Run one content app lookup as PULP-2346 and return ``(rows, queryset used)``.

Variant A is stock pulpcore (``content__in=version.content``). Variant B checks repository
membership per matching row with a correlated EXISTS. Both return the same rows. Always on:
half of the calls run variant B.
"""

def build(variant):
if variant == "A":
selected = queryset.filter(content__in=version.content, **filters)
else:
membership = version._content_relationships().filter(content_id=OuterRef("content_id"))
selected = queryset.filter(Exists(membership), **filters)
return mark_lazy_queryset(selected, _content_query_marker(variant, version.repository_id))

used = {}

def run(variant):
def evaluate():
used["queryset"] = build(variant)
return list(used["queryset"] if limit is None else used["queryset"][:limit])

return evaluate

rows = run_experiment(
CONTENT_QUERY_EXPERIMENT,
run("A"),
run("B"),
p_candidate=CONTENT_QUERY_P_CANDIDATE,
context={"query": query, "repository_id": str(version.repository_id)},
)
return rows, used["queryset"]


def single_or_raise(model, rows):
"""Return the only row, raising what ``QuerySet.get()`` would for none or several."""
if not rows:
raise model.DoesNotExist(f"{model.__name__} matching query does not exist.")
if len(rows) > 1:
raise model.MultipleObjectsReturned(f"get() returned more than one {model.__name__}.")
return rows[0]
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
from django.db import migrations

_INDEX_NAME = "core_ca_relpath_pattern_idx"
_INDEX_MARKER = "pulp-service:0024:core_ca_relpath_pattern_idx"
_DROP_INDEX_SQL = f'DROP INDEX CONCURRENTLY IF EXISTS "{_INDEX_NAME}"'


def remove_experiment_index(apps, schema_editor): # noqa: ARG001

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium · Release notes omit the cleanup

When towncrier builds release notes for this change, CHANGES/ has no fragment for this index-removal migration, so Towncrier omits the cleanup from the release notes.

Add a Towncrier fragment documenting the index cleanup.

Also at pulp_service/pulp_service/app/migrations/0025_remove_experiment_relative_path_index.py:35.

Prompt for AI agents
In `pulp_service/pulp_service/app/migrations/0025_remove_experiment_relative_path_index.py` at line 8:

**Release notes omit the cleanup**

When towncrier builds release notes for this change, `CHANGES/` has no fragment for this index-removal migration, so Towncrier omits the cleanup from the release notes.

Add a Towncrier fragment documenting the index cleanup.

Also at `pulp_service/pulp_service/app/migrations/0025_remove_experiment_relative_path_index.py:35`.

From CLAUDE.md L76:

Uses **towncrier**. For any non-trivial change, create a file in `CHANGES/` named `{issue_number}.{category}` where category is one of: `feature`, `bugfix`, `doc`, `removal`, `deprecation`, `misc`.

"""Remove only the index owned by the temporary pulp-service migration."""
with schema_editor.connection.cursor() as cursor:
cursor.execute(
"""
SELECT obj_description(indexrel.oid, 'pg_class')
FROM pg_class AS indexrel
JOIN pg_namespace AS namespace ON namespace.oid = indexrel.relnamespace
WHERE indexrel.relname = %s AND namespace.nspname = current_schema()
""",
[_INDEX_NAME],
)
row = cursor.fetchone()
if row and row[0] == _INDEX_MARKER:
cursor.execute(_DROP_INDEX_SQL)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium · Directory listings become slow

When a directory listing runs after this migration, especially on a large repository, remove_experiment_index drops the text_pattern_ops index while directory-listing queries still filter artifacts with relative_path__startswith; PostgreSQL scans the artifact tables and directory listings become slow.

Keep or add an index that supports the prefix queries for as long as directory listings use them.

Prompt for AI agents
In `pulp_service/pulp_service/app/migrations/0025_remove_experiment_relative_path_index.py` at line 22:

**Directory listings become slow**

When a directory listing runs after this migration, especially on a large repository, `remove_experiment_index` drops the `text_pattern_ops` index while directory-listing queries still filter artifacts with `relative_path__startswith`; PostgreSQL scans the artifact tables and directory listings become slow.

Keep or add an index that supports the prefix queries for as long as directory listings use them.

Comment thread
sourcery-ai[bot] marked this conversation as resolved.
Resolved

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

security (python.sqlalchemy.security.sqlalchemy-execute-raw-query): Avoiding SQL string concatenation: untrusted input concatenated with raw SQL query can result in SQL Injection. In order to execute raw query safely, prepared statement should be used. SQLAlchemy provides TextualSQL to easily used prepared statement with named parameters. For complex SQL composition, use SQL Expression Language or Schema Definition Language. In most cases, SQLAlchemy ORM will be a better option.

Source: opengrep



class Migration(migrations.Migration):
"""Remove the temporary index before pulpcore owns the replacement."""

atomic = False # DROP INDEX CONCURRENTLY cannot run in a transaction.

dependencies = [
("service", "0026_andgatecontentguard"),
]

operations = [
migrations.RunPython(remove_experiment_index, migrations.RunPython.noop),
]
157 changes: 0 additions & 157 deletions pulp_service/pulp_service/tests/unit/test_content_query_experiment.py

This file was deleted.

Loading