diff --git a/CHANGES/1276.feature b/CHANGES/1276.feature new file mode 100644 index 000000000..e892cf727 --- /dev/null +++ b/CHANGES/1276.feature @@ -0,0 +1 @@ +Added optional support for syncing DEP11 metadata files if available in the source repo. \ No newline at end of file diff --git a/pulp_deb/app/migrations/0045_aptremote_sync_dep11.py b/pulp_deb/app/migrations/0045_aptremote_sync_dep11.py new file mode 100644 index 000000000..8aae01213 --- /dev/null +++ b/pulp_deb/app/migrations/0045_aptremote_sync_dep11.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.18 on 2026-10-12 13:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('deb', '0044_remove_distributedpublication'), + ] + + operations = [ + migrations.AddField( + model_name='aptremote', + name='sync_dep11', + field=models.BooleanField(default=False), + ), + ] diff --git a/pulp_deb/app/models/remote.py b/pulp_deb/app/models/remote.py index cbc439db0..3d719debe 100644 --- a/pulp_deb/app/models/remote.py +++ b/pulp_deb/app/models/remote.py @@ -17,6 +17,7 @@ class AptRemote(Remote, AutoAddObjPermsMixin): sync_sources = models.BooleanField(default=False) sync_udebs = models.BooleanField(default=False) sync_installer = models.BooleanField(default=False) + sync_dep11 = models.BooleanField(default=False) gpgkey = models.TextField(null=True) ignore_missing_package_indices = models.BooleanField(default=False) excluded_package_metadata_fields = ArrayField(models.TextField(), default=list) diff --git a/pulp_deb/app/serializers/remote_serializers.py b/pulp_deb/app/serializers/remote_serializers.py index 05dfb5206..e1dbea794 100644 --- a/pulp_deb/app/serializers/remote_serializers.py +++ b/pulp_deb/app/serializers/remote_serializers.py @@ -57,6 +57,8 @@ class AptRemoteSerializer(RemoteSerializer): sync_installer = BooleanField(help_text="Sync installer files", required=False) + sync_dep11 = BooleanField(help_text="Sync dep11 metadata files", required=False) + gpgkey = CharField( help_text="Gpg public key to verify origin releases against", required=False, @@ -100,6 +102,7 @@ class Meta: "sync_sources", "sync_udebs", "sync_installer", + "sync_dep11", "gpgkey", "ignore_missing_package_indices", "excluded_package_metadata_fields", diff --git a/pulp_deb/app/tasks/publishing.py b/pulp_deb/app/tasks/publishing.py index 280b44664..4d65e1303 100644 --- a/pulp_deb/app/tasks/publishing.py +++ b/pulp_deb/app/tasks/publishing.py @@ -1,4 +1,5 @@ import asyncio +import gzip import logging import os import random @@ -8,7 +9,6 @@ from contextlib import suppress from datetime import datetime, timezone from gettext import gettext as _ -from gzip import GzipFile from pathlib import Path from debian import deb822 @@ -20,6 +20,7 @@ from pulpcore.plugin.models import ( Artifact, + ContentArtifact, PublishedArtifact, PublishedMetadata, RemoteArtifact, @@ -40,6 +41,7 @@ AptPublication, AptReleaseSigningService, AptRepository, + GenericContent, Package, PackageReleaseComponent, Release, @@ -302,8 +304,19 @@ def publish( release=release, temp_dir=temp_dir, signing_service=signing_service, + dep11_file_paths=[], ) + # TODO: do we need to check for sync_dep11 option? + # The problem is that if sync_dep11 is disable when syncing, publishing obviously won't + # include any dep11 files. But when they were synced in the past und sync_dep11 disabled afterwards + # dep11 files are still present and thus being published. + dep11_files = GenericContent.objects.filter( + pk__in=repo_version.content.order_by("-pulp_created"), + relative_path__contains="/dep11/", + ) + publish_dep11_metadata(dep11_files, release_helper, publication) + package_release_components = PackageReleaseComponent.objects.filter( pk__in=repo_version.content.order_by("-pulp_created"), release_component__in=release_components_filtered, @@ -408,6 +421,12 @@ def __init__(self, parent, component): source_index_path, ) + # DEP11 directory + self.dep11_dir = os.path.join( + "dists", self.parent.dists_subfolder, self.plain_component, "dep11" + ) + os.makedirs(self.dep11_dir, exist_ok=True) + def add_packages(self, package_pairs, artifact_dict, remote_artifact_dict): published_artifacts = [] @@ -611,6 +630,7 @@ def __init__( release, temp_dir, signing_service=None, + dep11_file_paths=None, ): self._release = release self.publication = publication @@ -653,6 +673,7 @@ def __init__( self.architectures = architectures self.components = {component: _ComponentHelper(self, component) for component in components} self.signing_service = publication.signing_service or signing_service + self.dep11_file_paths = dep11_file_paths def add_metadata(self, metadata): artifact = metadata._artifacts.get() @@ -718,7 +739,7 @@ def save_signed_metadata(self): def _zip_file(file_path): gz_file_path = file_path + ".gz" with open(file_path, "rb") as f_in: - with GzipFile(gz_file_path, "wb") as f_out: + with gzip.GzipFile(gz_file_path, "wb") as f_out: shutil.copyfileobj(f_in, f_out) return gz_file_path @@ -781,3 +802,47 @@ def _write_legacy_release_file(component, arch): with open(rel_path, "wb") as fh: rel.dump(fh) return rel_path + + +def publish_dep11_metadata(dep11_files, release_helper, publication): + log.debug("publish(): looking for dep11 files ...") + + for dep11_file in dep11_files: + release_helper.dep11_file_paths.append(dep11_file.relative_path) + # make sure that there actually are artifacts for dep11 files + try: + artifact = ContentArtifact.objects.get(content_id=dep11_file.content_ptr_id) + except Exception as e: + log.warning(f"DEP11: artifact not found for {dep11_file}: {e}, skipping") + continue + + artifact_path = f"{settings.MEDIA_ROOT}/{artifact.artifact.file}" + dep11_metadata = PublishedMetadata.create_from_file( + publication=publication, + file=File(open(artifact_path, "rb")), + relative_path=dep11_file.relative_path, + ) + dep11_metadata.save() + release_helper.add_metadata(dep11_metadata) + + # this is a "hack" because we need a mention of the + # uncompressed files in the Release file, + # for Appstream to find them + # We normally don't care about the artifact of the + # uncompressed files but every logic like + # sync and publish relies on the availability of an artifact. + # We also need to decompress those files to avoid hash mismatch errors + if "CID-Index" not in dep11_file.relative_path: + if Path(dep11_file.relative_path).suffix == ".gz": + dep11_file_uncompressed = Path(dep11_file.relative_path).with_suffix("") + with gzip.open(artifact_path, "rb") as f_in: + with open(dep11_file_uncompressed, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + + dep11_metadata_uncompressed = PublishedMetadata.create_from_file( + publication=publication, + file=File(open(dep11_file_uncompressed, "rb")), + relative_path=dep11_file_uncompressed, + ) + dep11_metadata_uncompressed.save() + release_helper.add_metadata(dep11_metadata_uncompressed) diff --git a/pulp_deb/app/tasks/synchronizing.py b/pulp_deb/app/tasks/synchronizing.py index 25d20c1e1..6f73e6308 100644 --- a/pulp_deb/app/tasks/synchronizing.py +++ b/pulp_deb/app/tasks/synchronizing.py @@ -813,6 +813,12 @@ async def _handle_component( pending_tasks.extend( [self._handle_source_index(release_file, release_component, file_references)] ) + + if self.remote.sync_dep11: + pending_tasks.extend( + [self._handle_dep11_files(release_file, release_component, file_references)] + ) + await asyncio.gather(*pending_tasks) async def _handle_flat_repo( @@ -845,6 +851,60 @@ async def _handle_flat_repo( # Await all tasks await asyncio.gather(*pending_tasks) + async def _handle_dep11_files(self, release_file, release_component, file_references): + dep11_dir = os.path.join(release_component.plain_component, "dep11") + paths = [path for path in file_references.keys() if path.startswith(dep11_dir)] + + log.info("sync: handle dep11 sync") + + if paths: + # CID-Index-amd64.json.gz is missing in file_references (not in Release file) + # Inject it manually? + dep11s = {} + supported_artifacts = [ + "CID-Index-amd64.json.gz", + "Components-amd64.yml.gz", + "Components-amd64.yml.xz", + "icons-48x48.tar.gz", + "icons-48x48@2.tar.gz", + "icons-64x64.tar.gz", + "icons-64x64@2.tar.gz", + "icons-128x128.tar.gz", + "icons-128x128@2.tar.gz", + ] + + for path in paths: + relative_path = os.path.join(os.path.dirname(release_file.relative_path), path) + basename = os.path.basename(relative_path) + + if basename not in supported_artifacts: + log.warning(f"DEP11: {basename} is not in supported artifacts, skipping") + continue + + d_artifact = self._to_d_artifact(relative_path, file_references[path]) + key = relative_path + + if key not in dep11s: + sha256 = d_artifact.artifact.sha256 + dep11s[key] = {"sha256": sha256, "d_artifacts": []} + log.warning(f"_handle_dep11_files: adding key={key}, sha256={sha256}") + + dep11s[key]["d_artifacts"].append(d_artifact) + + # handle CID-Index-amd64.json.gz separately + # because it is not listed in upstream Release file + cid_file_path = os.path.join( + os.path.dirname(release_file.relative_path), dep11_dir, "CID-Index-amd64.json.gz" + ) + artifact = self._to_d_artifact(cid_file_path) + dep11s[cid_file_path] = {"sha256": artifact.artifact.sha256, "d_artifacts": []} + dep11s[cid_file_path]["d_artifacts"].append(artifact) + for relative_path, dep11 in dep11s.items(): + content_unit = GenericContent(sha256=dep11["sha256"], relative_path=relative_path) + await self.put( + DeclarativeContent(content=content_unit, d_artifacts=dep11["d_artifacts"]) + ) + async def _handle_package_index( self, release_file,