Add support to sync DEP11 metadata files

CHANGES/1276.feature

@@ -0,0 +1 @@
+Added optional support for syncing DEP11 metadata files if available in the source repo.

pulp_deb/app/migrations/0045_aptremote_sync_dep11.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.18 on 2026-10-12 13:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('deb', '0044_remove_distributedpublication'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='aptremote',
+            name='sync_dep11',
+            field=models.BooleanField(default=False),
+        ),
+    ]

pulp_deb/app/models/remote.py

@@ -17,6 +17,7 @@ class AptRemote(Remote, AutoAddObjPermsMixin):
     sync_sources = models.BooleanField(default=False)
     sync_udebs = models.BooleanField(default=False)
     sync_installer = models.BooleanField(default=False)
+    sync_dep11 = models.BooleanField(default=False)
     gpgkey = models.TextField(null=True)
     ignore_missing_package_indices = models.BooleanField(default=False)
     excluded_package_metadata_fields = ArrayField(models.TextField(), default=list)

pulp_deb/app/serializers/remote_serializers.py

@@ -57,6 +57,8 @@ class AptRemoteSerializer(RemoteSerializer):
 
     sync_installer = BooleanField(help_text="Sync installer files", required=False)
 
+    sync_dep11 = BooleanField(help_text="Sync dep11 metadata files", required=False)
+
     gpgkey = CharField(
         help_text="Gpg public key to verify origin releases against",
         required=False,
@@ -100,6 +102,7 @@ class Meta:
             "sync_sources",
             "sync_udebs",
             "sync_installer",
+            "sync_dep11",
             "gpgkey",
             "ignore_missing_package_indices",
             "excluded_package_metadata_fields",

pulp_deb/app/tasks/publishing.py

@@ -1,4 +1,5 @@
 import asyncio
+import gzip
 import logging
 import os
 import random
@@ -8,7 +9,6 @@
 from contextlib import suppress
 from datetime import datetime, timezone
 from gettext import gettext as _
-from gzip import GzipFile
 from pathlib import Path
 
 from debian import deb822
@@ -20,6 +20,7 @@
 
 from pulpcore.plugin.models import (
     Artifact,
+    ContentArtifact,
     PublishedArtifact,
     PublishedMetadata,
     RemoteArtifact,
@@ -40,6 +41,7 @@
     AptPublication,
     AptReleaseSigningService,
     AptRepository,
+    GenericContent,
     Package,
     PackageReleaseComponent,
     Release,
@@ -302,8 +304,19 @@ def publish(
                             release=release,
                             temp_dir=temp_dir,
                             signing_service=signing_service,
+                            dep11_file_paths=[],
                         )
 
+                        # TODO: do we need to check for sync_dep11 option?
+                        # The problem is that if sync_dep11 is disable when syncing, publishing obviously won't
+                        # include any dep11 files. But when they were synced in the past und sync_dep11 disabled afterwards
+                        # dep11 files are still present and thus being published.
+                        dep11_files = GenericContent.objects.filter(
+                            pk__in=repo_version.content.order_by("-pulp_created"),
+                            relative_path__contains="/dep11/",
+                        )
+                        publish_dep11_metadata(dep11_files, release_helper, publication)
+
                         package_release_components = PackageReleaseComponent.objects.filter(
                             pk__in=repo_version.content.order_by("-pulp_created"),
                             release_component__in=release_components_filtered,
@@ -408,6 +421,12 @@ def __init__(self, parent, component):
             source_index_path,
         )
 
+        # DEP11 directory
+        self.dep11_dir = os.path.join(
+            "dists", self.parent.dists_subfolder, self.plain_component, "dep11"
+        )
+        os.makedirs(self.dep11_dir, exist_ok=True)
+
     def add_packages(self, package_pairs, artifact_dict, remote_artifact_dict):
         published_artifacts = []
 
@@ -611,6 +630,7 @@ def __init__(
         release,
         temp_dir,
         signing_service=None,
+        dep11_file_paths=None,
     ):
         self._release = release
         self.publication = publication
@@ -653,6 +673,7 @@ def __init__(
         self.architectures = architectures
         self.components = {component: _ComponentHelper(self, component) for component in components}
         self.signing_service = publication.signing_service or signing_service
+        self.dep11_file_paths = dep11_file_paths
 
     def add_metadata(self, metadata):
         artifact = metadata._artifacts.get()
@@ -718,7 +739,7 @@ def save_signed_metadata(self):
 def _zip_file(file_path):
     gz_file_path = file_path + ".gz"
     with open(file_path, "rb") as f_in:
-        with GzipFile(gz_file_path, "wb") as f_out:
+        with gzip.GzipFile(gz_file_path, "wb") as f_out:
             shutil.copyfileobj(f_in, f_out)
     return gz_file_path
 
@@ -781,3 +802,47 @@ def _write_legacy_release_file(component, arch):
     with open(rel_path, "wb") as fh:
         rel.dump(fh)
     return rel_path
+
+
+def publish_dep11_metadata(dep11_files, release_helper, publication):
+    log.debug("publish(): looking for dep11 files ...")
+
+    for dep11_file in dep11_files:
+        release_helper.dep11_file_paths.append(dep11_file.relative_path)
+        # make sure that there actually are artifacts for dep11 files
+        try:
+            artifact = ContentArtifact.objects.get(content_id=dep11_file.content_ptr_id)
+        except Exception as e:
+            log.warning(f"DEP11: artifact not found for {dep11_file}: {e}, skipping")
+            continue
+
+        artifact_path = f"{settings.MEDIA_ROOT}/{artifact.artifact.file}"
+        dep11_metadata = PublishedMetadata.create_from_file(
+            publication=publication,
+            file=File(open(artifact_path, "rb")),
+            relative_path=dep11_file.relative_path,
+        )
+        dep11_metadata.save()
+        release_helper.add_metadata(dep11_metadata)
+
+        # this is a "hack" because we need a mention of the
+        # uncompressed files in the Release file,
+        # for Appstream to find them
+        # We normally don't care about the artifact of the
+        # uncompressed files but every logic like
+        # sync and publish relies on the availability of an artifact.
+        # We also need to decompress those files to avoid hash mismatch errors
+        if "CID-Index" not in dep11_file.relative_path:
+            if Path(dep11_file.relative_path).suffix == ".gz":
+                dep11_file_uncompressed = Path(dep11_file.relative_path).with_suffix("")
+                with gzip.open(artifact_path, "rb") as f_in:
+                    with open(dep11_file_uncompressed, "wb") as f_out:
+                        shutil.copyfileobj(f_in, f_out)
+
+                dep11_metadata_uncompressed = PublishedMetadata.create_from_file(
+                    publication=publication,
+                    file=File(open(dep11_file_uncompressed, "rb")),
+                    relative_path=dep11_file_uncompressed,
+                )
+                dep11_metadata_uncompressed.save()
+                release_helper.add_metadata(dep11_metadata_uncompressed)

pulp_deb/app/tasks/synchronizing.py

@@ -813,6 +813,12 @@ async def _handle_component(
             pending_tasks.extend(
                 [self._handle_source_index(release_file, release_component, file_references)]
             )
+
+        if self.remote.sync_dep11:
+            pending_tasks.extend(
+                [self._handle_dep11_files(release_file, release_component, file_references)]
+            )
+
         await asyncio.gather(*pending_tasks)
 
     async def _handle_flat_repo(
@@ -845,6 +851,60 @@ async def _handle_flat_repo(
         # Await all tasks
         await asyncio.gather(*pending_tasks)
 
+    async def _handle_dep11_files(self, release_file, release_component, file_references):
+        dep11_dir = os.path.join(release_component.plain_component, "dep11")
+        paths = [path for path in file_references.keys() if path.startswith(dep11_dir)]
+
+        log.info("sync: handle dep11 sync")
+
+        if paths:
+            # CID-Index-amd64.json.gz is missing in file_references (not in Release file)
+            # Inject it manually?
+            dep11s = {}
+            supported_artifacts = [
+                "CID-Index-amd64.json.gz",
+                "Components-amd64.yml.gz",
+                "Components-amd64.yml.xz",
+                "icons-48x48.tar.gz",
+                "icons-48x48@2.tar.gz",
+                "icons-64x64.tar.gz",
+                "icons-64x64@2.tar.gz",
+                "icons-128x128.tar.gz",
+                "icons-128x128@2.tar.gz",
+            ]
+
+            for path in paths:
+                relative_path = os.path.join(os.path.dirname(release_file.relative_path), path)
+                basename = os.path.basename(relative_path)
+
+                if basename not in supported_artifacts:
+                    log.warning(f"DEP11: {basename} is not in supported artifacts, skipping")
+                    continue
+
+                d_artifact = self._to_d_artifact(relative_path, file_references[path])
+                key = relative_path
+
+                if key not in dep11s:
+                    sha256 = d_artifact.artifact.sha256
+                    dep11s[key] = {"sha256": sha256, "d_artifacts": []}
+                    log.warning(f"_handle_dep11_files: adding key={key}, sha256={sha256}")
+
+                dep11s[key]["d_artifacts"].append(d_artifact)
+
+            # handle CID-Index-amd64.json.gz separately
+            # because it is not listed in upstream Release file
+            cid_file_path = os.path.join(
+                os.path.dirname(release_file.relative_path), dep11_dir, "CID-Index-amd64.json.gz"
+            )
+            artifact = self._to_d_artifact(cid_file_path)
+            dep11s[cid_file_path] = {"sha256": artifact.artifact.sha256, "d_artifacts": []}
+            dep11s[cid_file_path]["d_artifacts"].append(artifact)
+            for relative_path, dep11 in dep11s.items():
+                content_unit = GenericContent(sha256=dep11["sha256"], relative_path=relative_path)
+                await self.put(
+                    DeclarativeContent(content=content_unit, d_artifacts=dep11["d_artifacts"])
+                )
+
     async def _handle_package_index(
         self,
         release_file,