feat(models): add a default-queryset hook for plugins

dojo/db_migrations/0305_archive_batch_indexes.py

@@ -0,0 +1,60 @@
+from django.contrib.postgres.operations import AddIndexConcurrently
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    # CREATE INDEX CONCURRENTLY cannot run inside a transaction, and a plain
+    # CREATE INDEX would lock dojo_finding and dojo_endpoint_status for the whole build.
+    atomic = False
+
+    dependencies = [
+        ("dojo", "0304_archive_columns"),
+    ]
+
+    operations = [
+        AddIndexConcurrently(
+            model_name="findingvulnerabilityreference",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_findvulnref_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="endpoint_status",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_endpstatus_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="engagement",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_engagement_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="finding",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_finding_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="jira_issue",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_jiraissue_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="locationfindingreference",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_locfindref_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="product",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_product_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="product_type",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_prod_type_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="test",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_test_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="test_import_finding_action",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_tifa_archive_batch"),
+        ),
+        AddIndexConcurrently(
+            model_name="vulnerability_id",
+            index=models.Index(condition=models.Q(("archive_batch__isnull", False)), fields=["archive_batch"], name="dojo_vulnid_archive_batch"),
+        ),
+    ]

dojo/endpoint/models.py

@@ -20,6 +20,7 @@
 # before the re-export that loads this module — resolves despite partial circular load.
 # Must keep their dojo.models.* path for Django migration serialization.
 from dojo.models import copy_model_util, get_current_date, get_current_datetime
+from dojo.query_hooks import HookedManager, UnfilteredManager
 
 logger = logging.getLogger(__name__)
 
@@ -44,8 +45,17 @@ class Endpoint_Status(models.Model):
     endpoint = models.ForeignKey("dojo.Endpoint", null=False, blank=False, on_delete=models.CASCADE, related_name="status_endpoint")
     finding = models.ForeignKey("dojo.Finding", null=False, blank=False, on_delete=models.CASCADE, related_name="status_finding")
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         indexes = [
+            models.Index(fields=["archive_batch"], condition=models.Q(archive_batch__isnull=False), name="dojo_endpstatus_archive_batch"),
             models.Index(fields=["finding", "mitigated"]),
             models.Index(fields=["endpoint", "mitigated"]),
             # Optimize frequent lookups of "active" statuses (mitigated/flags all False)
@@ -141,6 +151,14 @@ class Endpoint(models.Model):
     tags = TagField(blank=True, force_lowercase=True, help_text=_("Add tags that help describe this endpoint. Choose from the list or add new tags. Press Enter key to add."))
     inherited_tags = TagField(blank=True, force_lowercase=True, help_text=_("Internal use tags sepcifically for maintaining parity with product. This field will be present as a subset in the tags field"))
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         ordering = ["product", "host", "protocol", "port", "userinfo", "path", "query", "fragment"]
         indexes = [

dojo/engagement/models.py

@@ -10,7 +10,8 @@
 from django.utils.translation import gettext as _
 from django_tagulous.models import TagField
 
-from dojo.base_models.base import BaseModel
+from dojo.base_models.base import BaseManager, BaseModel
+from dojo.query_hooks import HookedBaseManager, HookedManager, UnfilteredManager
 
 logger = logging.getLogger(__name__)
 
@@ -24,6 +25,14 @@ class Engagement_Presets(models.Model):
     product = models.ForeignKey("dojo.Product", on_delete=models.CASCADE)
     created = models.DateTimeField(auto_now_add=True, null=False)
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         ordering = ["title"]
 
@@ -94,9 +103,18 @@ class Engagement(BaseModel):
     tags = TagField(blank=True, force_lowercase=True, help_text=_("Add tags that help describe this engagement. Choose from the list or add new tags. Press Enter key to add."))
     inherited_tags = TagField(blank=True, force_lowercase=True, help_text=_("Internal use tags sepcifically for maintaining parity with product. This field will be present as a subset in the tags field"))
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedBaseManager()
+    unfiltered = BaseManager()
+
     class Meta:
         ordering = ["-target_start"]
         indexes = [
+            models.Index(fields=["archive_batch"], condition=models.Q(archive_batch__isnull=False), name="dojo_engagement_archive_batch"),
             models.Index(fields=["product", "active"]),
             # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
             GinIndex(

dojo/finding/models.py

@@ -24,14 +24,15 @@
 from django_tagulous.models import TagField
 from titlecase import titlecase
 
-from dojo.base_models.base import BaseModel
+from dojo.base_models.base import BaseManager, BaseModel
 from dojo.finding.cwe import cwe_label, finding_cwe_labels
 from dojo.finding.vulnerability_id import resolve_vulnerability_id_type
 
 # get_current_date/tomorrow/copy_model_util are defined early in dojo.models, before the
 # re-export that loads this module — so this resolves despite the partial circular load, and
 # keeps their dojo.models.* path for Django migration serialization (used as field defaults).
 from dojo.models import copy_model_util, get_current_date, tomorrow
+from dojo.query_hooks import HookedBaseManager, HookedManager, UnfilteredManager
 from dojo.validators import cvss3_validator, cvss4_validator
 
 if TYPE_CHECKING:
@@ -454,9 +455,18 @@ class Finding(BaseModel):
     SEVERITIES = {"Info": 4, "Low": 3, "Medium": 2,
                   "High": 1, "Critical": 0}
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedBaseManager()
+    unfiltered = BaseManager()
+
     class Meta:
         ordering = ("numerical_severity", "-date", "title", "epss_score", "epss_percentile")
         indexes = [
+            models.Index(fields=["archive_batch"], condition=models.Q(archive_batch__isnull=False), name="dojo_finding_archive_batch"),
             # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
             GinIndex(
                 SearchVector("title", weight="A", config="english")
@@ -1562,11 +1572,20 @@ class Vulnerability_Id(models.Model):
     # prefix. Denormalized/indexed so type-scoped queries (e.g. GROUP BY type) stay cheap.
     vulnerability_id_type = models.CharField(max_length=20, null=True, blank=True, editable=False, db_index=True)
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         constraints = [
             models.UniqueConstraint(fields=["finding", "vulnerability_id"], name="unique_finding_vulnerability_id"),
         ]
         indexes = [
+            models.Index(fields=["archive_batch"], condition=models.Q(archive_batch__isnull=False), name="dojo_vulnid_archive_batch"),
             # Leading on vulnerability_id (the unique constraint's index leads on finding), for the
             # vulnerability-id Explorer's GROUP BY vulnerability_id / lookups by exact id.
             models.Index(fields=["vulnerability_id"], name="dojo_vuln_id_lookup_idx"),
@@ -1635,6 +1654,14 @@ class Finding_Group(TimeStampedModel):
     findings = models.ManyToManyField("dojo.Finding")
     creator = models.ForeignKey("dojo.Dojo_User", on_delete=models.RESTRICT)
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     def __str__(self):
         return self.name
 

dojo/jira/models.py

@@ -5,6 +5,8 @@
 from django.db import models
 from django.utils.translation import gettext as _
 
+from dojo.query_hooks import HookedManager, UnfilteredManager
+
 
 class JIRA_Instance(models.Model):
     configuration_name = models.CharField(max_length=2000, help_text=_("Enter a name to give to this configuration"), default="")
@@ -139,6 +141,14 @@ class JIRA_Project(models.Model):
         default=True,
         blank=True)
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         app_label = "dojo"
 
@@ -197,7 +207,18 @@ class JIRA_Issue(models.Model):
                                        verbose_name=_("Jira last update"),
                                        help_text=_("The date the linked Jira issue was last modified."))
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
+        indexes = [
+            models.Index(fields=["archive_batch"], condition=models.Q(archive_batch__isnull=False), name="dojo_jiraissue_archive_batch"),
+        ]
         app_label = "dojo"
 
     def __str__(self):

dojo/location/manager.py

@@ -4,6 +4,7 @@
 
 from dojo.base_models.base import BaseManager, BaseQuerySet
 from dojo.location.status import rollup_finding_location_status
+from dojo.query_hooks import HookedManagerMixin
 
 
 class LocationQueryset(BaseQuerySet):
@@ -72,3 +73,13 @@ class LocationFindingReferenceManager(BaseManager):
     """LocationFindingReference manager to manipulate all objects with."""
 
     QUERY_SET_CLASS = LocationFindingReferenceQueryset
+
+
+class HookedLocationProductReferenceManager(HookedManagerMixin, LocationProductReferenceManager):
+
+    """LocationProductReference manager that applies the registered default-queryset filters."""
+
+
+class HookedLocationFindingReferenceManager(HookedManagerMixin, LocationFindingReferenceManager):
+
+    """LocationFindingReference manager that applies the registered default-queryset filters."""

dojo/location/models.py

@@ -23,13 +23,16 @@
     QuerySet,
     TextChoices,
     UniqueConstraint,
+    UUIDField,
 )
 from django.utils.translation import gettext_lazy as _
 from django_tagulous.models import TagField
 
 from dojo.base_models.base import BaseModel, BaseModelWithoutTimeMeta
 from dojo.base_models.validators import validate_not_empty
 from dojo.location.manager import (
+    HookedLocationFindingReferenceManager,
+    HookedLocationProductReferenceManager,
     LocationFindingReferenceManager,
     LocationFindingReferenceQueryset,
     LocationManager,
@@ -654,7 +657,13 @@ class LocationFindingReference(BaseModel, ReferenceDataMixin):
         help_text=_("The status of the the given Location"),
     )
 
-    objects = LocationFindingReferenceManager().from_queryset(LocationFindingReferenceQueryset)()
+    # Read and written only by plugins; open source never sets them.
+    archived_at = DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedLocationFindingReferenceManager().from_queryset(LocationFindingReferenceQueryset)()
+    unfiltered = LocationFindingReferenceManager().from_queryset(LocationFindingReferenceQueryset)()
 
     class Meta:
         verbose_name = "Locations - FindingReference"
@@ -666,6 +675,7 @@ class Meta:
             ),
         ]
         indexes = [
+            Index(fields=["archive_batch"], condition=Q(archive_batch__isnull=False), name="dojo_locfindref_archive_batch"),
             Index(fields=["location"]),
             Index(fields=["finding"]),
         ]
@@ -717,7 +727,13 @@ class LocationProductReference(BaseModel, ReferenceDataMixin):
         help_text=_("The status of the the given Location"),
     )
 
-    objects = LocationProductReferenceManager().from_queryset(LocationProductReferenceQueryset)()
+    # Read and written only by plugins; open source never sets them.
+    archived_at = DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedLocationProductReferenceManager().from_queryset(LocationProductReferenceQueryset)()
+    unfiltered = LocationProductReferenceManager().from_queryset(LocationProductReferenceQueryset)()
 
     class Meta:
         verbose_name = "Locations - ProductReference"

dojo/models.py

@@ -173,6 +173,7 @@ def get_current_datetime():
     Product_Line,  # noqa: F401 -- re-export
 )
 from dojo.product_type.models import Product_Type  # noqa: E402, F401 -- re-export
+from dojo.query_hooks import HookedManager, UnfilteredManager  # noqa: E402
 from dojo.reports.models import Report_Type  # noqa: E402, F401 -- re-export
 from dojo.test.models import (  # noqa: E402 -- re-export; class-body FKs below reference these
     IMPORT_ACTIONS,  # noqa: F401 -- re-export
@@ -218,6 +219,14 @@ class DojoMeta(models.Model):
                                  editable=False,
                                  related_name="location_meta_scope")
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         unique_together = (("product", "name"),
                            ("endpoint", "name"),
@@ -545,6 +554,14 @@ class Languages(models.Model):
     code = models.IntegerField(blank=True, null=True, verbose_name=_("Number of code lines"))
     created = models.DateTimeField(auto_now_add=True, null=False)
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         unique_together = [("language", "product")]
 
@@ -565,6 +582,14 @@ class App_Analysis(models.Model):
 
     tags = TagField(blank=True, force_lowercase=True)
 
+    # Read and written only by plugins; open source never sets them.
+    archived_at = models.DateTimeField(null=True, blank=True, editable=False)
+    archive_batch = models.UUIDField(null=True, blank=True, editable=False)
+
+    # objects is declared first, so it is the default manager; see dojo.query_hooks.
+    objects = HookedManager()
+    unfiltered = UnfilteredManager()
+
     class Meta:
         indexes = [
             # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.