allow PAP marking on upload and add PAP/lang/language filters (#418, …

requirements.in

@@ -18,7 +18,7 @@ playwright==1.58.0
 
 ## dogesec
 file2txt
-txt2stix[llms]>=1.7.0
+txt2stix[llms]>=1.7.1
 dogesec_commons[stixifier]
 stix2arango
 

requirements.txt

@@ -424,6 +424,8 @@ psycopg2-binary==2.9.11
     # via
     #   -r requirements.in
     #   dogesec-commons
+py3langid==0.3.0
+    # via txt2stix
 pyasn1==0.6.4
     # via pyasn1-modules
 pyasn1-modules==0.4.2
@@ -604,7 +606,7 @@ tqdm==4.67.3
     #   transformers
 transformers==4.57.6
     # via llama-index-llms-openai-like
-txt2stix[llms]==1.7.0
+txt2stix[llms]==1.7.1
     # via
     #   -r requirements.in
     #   dogesec-commons

stixify/web/migrations/0026_file_pap_level.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.12 on 2026-07-30 16:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stixify_core', '0025_alter_file_created'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='file',
+            name='pap_level',
+            field=models.CharField(blank=True, choices=[('red', 'Red'), ('amber', 'Amber'), ('green', 'Green'), ('clear', 'Clear'), ('white', 'White')], default=None, help_text='This will be assigned to all SDOs and SROs created. If no value passed, no PAP marking will be assigned.', max_length=16, null=True),
+        ),
+    ]

stixify/web/models.py

@@ -52,6 +52,23 @@ class TLP_Levels(models.TextChoices):
     TLP_Levels.AMBER_STRICT: "marking-definition--939a9414-2ddd-4d32-a0cd-375ea402b003",
 }
 
+
+class PAP_Levels(models.TextChoices):
+    RED = "red"
+    AMBER = "amber"
+    GREEN = "green"
+    CLEAR = "clear"
+    WHITE = "white"
+
+# https://github.com/oasis-open/cti-stix-common-objects/tree/main/extension-definition-specifications/pap-marking-definition-f8d
+PAP_LEVEL_STIX_ID_MAPPING = {
+    PAP_Levels.RED: "marking-definition--740d36e5-7714-4c30-961a-3ae632ceee0e",
+    PAP_Levels.AMBER: "marking-definition--60f8932b-e51e-4458-b265-a2e8be9a80ab",
+    PAP_Levels.GREEN: "marking-definition--c43594d1-4b11-4c59-93ab-1c9b14d53ce9",
+    PAP_Levels.CLEAR: "marking-definition--ad15a0cd-55b6-4588-a14c-a66105329b92",
+    PAP_Levels.WHITE: "marking-definition--a3bea94c-b469-41dc-9cfe-d6e7daba7730",
+}
+
 def create_report_id():
     return ""
 
@@ -72,6 +89,7 @@ def validate_identity(value):
 class CommonSTIXProps(models.Model):
     name = models.CharField(max_length=256, help_text="This will be used as the `name` value of the STIX Report object generated")
     tlp_level = models.CharField(choices=TLP_Levels.choices, default=TLP_Levels.RED, help_text="This will be assigned to all SDOs and SROs created. If no value passed, `TLP:Clear` will be assigned.")
+    pap_level = models.CharField(choices=PAP_Levels.choices, max_length=16, null=True, blank=True, default=None, help_text="This will be assigned to all SDOs and SROs created. If no value passed, no PAP marking will be assigned.")
     confidence = models.IntegerField(default=None, null=True)
 
 

stixify/web/views.py

@@ -44,11 +44,13 @@
 if typing.TYPE_CHECKING:
     from stixify import settings
 from .models import (
+    PAP_LEVEL_STIX_ID_MAPPING,
     TLP_LEVEL_STIX_ID_MAPPING,
     File,
     FileImage,
     Job,
     JobType,
+    PAP_Levels,
     TLP_Levels,
     JobState,
 )
@@ -277,6 +279,15 @@ class filterset_class(FilterSet):
             choices=File._meta.get_field("admiralty_information_credibility").choices,
             help_text="Filter Files by the Admiralty information credibility rating assigned to them (e.g. `1`).",
         )
+        pap_level = filters.ChoiceFilter(
+            choices=PAP_Levels.choices,
+            help_text="Filter Files by the PAP (Permissible Actions Protocol) marking assigned to them (e.g. `clear`).",
+        )
+        language = filters.CharFilter(
+            field_name="txt2stix_data__language",
+            lookup_expr="iexact",
+            help_text="Filter Files by the ISO 639-1 language code detected for the File content (found at `txt2stix_data.language`). e.g. `en`.",
+        )
 
         ai_describes_incident = filters.BooleanFilter(
             help_text="If `ai_content_check_provider` set in profile used to process report, AI will answer if file describes security incident. Default will show all reports, can filter those that only describe incident by setting to true."
@@ -707,6 +718,15 @@ def get_report(cls, report_id, request=None):
                 description="Filter the results by the Admiralty information credibility marking applied to the Report object (set at file upload time). Checks the `object_marking_refs` of the Report object for the marking definition `id` matching the rating selected.",
                 enum=[f[0] for f in File._meta.get_field("admiralty_information_credibility").choices],
             ),
+            OpenApiParameter(
+                "pap_level",
+                description="Filter the results by PAP (Permissible Actions Protocol) marking of the Report object (set at file upload time). Checks the `object_marking_refs` of the Report object for the marking definition `id` matching the level selected.",
+                enum=[f[0] for f in PAP_Levels.choices],
+            ),
+            OpenApiParameter(
+                "lang",
+                description="Filter the results by the `lang` property of the Report object. This is the ISO 639-1 language code detected for the report content (e.g. `en`).",
+            ),
             OpenApiParameter(
                 "description",
                 description="Filter by the content in a report `description` (which contains the markdown version of the report). Will search for descriptions that contain the value entered. Search is wildcard so `exploit` will match `exploited`, `exploits`, etc.",
@@ -913,6 +933,14 @@ def get_reports(self, id=None):
             bind_vars["admiralty_information_credibility_stix_id"] = ADMIRALTY_MARKING_MAPPING["INFORMATION"].get(rating)
             filters.append("FILTER @admiralty_information_credibility_stix_id IN doc.object_marking_refs")
 
+        if pap_level := helper.query.get("pap_level"):
+            bind_vars["pap_level_stix_id"] = PAP_LEVEL_STIX_ID_MAPPING.get(pap_level)
+            filters.append("FILTER @pap_level_stix_id IN doc.object_marking_refs")
+
+        if q := helper.query.get("lang"):
+            bind_vars["lang"] = q.lower()
+            filters.append("FILTER LOWER(doc.lang) == @lang")
+
         if q := helper.query.get("name"):
             bind_vars["name"] = q.lower()
             filters.append("FILTER CONTAINS(LOWER(doc.name), @name)")

stixify/worker/tasks.py

@@ -82,6 +82,7 @@ def process_post(job_id, *args):
                 external_references=external_refs,
                 admiralty_source_reliability=file.admiralty_source_reliability,
                 admiralty_information_credibility=file.admiralty_information_credibility,
+                pap_level=file.pap_level,
             ),
         )
         processor.setup(

tests/conftest.py

@@ -117,6 +117,8 @@ def more_files(stixifier_profile, identity):
             identity=identity,
             admiralty_source_reliability="A",
             admiralty_information_credibility="1",
+            pap_level="clear",
+            txt2stix_data={"language": "en"},
         ),
         models.File.objects.create(
             id="aadbe23d-192c-488d-8ce9-96aa2613453f",
@@ -128,6 +130,8 @@ def more_files(stixifier_profile, identity):
             identity=identity,
             admiralty_source_reliability="B",
             admiralty_information_credibility="1",
+            pap_level="amber",
+            txt2stix_data={"language": "fr"},
         ),
         models.File.objects.create(
             id="bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb",
@@ -142,5 +146,7 @@ def more_files(stixifier_profile, identity):
             identity=identity,
             admiralty_source_reliability="B",
             admiralty_information_credibility="3",
+            pap_level="amber",
+            txt2stix_data={"language": "en"},
         ),
     ]

tests/src/views/bundles.py

@@ -56,6 +56,7 @@
             "modified": "2022-08-11T15:18:11.499288Z",
             "name": "The original report",
             "description": "aexample.com.ng (13.59.11.21), located in Nigeria is compromised to Gather Victim Host Information",
+            "lang": "en",
             "published": "2022-08-11T15:18:11.499288Z",
             "object_refs": [
                 "indicator--dd695028-06bc-5a67-8f4c-b572916f925e",
@@ -84,6 +85,7 @@
                 "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487",
                 "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5",
                 "marking-definition--cf438540-077a-56c7-b68e-82fcc2bb0208",
+                "marking-definition--ad15a0cd-55b6-4588-a14c-a66105329b92",
             ],
         },
         {
@@ -259,6 +261,7 @@
             "modified": "2025-06-17T15:26:48.932465Z",
             "name": "This is another report",
             "description": "T1120 is followed by T1123. \nTarget is located in Nigeria and the red team of ak99za hacked the hospital using CVE-2025-19123\nncsc.gov.uk",
+            "lang": "fr",
             "published": "2025-06-17T15:26:48.932465Z",
             "object_refs": [
                 "indicator--21c8753d-a681-5159-949d-72d6b1fefb89",
@@ -286,7 +289,8 @@
             "object_marking_refs": [
                 "marking-definition--55d920b0-5e8b-4f79-9ee9-91f868d9b421",
                 "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5",
-                "marking-definition--9cf59b27-57f8-5250-98f4-16c462d5652c"
+                "marking-definition--9cf59b27-57f8-5250-98f4-16c462d5652c",
+                "marking-definition--60f8932b-e51e-4458-b265-a2e8be9a80ab"
             ]
         },
         {

tests/src/views/test_file_view.py

@@ -67,6 +67,56 @@ def test_create_with_custom_created(client, stixifier_profile, api_schema, ident
         api_schema['/api/v1/files/']['POST'].validate_response(Transport.get_st_response(resp))
 
 
+@pytest.mark.django_db
+def test_create_with_pap_level(client, stixifier_profile, api_schema, identity):
+    payload = dict(
+        file=SimpleUploadedFile(name="name.pdf", content=b"file content"),
+        profile_id=stixifier_profile.id,
+        identity_id=identity.id,
+        mode="md",
+        name="Upload test with PAP level",
+        report_id="report--567681d6-2817-4d84-84fb-87b2f059b92e",
+        pap_level="amber",
+    )
+    with (
+        patch(
+            "stixify.web.views.JobSerializer", side_effect=JobSerializer
+        ) as mock_job_serializer_cls,
+        patch("stixify.web.views.new_task") as mock_new_task,
+    ):
+        resp = client.post("/api/v1/files/", data=payload)
+        assert resp.status_code == 201, resp.content
+        file = models.File.objects.get(pk="567681d6-2817-4d84-84fb-87b2f059b92e")
+        assert file.pap_level == "amber"
+        assert resp.data["file"]["pap_level"] == "amber"
+        resp.wsgi_request.FILES.clear()
+        api_schema['/api/v1/files/']['POST'].validate_response(Transport.get_st_response(resp))
+
+
+@pytest.mark.django_db
+def test_create_without_pap_level(client, stixifier_profile, api_schema, identity):
+    payload = dict(
+        file=SimpleUploadedFile(name="name.pdf", content=b"file content"),
+        profile_id=stixifier_profile.id,
+        identity_id=identity.id,
+        mode="md",
+        name="Upload test without PAP level",
+        report_id="report--567681d6-2817-4d84-84fb-87b2f059b92e",
+    )
+    with (
+        patch(
+            "stixify.web.views.JobSerializer", side_effect=JobSerializer
+        ) as mock_job_serializer_cls,
+        patch("stixify.web.views.new_task") as mock_new_task,
+    ):
+        resp = client.post("/api/v1/files/", data=payload)
+        assert resp.status_code == 201, resp.content
+        file = models.File.objects.get(pk="567681d6-2817-4d84-84fb-87b2f059b92e")
+        assert file.pap_level is None
+        resp.wsgi_request.FILES.clear()
+        api_schema['/api/v1/files/']['POST'].validate_response(Transport.get_st_response(resp))
+
+
 @pytest.mark.django_db
 def test_patch_file_cannot_change_created(client, stixify_file, api_schema):
     original_created = stixify_file.created
@@ -403,6 +453,39 @@ def search_files(stixifier_profile, identity):
                 "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb",
             ],
         ),
+        (
+            dict(pap_level="clear"),
+            [
+                "f3848d80-b14d-4aa6-b3a6-94bce54b217e",
+            ],
+        ),
+        (
+            dict(pap_level="amber"),
+            [
+                "aadbe23d-192c-488d-8ce9-96aa2613453f",
+                "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb",
+            ],
+        ),
+        (
+            dict(language="en"),
+            [
+                "f3848d80-b14d-4aa6-b3a6-94bce54b217e",
+                "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb",
+            ],
+        ),
+        (
+            dict(language="EN"),
+            [
+                "f3848d80-b14d-4aa6-b3a6-94bce54b217e",
+                "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb",
+            ],
+        ),
+        (
+            dict(language="fr"),
+            [
+                "aadbe23d-192c-488d-8ce9-96aa2613453f",
+            ],
+        ),
     ],
 )
 @pytest.mark.django_db

tests/src/views/test_report_view.py

@@ -332,6 +332,24 @@ def test_report_objects_types(client, report_id, types, api_schema):
             dict(admiralty_source_reliability="A", name="rig"),
             ["report--52d2146c-798a-440f-942f-6fe039fb8995"],
         ),
+        (
+            dict(pap_level="clear"),
+            ["report--52d2146c-798a-440f-942f-6fe039fb8995"],
+        ),
+        (
+            dict(pap_level="amber"),
+            ["report--ed758a1b-34fe-4fca-8178-0c30d93a03ab"],
+        ),
+        (dict(pap_level="red"), []),
+        (
+            dict(lang="en"),
+            ["report--52d2146c-798a-440f-942f-6fe039fb8995"],
+        ),
+        (
+            dict(lang="FR"),
+            ["report--ed758a1b-34fe-4fca-8178-0c30d93a03ab"],
+        ),
+        (dict(lang="de"), []),
         (dict(labels="ploit"), ["report--ed758a1b-34fe-4fca-8178-0c30d93a03ab"]),
         (dict(labels="steal"), ["report--52d2146c-798a-440f-942f-6fe039fb8995"]),
         (