diff --git a/requirements.in b/requirements.in index 8a7f108..5a413a4 100644 --- a/requirements.in +++ b/requirements.in @@ -18,7 +18,7 @@ playwright==1.58.0 ## dogesec file2txt -txt2stix[llms]>=1.7.0 +txt2stix[llms]>=1.7.1 dogesec_commons[stixifier] stix2arango diff --git a/requirements.txt b/requirements.txt index 831ec15..bcb9d14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -424,6 +424,8 @@ psycopg2-binary==2.9.11 # via # -r requirements.in # dogesec-commons +py3langid==0.3.0 + # via txt2stix pyasn1==0.6.4 # via pyasn1-modules pyasn1-modules==0.4.2 @@ -604,7 +606,7 @@ tqdm==4.67.3 # transformers transformers==4.57.6 # via llama-index-llms-openai-like -txt2stix[llms]==1.7.0 +txt2stix[llms]==1.7.1 # via # -r requirements.in # dogesec-commons diff --git a/stixify/web/migrations/0026_file_pap_level.py b/stixify/web/migrations/0026_file_pap_level.py new file mode 100644 index 0000000..a37caa0 --- /dev/null +++ b/stixify/web/migrations/0026_file_pap_level.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.12 on 2026-07-30 16:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('stixify_core', '0025_alter_file_created'), + ] + + operations = [ + migrations.AddField( + model_name='file', + name='pap_level', + field=models.CharField(blank=True, choices=[('red', 'Red'), ('amber', 'Amber'), ('green', 'Green'), ('clear', 'Clear'), ('white', 'White')], default=None, help_text='This will be assigned to all SDOs and SROs created. If no value passed, no PAP marking will be assigned.', max_length=16, null=True), + ), + ] diff --git a/stixify/web/models.py b/stixify/web/models.py index 0cdc3b9..b9e7616 100644 --- a/stixify/web/models.py +++ b/stixify/web/models.py @@ -52,6 +52,23 @@ class TLP_Levels(models.TextChoices): TLP_Levels.AMBER_STRICT: "marking-definition--939a9414-2ddd-4d32-a0cd-375ea402b003", } + +class PAP_Levels(models.TextChoices): + RED = "red" + AMBER = "amber" + GREEN = "green" + CLEAR = "clear" + WHITE = "white" + +# https://github.com/oasis-open/cti-stix-common-objects/tree/main/extension-definition-specifications/pap-marking-definition-f8d +PAP_LEVEL_STIX_ID_MAPPING = { + PAP_Levels.RED: "marking-definition--740d36e5-7714-4c30-961a-3ae632ceee0e", + PAP_Levels.AMBER: "marking-definition--60f8932b-e51e-4458-b265-a2e8be9a80ab", + PAP_Levels.GREEN: "marking-definition--c43594d1-4b11-4c59-93ab-1c9b14d53ce9", + PAP_Levels.CLEAR: "marking-definition--ad15a0cd-55b6-4588-a14c-a66105329b92", + PAP_Levels.WHITE: "marking-definition--a3bea94c-b469-41dc-9cfe-d6e7daba7730", +} + def create_report_id(): return "" @@ -72,6 +89,7 @@ def validate_identity(value): class CommonSTIXProps(models.Model): name = models.CharField(max_length=256, help_text="This will be used as the `name` value of the STIX Report object generated") tlp_level = models.CharField(choices=TLP_Levels.choices, default=TLP_Levels.RED, help_text="This will be assigned to all SDOs and SROs created. If no value passed, `TLP:Clear` will be assigned.") + pap_level = models.CharField(choices=PAP_Levels.choices, max_length=16, null=True, blank=True, default=None, help_text="This will be assigned to all SDOs and SROs created. If no value passed, no PAP marking will be assigned.") confidence = models.IntegerField(default=None, null=True) diff --git a/stixify/web/views.py b/stixify/web/views.py index 189dc1d..8aaf5dc 100644 --- a/stixify/web/views.py +++ b/stixify/web/views.py @@ -44,11 +44,13 @@ if typing.TYPE_CHECKING: from stixify import settings from .models import ( + PAP_LEVEL_STIX_ID_MAPPING, TLP_LEVEL_STIX_ID_MAPPING, File, FileImage, Job, JobType, + PAP_Levels, TLP_Levels, JobState, ) @@ -277,6 +279,15 @@ class filterset_class(FilterSet): choices=File._meta.get_field("admiralty_information_credibility").choices, help_text="Filter Files by the Admiralty information credibility rating assigned to them (e.g. `1`).", ) + pap_level = filters.ChoiceFilter( + choices=PAP_Levels.choices, + help_text="Filter Files by the PAP (Permissible Actions Protocol) marking assigned to them (e.g. `clear`).", + ) + language = filters.CharFilter( + field_name="txt2stix_data__language", + lookup_expr="iexact", + help_text="Filter Files by the ISO 639-1 language code detected for the File content (found at `txt2stix_data.language`). e.g. `en`.", + ) ai_describes_incident = filters.BooleanFilter( help_text="If `ai_content_check_provider` set in profile used to process report, AI will answer if file describes security incident. Default will show all reports, can filter those that only describe incident by setting to true." @@ -707,6 +718,15 @@ def get_report(cls, report_id, request=None): description="Filter the results by the Admiralty information credibility marking applied to the Report object (set at file upload time). Checks the `object_marking_refs` of the Report object for the marking definition `id` matching the rating selected.", enum=[f[0] for f in File._meta.get_field("admiralty_information_credibility").choices], ), + OpenApiParameter( + "pap_level", + description="Filter the results by PAP (Permissible Actions Protocol) marking of the Report object (set at file upload time). Checks the `object_marking_refs` of the Report object for the marking definition `id` matching the level selected.", + enum=[f[0] for f in PAP_Levels.choices], + ), + OpenApiParameter( + "lang", + description="Filter the results by the `lang` property of the Report object. This is the ISO 639-1 language code detected for the report content (e.g. `en`).", + ), OpenApiParameter( "description", description="Filter by the content in a report `description` (which contains the markdown version of the report). Will search for descriptions that contain the value entered. Search is wildcard so `exploit` will match `exploited`, `exploits`, etc.", @@ -913,6 +933,14 @@ def get_reports(self, id=None): bind_vars["admiralty_information_credibility_stix_id"] = ADMIRALTY_MARKING_MAPPING["INFORMATION"].get(rating) filters.append("FILTER @admiralty_information_credibility_stix_id IN doc.object_marking_refs") + if pap_level := helper.query.get("pap_level"): + bind_vars["pap_level_stix_id"] = PAP_LEVEL_STIX_ID_MAPPING.get(pap_level) + filters.append("FILTER @pap_level_stix_id IN doc.object_marking_refs") + + if q := helper.query.get("lang"): + bind_vars["lang"] = q.lower() + filters.append("FILTER LOWER(doc.lang) == @lang") + if q := helper.query.get("name"): bind_vars["name"] = q.lower() filters.append("FILTER CONTAINS(LOWER(doc.name), @name)") diff --git a/stixify/worker/tasks.py b/stixify/worker/tasks.py index 863adfa..3f314ba 100644 --- a/stixify/worker/tasks.py +++ b/stixify/worker/tasks.py @@ -82,6 +82,7 @@ def process_post(job_id, *args): external_references=external_refs, admiralty_source_reliability=file.admiralty_source_reliability, admiralty_information_credibility=file.admiralty_information_credibility, + pap_level=file.pap_level, ), ) processor.setup( diff --git a/tests/conftest.py b/tests/conftest.py index c3a60e8..c381b18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -117,6 +117,8 @@ def more_files(stixifier_profile, identity): identity=identity, admiralty_source_reliability="A", admiralty_information_credibility="1", + pap_level="clear", + txt2stix_data={"language": "en"}, ), models.File.objects.create( id="aadbe23d-192c-488d-8ce9-96aa2613453f", @@ -128,6 +130,8 @@ def more_files(stixifier_profile, identity): identity=identity, admiralty_source_reliability="B", admiralty_information_credibility="1", + pap_level="amber", + txt2stix_data={"language": "fr"}, ), models.File.objects.create( id="bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb", @@ -142,5 +146,7 @@ def more_files(stixifier_profile, identity): identity=identity, admiralty_source_reliability="B", admiralty_information_credibility="3", + pap_level="amber", + txt2stix_data={"language": "en"}, ), ] diff --git a/tests/src/views/bundles.py b/tests/src/views/bundles.py index 908d846..fe2a2da 100644 --- a/tests/src/views/bundles.py +++ b/tests/src/views/bundles.py @@ -56,6 +56,7 @@ "modified": "2022-08-11T15:18:11.499288Z", "name": "The original report", "description": "aexample.com.ng (13.59.11.21), located in Nigeria is compromised to Gather Victim Host Information", + "lang": "en", "published": "2022-08-11T15:18:11.499288Z", "object_refs": [ "indicator--dd695028-06bc-5a67-8f4c-b572916f925e", @@ -84,6 +85,7 @@ "marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487", "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5", "marking-definition--cf438540-077a-56c7-b68e-82fcc2bb0208", + "marking-definition--ad15a0cd-55b6-4588-a14c-a66105329b92", ], }, { @@ -259,6 +261,7 @@ "modified": "2025-06-17T15:26:48.932465Z", "name": "This is another report", "description": "T1120 is followed by T1123. \nTarget is located in Nigeria and the red team of ak99za hacked the hospital using CVE-2025-19123\nncsc.gov.uk", + "lang": "fr", "published": "2025-06-17T15:26:48.932465Z", "object_refs": [ "indicator--21c8753d-a681-5159-949d-72d6b1fefb89", @@ -286,7 +289,8 @@ "object_marking_refs": [ "marking-definition--55d920b0-5e8b-4f79-9ee9-91f868d9b421", "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5", - "marking-definition--9cf59b27-57f8-5250-98f4-16c462d5652c" + "marking-definition--9cf59b27-57f8-5250-98f4-16c462d5652c", + "marking-definition--60f8932b-e51e-4458-b265-a2e8be9a80ab" ] }, { diff --git a/tests/src/views/test_file_view.py b/tests/src/views/test_file_view.py index ffb776f..e9d3d45 100644 --- a/tests/src/views/test_file_view.py +++ b/tests/src/views/test_file_view.py @@ -67,6 +67,56 @@ def test_create_with_custom_created(client, stixifier_profile, api_schema, ident api_schema['/api/v1/files/']['POST'].validate_response(Transport.get_st_response(resp)) +@pytest.mark.django_db +def test_create_with_pap_level(client, stixifier_profile, api_schema, identity): + payload = dict( + file=SimpleUploadedFile(name="name.pdf", content=b"file content"), + profile_id=stixifier_profile.id, + identity_id=identity.id, + mode="md", + name="Upload test with PAP level", + report_id="report--567681d6-2817-4d84-84fb-87b2f059b92e", + pap_level="amber", + ) + with ( + patch( + "stixify.web.views.JobSerializer", side_effect=JobSerializer + ) as mock_job_serializer_cls, + patch("stixify.web.views.new_task") as mock_new_task, + ): + resp = client.post("/api/v1/files/", data=payload) + assert resp.status_code == 201, resp.content + file = models.File.objects.get(pk="567681d6-2817-4d84-84fb-87b2f059b92e") + assert file.pap_level == "amber" + assert resp.data["file"]["pap_level"] == "amber" + resp.wsgi_request.FILES.clear() + api_schema['/api/v1/files/']['POST'].validate_response(Transport.get_st_response(resp)) + + +@pytest.mark.django_db +def test_create_without_pap_level(client, stixifier_profile, api_schema, identity): + payload = dict( + file=SimpleUploadedFile(name="name.pdf", content=b"file content"), + profile_id=stixifier_profile.id, + identity_id=identity.id, + mode="md", + name="Upload test without PAP level", + report_id="report--567681d6-2817-4d84-84fb-87b2f059b92e", + ) + with ( + patch( + "stixify.web.views.JobSerializer", side_effect=JobSerializer + ) as mock_job_serializer_cls, + patch("stixify.web.views.new_task") as mock_new_task, + ): + resp = client.post("/api/v1/files/", data=payload) + assert resp.status_code == 201, resp.content + file = models.File.objects.get(pk="567681d6-2817-4d84-84fb-87b2f059b92e") + assert file.pap_level is None + resp.wsgi_request.FILES.clear() + api_schema['/api/v1/files/']['POST'].validate_response(Transport.get_st_response(resp)) + + @pytest.mark.django_db def test_patch_file_cannot_change_created(client, stixify_file, api_schema): original_created = stixify_file.created @@ -403,6 +453,39 @@ def search_files(stixifier_profile, identity): "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb", ], ), + ( + dict(pap_level="clear"), + [ + "f3848d80-b14d-4aa6-b3a6-94bce54b217e", + ], + ), + ( + dict(pap_level="amber"), + [ + "aadbe23d-192c-488d-8ce9-96aa2613453f", + "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb", + ], + ), + ( + dict(language="en"), + [ + "f3848d80-b14d-4aa6-b3a6-94bce54b217e", + "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb", + ], + ), + ( + dict(language="EN"), + [ + "f3848d80-b14d-4aa6-b3a6-94bce54b217e", + "bd5c8992-e1f2-42ef-8ad2-8003bc4fcedb", + ], + ), + ( + dict(language="fr"), + [ + "aadbe23d-192c-488d-8ce9-96aa2613453f", + ], + ), ], ) @pytest.mark.django_db diff --git a/tests/src/views/test_report_view.py b/tests/src/views/test_report_view.py index f77b279..3783986 100644 --- a/tests/src/views/test_report_view.py +++ b/tests/src/views/test_report_view.py @@ -332,6 +332,24 @@ def test_report_objects_types(client, report_id, types, api_schema): dict(admiralty_source_reliability="A", name="rig"), ["report--52d2146c-798a-440f-942f-6fe039fb8995"], ), + ( + dict(pap_level="clear"), + ["report--52d2146c-798a-440f-942f-6fe039fb8995"], + ), + ( + dict(pap_level="amber"), + ["report--ed758a1b-34fe-4fca-8178-0c30d93a03ab"], + ), + (dict(pap_level="red"), []), + ( + dict(lang="en"), + ["report--52d2146c-798a-440f-942f-6fe039fb8995"], + ), + ( + dict(lang="FR"), + ["report--ed758a1b-34fe-4fca-8178-0c30d93a03ab"], + ), + (dict(lang="de"), []), (dict(labels="ploit"), ["report--ed758a1b-34fe-4fca-8178-0c30d93a03ab"]), (dict(labels="steal"), ["report--52d2146c-798a-440f-942f-6fe039fb8995"]), (