From 0d6676a2b202690aa0de99636a5ae2262feed74c Mon Sep 17 00:00:00 2001 From: Derek Date: Sun, 15 Nov 2026 07:50:06 +1100 Subject: [PATCH 1/2] fix(capture): stop seeding the loader's disable lists hyperi-io/dfe-loader#347 drops disable_json_tables and disable_raw_tables from the loader, so both loader seeds stop writing them. The loader acknowledgement fixture is the loader branch's docs/config-schema.json, copied byte for byte. The comment names the matching commit. tests/fixtures/contract/dfe-loader still matches the pinned loader release. The fixture changes with the pin once a loader release has the change. --- .../default_configs/loader-default.yaml | 2 - .../default_configs/loader-production.yaml | 2 - .../dfe-loader/config-schema.json | 837 ++++++++++++++---- tests/unit/test_appmgmt/test_contract.py | 2 +- 4 files changed, 687 insertions(+), 156 deletions(-) diff --git a/src/dfe_engine/services/default_configs/loader-default.yaml b/src/dfe_engine/services/default_configs/loader-default.yaml index 605e83793..25796dedb 100644 --- a/src/dfe_engine/services/default_configs/loader-default.yaml +++ b/src/dfe_engine/services/default_configs/loader-default.yaml @@ -93,8 +93,6 @@ metadata: capture_raw: true raw_source_fields: [] raw_output: _raw - disable_json_tables: [] - disable_raw_tables: [] remove_routing_fields: true schema: diff --git a/src/dfe_engine/services/default_configs/loader-production.yaml b/src/dfe_engine/services/default_configs/loader-production.yaml index f18ab5884..cbd1d1c05 100644 --- a/src/dfe_engine/services/default_configs/loader-production.yaml +++ b/src/dfe_engine/services/default_configs/loader-production.yaml @@ -104,8 +104,6 @@ metadata: capture_raw: true raw_source_fields: [] raw_output: _raw - disable_json_tables: [] - disable_raw_tables: [] remove_routing_fields: true schema: diff --git a/tests/fixtures/contract-acknowledgements/dfe-loader/config-schema.json b/tests/fixtures/contract-acknowledgements/dfe-loader/config-schema.json index d59da693d..fd06f5edd 100644 --- a/tests/fixtures/contract-acknowledgements/dfe-loader/config-schema.json +++ b/tests/fixtures/contract-acknowledgements/dfe-loader/config-schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "Config", - "description": "Main configuration for dfe-loader.\n\n## Hot-Reload Behaviour\n\n**Hot-reloaded (takes effect on next batch):**\n- `routing.*` -- table fields, defaults, the tenant field and default org, org routing.\n The deployment contract lists `routing` in `hot_paths`, so an edit there\n reaches a running pod without rolling it\n- `timestamp_dq.*` -- timestamp validation thresholds\n- `metadata.*` -- common header injection, tags, _raw handling\n- `field_sanitization.*` -- field name sanitisation rules\n- `buffer.flush_rows` / `buffer.flush_bytes` / `buffer.flush_age_secs`\n\n**Requires pod restart (connections/state established at startup):**\n- `transport` -- transport type (kafka/grpc) bound at startup\n- `kafka.*` -- Kafka consumer created at startup\n- `grpc.*` -- gRPC server binds at startup\n- `clickhouse.*` -- HTTP client + `clickhouse::Client` created at startup\n- `payload.pipeline_mode` -- pipeline path chosen at startup\n- `metrics.*` -- HTTP metrics server binds at startup\n- `logging.*` -- tracing subscriber installed at startup\n- `scaling.*` -- scaling pressure built at startup\n- `hot_reload.*` -- watcher config set at startup\n- `schema.*` -- schema cache created at startup\n- `geoip.*` -- MMDB readers opened at startup\n- `computed_columns.*` -- computed column cache built at startup\n- `column_directives.*` -- column directive cache built at startup\n- `routing.dlq` -- DLQ backends opened at startup\n- `field_mapping.*` -- mapping rules built at startup\n- `enrichment.*` -- enrichment pipeline built at startup", + "description": "Main configuration for dfe-loader.\n\n## Hot-Reload Behaviour\n\n**Hot-reloaded (takes effect on next batch):**\n- `routing.*` -- table fields, defaults, the tenant field and default org, org routing.\n The deployment contract lists `routing` in `hot_paths`, so an edit there\n reaches a running pod without rolling it\n- `timestamp_dq.*` -- timestamp validation thresholds\n- `metadata.*` -- common header injection, tags, _raw handling\n- `field_sanitization.*` -- field name sanitisation rules\n- `buffer.flush_rows` / `buffer.flush_bytes` / `buffer.flush_age_secs`\n\n**Requires pod restart (connections/state established at startup):**\n- `transport` -- transport type (kafka/grpc) bound at startup\n- `kafka.*` -- Kafka consumer created at startup\n- `grpc.*` -- gRPC server binds at startup\n- `clickhouse.*` -- HTTP client + `clickhouse::Client` created at startup\n- `payload.pipeline_mode` -- pipeline path chosen at startup\n- `metrics.*` -- HTTP metrics server binds at startup\n- `logging.*` -- tracing subscriber installed at startup\n- `scaling.*` -- scaling pressure built at startup\n- `hot_reload.*` -- watcher config set at startup\n- `schema.*` -- schema cache created at startup\n- `geoip.*` -- MMDB readers opened at startup\n- `computed_columns.*` -- computed column cache built at startup\n- `column_directives.*` -- column directive cache built at startup\n- `routing.dlq` -- DLQ backends opened at startup\n- `dlq_load.*` -- DLQ topics subscribed at startup\n- `field_mapping.*` -- mapping rules built at startup\n- `enrichment.*` -- enrichment pipeline built at startup", "type": "object", "properties": { "transport": { @@ -181,6 +181,20 @@ ] } }, + "dlq_load": { + "description": "Kafka DLQ topics loaded into a table of their own. **Restart required.**", + "$ref": "#/$defs/DlqLoadConfig", + "default": { + "topics": [ + "dfe_receiver_dlq", + "dfe_loader_dlq", + "dfe_archiver_dlq", + "dfe_fetcher_dlq", + "dfe_transform_dlq" + ], + "table": "dlq" + } + }, "routing": { "description": "Table and tenant routing. **Hot-reloaded**, `routing.dlq` aside.", "$ref": "#/$defs/RoutingConfig", @@ -193,6 +207,7 @@ "org_id_field": "_org_id", "default_org_id": "default", "org_routes": [], + "external_topics": [], "source_to_table": {}, "topic_suffixes": [ "_land", @@ -270,49 +285,16 @@ "capture_json": true, "json_output": "_json", "capture_raw": true, - "raw_source_fields": [ - "logoriginal" - ], + "raw_source_fields": [], "raw_output": "_raw", "capture_source": true, "source_fields": [ "_source" ], "source_output": "_source", - "disable_json_tables": [], - "disable_raw_tables": [], "remove_routing_fields": true } }, - "coercion": { - "description": "Type coercion config. Refused unless every field is at its default: the\ndynamic insert applies fixed coercion rules.", - "$ref": "#/$defs/CoercionConfig", - "default": { - "type_mappings": {}, - "unknown_type_fallback": "String", - "null_handling": "default", - "null_strings": [ - "null", - "NULL", - "Null", - "None", - "nil", - "undefined", - "\\N", - "", - "NA", - "N/A", - "n/a", - "NaN" - ], - "default_timezone": "UTC", - "timezone_fields": [ - "tags_collector_timezone" - ], - "array_to_json": true, - "strict": false - } - }, "field_mapping": { "description": "Field mapping overrides. **Restart required.**", "$ref": "#/$defs/FieldMappingConfig", @@ -341,7 +323,117 @@ }, "risk_scoring": { "enabled": false, - "preset": "global" + "preset": "global", + "presets": { + "apac_enterprise": { + "high": [ + "KP", + "AF", + "MM" + ], + "medium": [ + "RU", + "IR" + ] + }, + "eu_enterprise": { + "high": [ + "RU", + "KP", + "IR", + "SY", + "BY" + ], + "medium": [ + "CN", + "UA", + "PK" + ] + }, + "global": { + "high": [ + "KP", + "IR", + "SY" + ], + "medium": [ + "RU", + "CN", + "VE", + "CU" + ] + }, + "high_security": { + "high": [ + "CN", + "RU", + "KP", + "IR", + "SY", + "CU", + "VE", + "BY", + "MM" + ], + "medium": [ + "UA", + "PK", + "NG", + "VN", + "BD", + "NP" + ] + }, + "us_enterprise": { + "high": [ + "CN", + "RU", + "KP", + "IR", + "SY", + "CU", + "VE" + ], + "medium": [ + "UA", + "BY", + "PK", + "NG", + "VN" + ] + } + }, + "weights": { + "geo": 15, + "reputation": 25, + "privacy": 25, + "threat": 35 + }, + "scores": { + "unknown_country": 35, + "high_risk_country": 90, + "medium_risk_country": 60, + "other_country": 15, + "vpn": 50, + "proxy": 60, + "tor": 80, + "datacenter": 40, + "relay": 45, + "one_anonymiser": 40, + "two_anonymisers": 70, + "three_anonymisers": 95, + "botnet": 95, + "malicious": 90, + "spam": 70, + "scanner": 60, + "high_abuse_score": 80 + }, + "levels": { + "low": 20, + "medium": 40, + "high": 60, + "critical": 80 + } } } } @@ -406,7 +498,7 @@ "default": null }, "allow_insecure_transport": { - "description": "Opt in to an unencrypted Kafka transport (`plaintext` / `sasl_plaintext`)\nin production. scalo (>=2.8) rejects unencrypted transports under a\nproduction profile at transport construction unless this is set —\ndata and SASL/PLAIN credentials would otherwise ship in the clear. Leave\n`false` (the default) and configure TLS, or set `true` only when the\nin-cluster traffic is already mesh-encrypted.", + "description": "Opt in to an unencrypted Kafka transport (`plaintext` / `sasl_plaintext`)\nin production. scalo rejects unencrypted transports under a\nproduction profile at transport construction unless this is set —\ndata and SASL/PLAIN credentials would otherwise ship in the clear. Leave\n`false` (the default) and configure TLS, or set `true` only when the\nin-cluster traffic is already mesh-encrypted.", "type": "boolean", "default": false }, @@ -1250,6 +1342,31 @@ } ] }, + "DlqLoadConfig": { + "description": "Kafka DLQ topics whose dead letters the loader loads into a table of their own.\n\nA dead letter carries the routing fields of the record it wraps, so each\ntopic here routes to `table` whatever its records name. One that fails is\ndropped and counted, never dead-lettered again. Empty `topics` loads none.", + "type": "object", + "properties": { + "topics": { + "description": "DLQ topics to consume, alongside the data topics.", + "type": "array", + "items": { + "type": "string" + }, + "default": [ + "dfe_receiver_dlq", + "dfe_loader_dlq", + "dfe_archiver_dlq", + "dfe_fetcher_dlq", + "dfe_transform_dlq" + ] + }, + "table": { + "description": "Table every record from `topics` lands in, in the database `routing`\npicks for its tenant.", + "type": "string", + "default": "dlq" + } + } + }, "RoutingConfig": { "type": "object", "properties": { @@ -1294,6 +1411,14 @@ }, "default": [] }, + "external_topics": { + "description": "Topics an external pipeline writes directly, each with the tenant policy\nthat keeps its records in their own `_org_id` and `_source`. A topic not\nlisted here is trusted as it arrives.", + "type": "array", + "items": { + "$ref": "#/$defs/ExternalTopic" + }, + "default": [] + }, "source_to_table": { "description": "Source value to table name mapping\nMaps extracted source values to destination table names", "type": "object", @@ -1349,6 +1474,45 @@ "org_id" ] }, + "ExternalTopic": { + "description": "Tenant policy for a topic an external pipeline writes straight to.\n\nNo dfe-receiver stands in front of such a topic, so the loader itself keeps\na record from claiming another tenant or table. A record's `_org_id` and\n`_source` stay only when they are the source's own value or on the matching\nallow-list, and are replaced with the source's own otherwise.", + "type": "object", + "properties": { + "topic": { + "description": "The Kafka topic the policy covers, for example `cisco-ios_land`.", + "type": "string" + }, + "org_id": { + "description": "The source's own tenant: the `_org_id` a record gets when it carries none\nor one outside the allow-list.", + "type": "string" + }, + "allowed_org_ids": { + "description": "Further tenants a record on this topic may carry. Empty allows only `org_id`.", + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "source": { + "description": "The source's own name: the `_source` a record gets when it carries none\nor one outside the allow-list.", + "type": "string" + }, + "allowed_sources": { + "description": "Further sources a record on this topic may carry. Empty allows only `source`.", + "type": "array", + "items": { + "type": "string" + }, + "default": [] + } + }, + "required": [ + "topic", + "org_id", + "source" + ] + }, "DlqConfig": { "type": "object", "properties": { @@ -1552,7 +1716,7 @@ "default": {} }, "capture_json": { - "description": "Store complete original Kafka message as JSON before transformation", + "description": "Not read: `capture_mode` decides whether `_json` is written.", "type": "boolean", "default": true }, @@ -1567,14 +1731,12 @@ "default": true }, "raw_source_fields": { - "description": "Source fields to try for rename (first match wins). Default: [\"logoriginal\"]", + "description": "Fields renamed to `_raw` when the record has no `_raw` of its own, first\nmatch wins. Default: none, because `_json` already stores the field's\ntext.", "type": "array", "items": { "type": "string" }, - "default": [ - "logoriginal" - ] + "default": [] }, "raw_output": { "description": "Output field name for raw log line", @@ -1601,22 +1763,6 @@ "type": "string", "default": "_source" }, - "disable_json_tables": { - "description": "Tables where _json capture is disabled (e.g., [\"dfe.metrics\"])", - "type": "array", - "items": { - "type": "string" - }, - "default": [] - }, - "disable_raw_tables": { - "description": "Tables where _raw capture is disabled (e.g., [\"dfe.metrics\"])", - "type": "array", - "items": { - "type": "string" - }, - "default": [] - }, "remove_routing_fields": { "description": "Remove routing fields from output after extraction", "type": "boolean", @@ -1649,96 +1795,6 @@ } ] }, - "CoercionConfig": { - "description": "Type coercion configuration. No encoder reads it, so `Config::validate`\nrefuses any field set away from its default.", - "type": "object", - "properties": { - "type_mappings": { - "description": "Map custom type names to base coercer categories\ne.g., {\"MyCustomInt\": \"Int\", \"SpecialString\": \"String\"}", - "type": "object", - "additionalProperties": { - "type": "string" - }, - "default": {} - }, - "unknown_type_fallback": { - "description": "Fallback coercer for unknown types (default: \"String\")", - "type": "string", - "default": "String" - }, - "null_handling": { - "description": "How to handle null values in non-nullable columns", - "$ref": "#/$defs/NullHandling", - "default": "default" - }, - "null_strings": { - "description": "Strings to recognise as null (case-sensitive)", - "type": "array", - "items": { - "type": "string" - }, - "default": [ - "null", - "NULL", - "Null", - "None", - "nil", - "undefined", - "\\N", - "", - "NA", - "N/A", - "n/a", - "NaN" - ] - }, - "default_timezone": { - "description": "Default timezone for naive timestamps (IANA name or +HH:MM)", - "type": "string", - "default": "UTC" - }, - "timezone_fields": { - "description": "Event fields to check for timezone info", - "type": "array", - "items": { - "type": "string" - }, - "default": [ - "tags_collector_timezone" - ] - }, - "array_to_json": { - "description": "Convert arrays to JSON strings if true", - "type": "boolean", - "default": true - }, - "strict": { - "description": "Strict mode: fail on any coercion error", - "type": "boolean", - "default": false - } - } - }, - "NullHandling": { - "description": "Null handling strategy", - "oneOf": [ - { - "description": "Substitute type-appropriate default value (recommended)", - "type": "string", - "const": "default" - }, - { - "description": "Return error for null in non-nullable column", - "type": "string", - "const": "error" - }, - { - "description": "Pass null through (may cause `ClickHouse` errors)", - "type": "string", - "const": "passthrough" - } - ] - }, "FieldMappingConfig": { "description": "Field mapping configuration for normalising source field names.\n\nSupports renaming or copying fields from source to destination names.\nRules come from two sources with clear precedence:\n1. `ClickHouse` column comments (`@renamed` directives) — highest priority\n2. External remap files (CSV/YAML/JSON) and built-in presets — lower priority\n\nCSV files are compatible with the elastic/ecs-mapper format:\n`source_field,destination_field,copy_action`", "type": "object", @@ -1820,7 +1876,117 @@ "$ref": "#/$defs/RiskScoringConfig", "default": { "enabled": false, - "preset": "global" + "preset": "global", + "presets": { + "apac_enterprise": { + "high": [ + "KP", + "AF", + "MM" + ], + "medium": [ + "RU", + "IR" + ] + }, + "eu_enterprise": { + "high": [ + "RU", + "KP", + "IR", + "SY", + "BY" + ], + "medium": [ + "CN", + "UA", + "PK" + ] + }, + "global": { + "high": [ + "KP", + "IR", + "SY" + ], + "medium": [ + "RU", + "CN", + "VE", + "CU" + ] + }, + "high_security": { + "high": [ + "CN", + "RU", + "KP", + "IR", + "SY", + "CU", + "VE", + "BY", + "MM" + ], + "medium": [ + "UA", + "PK", + "NG", + "VN", + "BD", + "NP" + ] + }, + "us_enterprise": { + "high": [ + "CN", + "RU", + "KP", + "IR", + "SY", + "CU", + "VE" + ], + "medium": [ + "UA", + "BY", + "PK", + "NG", + "VN" + ] + } + }, + "weights": { + "geo": 15, + "reputation": 25, + "privacy": 25, + "threat": 35 + }, + "scores": { + "unknown_country": 35, + "high_risk_country": 90, + "medium_risk_country": 60, + "other_country": 15, + "vpn": 50, + "proxy": 60, + "tor": 80, + "datacenter": 40, + "relay": 45, + "one_anonymiser": 40, + "two_anonymisers": 70, + "three_anonymisers": 95, + "botnet": 95, + "malicious": 90, + "spam": 70, + "scanner": 60, + "high_abuse_score": 80 + }, + "levels": { + "low": 20, + "medium": 40, + "high": 60, + "critical": 80 + } } } } @@ -1852,7 +2018,7 @@ } }, "RiskScoringConfig": { - "description": "Risk scoring configuration", + "description": "Risk scoring configuration. Every country list, weight, score and level band\nbelow is a default a deployment can override.", "type": "object", "properties": { "enabled": { @@ -1861,9 +2027,378 @@ "default": false }, "preset": { - "description": "Risk preset to use for country risk tables\n\nOptions: \"global\" (default), \"`us_enterprise`\", \"`eu_enterprise`\",\n \"`apac_enterprise`\", \"`high_security`\"", + "description": "The entry in `presets` whose country lists apply. A name `presets` does\nnot hold is refused at load.", "type": "string", "default": "global" + }, + "presets": { + "description": "Country lists by preset name. The loader ships `global`,\n`us_enterprise`, `eu_enterprise`, `apac_enterprise` and `high_security`.\nAn entry set in config replaces the shipped entry of that name, or adds\na new preset, and leaves the others as shipped.", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/RiskCountries" + }, + "default": { + "apac_enterprise": { + "high": [ + "KP", + "AF", + "MM" + ], + "medium": [ + "RU", + "IR" + ] + }, + "eu_enterprise": { + "high": [ + "RU", + "KP", + "IR", + "SY", + "BY" + ], + "medium": [ + "CN", + "UA", + "PK" + ] + }, + "global": { + "high": [ + "KP", + "IR", + "SY" + ], + "medium": [ + "RU", + "CN", + "VE", + "CU" + ] + }, + "high_security": { + "high": [ + "CN", + "RU", + "KP", + "IR", + "SY", + "CU", + "VE", + "BY", + "MM" + ], + "medium": [ + "UA", + "PK", + "NG", + "VN", + "BD", + "NP" + ] + }, + "us_enterprise": { + "high": [ + "CN", + "RU", + "KP", + "IR", + "SY", + "CU", + "VE" + ], + "medium": [ + "UA", + "BY", + "PK", + "NG", + "VN" + ] + } + } + }, + "weights": { + "description": "Weight of each component in the composite score.", + "$ref": "#/$defs/RiskWeights", + "default": { + "geo": 15, + "reputation": 25, + "privacy": 25, + "threat": 35 + } + }, + "scores": { + "description": "Score each signal gives its component, 0-100.", + "$ref": "#/$defs/RiskScores", + "default": { + "unknown_country": 35, + "high_risk_country": 90, + "medium_risk_country": 60, + "other_country": 15, + "vpn": 50, + "proxy": 60, + "tor": 80, + "datacenter": 40, + "relay": 45, + "one_anonymiser": 40, + "two_anonymisers": 70, + "three_anonymisers": 95, + "botnet": 95, + "malicious": 90, + "spam": 70, + "scanner": 60, + "high_abuse_score": 80 + } + }, + "levels": { + "description": "Lowest composite score of each level above `minimal`.", + "$ref": "#/$defs/RiskLevelBands", + "default": { + "low": 20, + "medium": 40, + "high": 60, + "critical": 80 + } + } + } + }, + "RiskCountries": { + "description": "The high- and medium-risk country codes of one preset (ISO 3166-1 alpha-2).", + "type": "object", + "properties": { + "high": { + "description": "Countries scored `scores.high_risk_country`.", + "type": "array", + "items": { + "type": "string" + }, + "default": [] + }, + "medium": { + "description": "Countries scored `scores.medium_risk_country`.", + "type": "array", + "items": { + "type": "string" + }, + "default": [] + } + } + }, + "RiskWeights": { + "description": "Weight of each component in the composite score. All zero scores 0.", + "type": "object", + "properties": { + "geo": { + "description": "Geographic component.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 15 + }, + "reputation": { + "description": "Reputation component.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 25 + }, + "privacy": { + "description": "Privacy (anonymiser) component.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 25 + }, + "threat": { + "description": "Threat component.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 35 + } + } + }, + "RiskScores": { + "description": "Score each signal gives its component. Reputation and threat signals add\nup, capped at 100.", + "type": "object", + "properties": { + "unknown_country": { + "description": "Geo: a public IP with no country.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 35 + }, + "high_risk_country": { + "description": "Geo: a country in the preset's `high` list.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 90 + }, + "medium_risk_country": { + "description": "Geo: a country in the preset's `medium` list.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 60 + }, + "other_country": { + "description": "Geo: any other country.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 15 + }, + "vpn": { + "description": "Reputation: VPN exit.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 50 + }, + "proxy": { + "description": "Reputation: open proxy.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 60 + }, + "tor": { + "description": "Reputation: Tor exit.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 80 + }, + "datacenter": { + "description": "Reputation: datacenter range.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 40 + }, + "relay": { + "description": "Reputation: private relay.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 45 + }, + "one_anonymiser": { + "description": "Privacy: one anonymiser among VPN, proxy, Tor and relay.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 40 + }, + "two_anonymisers": { + "description": "Privacy: two anonymisers.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 70 + }, + "three_anonymisers": { + "description": "Privacy: three or more anonymisers.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 95 + }, + "botnet": { + "description": "Threat: botnet member.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 95 + }, + "malicious": { + "description": "Threat: known malicious.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 90 + }, + "spam": { + "description": "Threat: spam source.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 70 + }, + "scanner": { + "description": "Threat: scanner.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 60 + }, + "high_abuse_score": { + "description": "Threat: the external abuse score at or above which the row carries the\n`high_abuse_score` factor. The abuse score itself is added as given.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 80 + } + } + }, + "RiskLevelBands": { + "description": "Lowest composite score of each risk level. Below `low` is `minimal`.", + "type": "object", + "properties": { + "low": { + "description": "Lowest `low` score.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 20 + }, + "medium": { + "description": "Lowest `medium` score.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 40 + }, + "high": { + "description": "Lowest `high` score.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 60 + }, + "critical": { + "description": "Lowest `critical` score.", + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "default": 80 } } } diff --git a/tests/unit/test_appmgmt/test_contract.py b/tests/unit/test_appmgmt/test_contract.py index 4e30b66c7..2fd48ac9d 100644 --- a/tests/unit/test_appmgmt/test_contract.py +++ b/tests/unit/test_appmgmt/test_contract.py @@ -34,7 +34,7 @@ def _env(service: str, name: str) -> str: # Schemas from the builds that hold each source acknowledgement until delivery, # copied byte for byte from each app's docs/config-schema.json: dfe-receiver -# f61c903, dfe-loader edef368, dfe-transform-vrl da42bd2, dfe-archiver 29f201a, +# f61c903, dfe-loader 36a2afe, dfe-transform-vrl da42bd2, dfe-archiver 29f201a, # dfe-transform-vector 36dbfa6, dfe-fetcher 0ab0317. HELD = Path(__file__).parents[2] / "fixtures" / "contract-acknowledgements" From 6c55b058ea47a7fac91a695625bf17778a56da47 Mon Sep 17 00:00:00 2001 From: Derek Date: Sun, 15 Nov 2026 08:02:48 +1100 Subject: [PATCH 2/2] fix(capture): name the merged loader commit for its fixture The loader acknowledgement fixture comment named a branch commit. It now names 412f65d82d393b0fde0e0c92deca90d7ce3ff818, the merge of hyperi-io/dfe-loader#347, whose docs/config-schema.json matches the fixture byte for byte. --- tests/unit/test_appmgmt/test_contract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_appmgmt/test_contract.py b/tests/unit/test_appmgmt/test_contract.py index 2fd48ac9d..a1cd5b94c 100644 --- a/tests/unit/test_appmgmt/test_contract.py +++ b/tests/unit/test_appmgmt/test_contract.py @@ -34,7 +34,7 @@ def _env(service: str, name: str) -> str: # Schemas from the builds that hold each source acknowledgement until delivery, # copied byte for byte from each app's docs/config-schema.json: dfe-receiver -# f61c903, dfe-loader 36a2afe, dfe-transform-vrl da42bd2, dfe-archiver 29f201a, +# f61c903, dfe-loader 412f65d82d393b0fde0e0c92deca90d7ce3ff818, dfe-transform-vrl da42bd2, dfe-archiver 29f201a, # dfe-transform-vector 36dbfa6, dfe-fetcher 0ab0317. HELD = Path(__file__).parents[2] / "fixtures" / "contract-acknowledgements"