From f0ba52615b2329229e88581ad4461d2e08dcc6fc Mon Sep 17 00:00:00 2001 From: Brigs Date: Sun, 16 Aug 2026 20:43:27 -0400 Subject: [PATCH] Order conversation artifact columns by their declared roles Puts the fields an examiner reads first at the front of every conversation artifact table: the event timestamp, any other date or time columns, then direction, sender, conversation label, message text and media. Remaining columns keep their existing relative order. The order is derived from the roles each artifact already declares in its data_views conversation block, so the table matches what LAVA renders. admin/scripts/check_conversation_column_order.py enforces it in the lint job and admin/docs/conversation_column_order.md records the convention. Row counts and values are unchanged; only column order moves. Co-Authored-By: Claude Opus 5 --- .github/workflows/python_lint.yml | 7 + admin/docs/conversation_column_order.md | 65 +++++ .../check_conversation_column_order.py | 234 ++++++++++++++++++ scripts/artifacts/discordMessages.py | 26 +- scripts/artifacts/signalMessages.py | 29 ++- scripts/artifacts/whatsappMessages.py | 23 +- scripts/artifacts/wireIndexedDb.py | 12 +- 7 files changed, 370 insertions(+), 26 deletions(-) create mode 100644 admin/docs/conversation_column_order.md create mode 100644 admin/scripts/check_conversation_column_order.py diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index 88b99d8..955f62a 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -42,6 +42,13 @@ jobs: if: steps.changed-files-py.outputs.any_changed == 'true' run: python admin/scripts/check_claim_language.py + # A conversation artifact's columns are ordered from the roles it declares in + # data_views: timestamp, other dates, direction, sender, conversation label, + # message text, media, then the rest. See admin/docs/conversation_column_order.md. + - name: Check conversation artifact column order + if: steps.changed-files-py.outputs.any_changed == 'true' + run: python admin/scripts/check_conversation_column_order.py + # html_columns cells are written to the report without html.escape, so evidence # placed there can inject markup, and any remote href/src makes opening a report # beacon to a third party. Pre-existing findings are carried in the script's diff --git a/admin/docs/conversation_column_order.md b/admin/docs/conversation_column_order.md new file mode 100644 index 0000000..e780462 --- /dev/null +++ b/admin/docs/conversation_column_order.md @@ -0,0 +1,65 @@ +# Column order for conversation artifacts + +An artifact that reports messages, chats, conversations, comments or posts puts the fields an +examiner reads first at the front of the table. The order is not a per-artifact choice. It is +derived from the roles the artifact already declares in its `data_views.conversation` block. + +## The order + +1. The declared `timeColumn`. +2. Every other column typed `datetime` or `date`, keeping their existing relative order. +3. `directionColumn` +4. `senderColumn` +5. `conversationLabelColumn` +6. `textColumn` +7. `mediaColumn` +8. Everything else, in whatever order the artifact already used. + +A role that the artifact does not declare is simply skipped, and the columns after it move up. + +`admin/scripts/check_conversation_column_order.py` enforces this and runs in CI. + +## Why these fields, in this order + +The timestamp leads because the report is read chronologically, and because an artifact that +goes to the timeline needs a `datetime` or `date` first column anyway. + +Direction, sender, conversation and message text are what identify a message: when, which way, +who, in what thread, saying what. Media sits next to the message text because a media column +renders a thumbnail and is physically wide, so keeping it beside the text keeps the row that +carries the content readable. + +Everything else is identifiers, state flags and raw values. Those matter, they are just not +what the examiner reads first. + +## Reorder the row tuple in the same edit + +`data_headers` and the row appended to `data_list` are co-indexed. Move one without the other +and every value lands under the wrong header. Nothing raises, and each value still looks +plausible, so this is not caught by reading the report. + +Where the row comes straight from a query (`data_list.append(tuple(row))`), the `SELECT` list +is the row order. Reorder the `SELECT`, not the Python. + +## What to do when a role is not declared + +If a conversation artifact has a direction column that `data_views` does not name as +`directionColumn`, the mechanical order leaves it where it is, because the check only moves +what is declared. + +Adding the declaration is worth doing, but it is a behaviour change, not a formatting one: +`directionColumn` and `directionSentValue` decide which side of LAVA's conversation view a +message renders on. Establish what the stored value means from a source before declaring it. +Do not guess a `directionSentValue` to make the column move. + +## There is no external standard for this + +CASE/UCO standardises the vocabulary for exchanging message data, not the layout of a report. +Its `observable:MessageFacet` defines `application`, `from`, `to`, `sentTime`, `messageID`, +`messageText`, `messageType` and `sessionID`, and nothing about column order. It has no +direction property at all, and no attachment property on the message facet, so the Direction +and Media columns here are LEAPP concepts with no outside standard to defer to. + +Reference: Unified Cyber Ontology, `ontology/uco/observable/observable.ttl`, commit +`7ebb3957e9e9a2e1bb9c66cd1ede8c912a726344`, +https://github.com/ucoProject/UCO/blob/7ebb3957e9e9a2e1bb9c66cd1ede8c912a726344/ontology/uco/observable/observable.ttl diff --git a/admin/scripts/check_conversation_column_order.py b/admin/scripts/check_conversation_column_order.py new file mode 100644 index 0000000..a88da54 --- /dev/null +++ b/admin/scripts/check_conversation_column_order.py @@ -0,0 +1,234 @@ +"""Guard the column order of conversation artifacts. + +An artifact that reports messages declares, in its `__artifacts_v2__` entry, a +`data_views.conversation` block naming which of its columns carry the time, the +direction, the sender, the conversation label, the message text and the media. +Those declarations already exist for LAVA's conversation view, so the report +table can be ordered from them rather than from per-artifact taste. + +The order this check enforces: + + 1. the declared timeColumn + 2. every other column typed 'datetime' or 'date' + 3. directionColumn, senderColumn, conversationLabelColumn, textColumn, + mediaColumn, in that relative order + 4. everything else, in whatever order the artifact already used + +The point is the examiner's first read. Before this order was applied, the +declared direction column sat at a median of column 9 and as far right as column +23, so the single field that says whether a message was sent or received was off +the visible width of a wide table. Two artifacts did not even have their declared +time column first. + +The check parses each module with `ast`, resolves `data_headers` (a literal in +the artifact function, a module-level constant, or a concatenation of the two), +and reports: + + * a declared timeColumn that is not the first column + * declared role columns that appear out of the order above + * a declared role naming a column the artifact does not emit, which would also + leave LAVA's conversation view without that field + +Headers built at run time cannot be resolved statically. Those modules are listed +as unchecked rather than silently passed, so the gap stays visible. +""" +import argparse +import ast +import os +import sys + +ROLE_SEQUENCE = ['directionColumn', 'senderColumn', 'conversationLabelColumn', + 'textColumn', 'mediaColumn'] +# Older modules spell two of the keys differently; lavafuncs remaps them on write. +CONVERT = {'threadDiscriminatorColumn': 'conversationDiscriminatorColumn', + 'threadLabelColumn': 'conversationLabelColumn'} +DATE_TYPES = ('datetime', 'date') + +STANDARD_NOTE = ( + 'Order a conversation artifact\'s columns as: the declared timeColumn, then any other\n' + 'datetime/date columns, then direction, sender, conversation label, message text and\n' + 'media, then everything else unchanged. Reorder the row tuple in the same edit, or\n' + 'every value lands under the wrong header.') + + +def artifacts_block(tree): + for node in tree.body: + if isinstance(node, ast.Assign): + for t in node.targets: + if isinstance(t, ast.Name) and t.id == '__artifacts_v2__': + try: + return ast.literal_eval(node.value) + except (ValueError, TypeError, SyntaxError): + return None + return None + + +def module_constants(tree): + """Module-level names bound to a tuple/list literal, for header constants.""" + out = {} + for node in tree.body: + if isinstance(node, ast.Assign) and isinstance(node.value, (ast.Tuple, ast.List)): + for t in node.targets: + if isinstance(t, ast.Name): + try: + out[t.id] = list(ast.literal_eval(node.value)) + except (ValueError, TypeError, SyntaxError): + pass + return out + + +def resolve(expr, consts): + """Evaluate a header expression: literal, module constant, or their concatenation.""" + if isinstance(expr, (ast.Tuple, ast.List)): + try: + return list(ast.literal_eval(expr)) + except (ValueError, TypeError, SyntaxError): + return None + if isinstance(expr, ast.Name): + return list(consts[expr.id]) if expr.id in consts else None + if isinstance(expr, ast.BinOp) and isinstance(expr.op, ast.Add): + left, right = resolve(expr.left, consts), resolve(expr.right, consts) + return None if left is None or right is None else left + right + return None + + +def headers_for(tree, func_name, consts): + """Headers an artifact function returns, or None when built at run time.""" + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef) or node.name != func_name: + continue + for n in ast.walk(node): + if isinstance(n, ast.Assign): + for t in n.targets: + if isinstance(t, ast.Name) and t.id == 'data_headers': + got = resolve(n.value, consts) + if got is not None: + return got + # No local assignment: a return of a constant, or a delegation to another artifact. + for n in ast.walk(node): + if isinstance(n, ast.Return) and isinstance(n.value, ast.Tuple) and n.value.elts: + got = resolve(n.value.elts[0], consts) + if got is not None: + return got + if (isinstance(n, ast.Return) and isinstance(n.value, ast.Call) + and isinstance(n.value.func, ast.Attribute) + and n.value.func.attr == '__wrapped__' + and isinstance(n.value.func.value, ast.Name)): + return headers_for(tree, n.value.func.value.id, consts) + return None + + +def expected(names, types, view): + order, used = [], set() + tc = view.get('timeColumn') + if tc in names: + i = names.index(tc) + order.append(i) + used.add(i) + for i, t in enumerate(types): + if i not in used and t in DATE_TYPES: + order.append(i) + used.add(i) + for role in ROLE_SEQUENCE: + c = view.get(role) + if c in names and names.index(c) not in used: + i = names.index(c) + order.append(i) + used.add(i) + order += [i for i in range(len(names)) if i not in used] + return order + + +def check_module(path): + problems, unchecked, checked = [], [], 0 + src = open(path, encoding='utf-8', errors='replace').read() + if 'data_views' not in src: + return problems, unchecked, checked + try: + tree = ast.parse(src) + except SyntaxError as ex: + return [f'{os.path.basename(path)}: could not parse ({ex})'], unchecked, checked + blk = artifacts_block(tree) + if not isinstance(blk, dict): + return problems, unchecked, checked + consts = module_constants(tree) + rel = os.path.basename(path) + + for art, meta in blk.items(): + dv = (meta or {}).get('data_views') or {} + if not isinstance(dv, dict): + continue + raw = dv.get('conversation') or dv.get('chat') + if not raw: + continue + view = {CONVERT.get(k, k): v for k, v in raw.items()} + headers = headers_for(tree, art, consts) + if headers is None: + unchecked.append(f'{rel}::{art} (headers built at run time)') + continue + checked += 1 + names = [h[0] if isinstance(h, (tuple, list)) else h for h in headers] + types = [(h[1] if isinstance(h, (tuple, list)) and len(h) > 1 else None) for h in headers] + + for role in ['timeColumn'] + ROLE_SEQUENCE: + col = view.get(role) + if col and col not in names: + problems.append(f'{rel}::{art}: {role} names {col!r}, which is not a column ' + f'this artifact emits') + + tc = view.get('timeColumn') + if tc in names and names[0] != tc: + problems.append(f'{rel}::{art}: timeColumn {tc!r} is column ' + f'{names.index(tc) + 1}, not column 1') + + want = expected(names, types, view) + if want != list(range(len(names))): + shown = [names[i] for i in want] + problems.append(f'{rel}::{art}: columns are not in declared-role order\n' + f' is: {names[:7]}\n' + f' expect: {shown[:7]}') + return problems, unchecked, checked + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('paths', nargs='*', help='artifact modules to check (default: all)') + args = ap.parse_args() + + paths = args.paths + if not paths: + root = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__)))), 'scripts', 'artifacts') + paths = [os.path.join(root, f) for f in sorted(os.listdir(root)) if f.endswith('.py')] + + problems, unchecked, checked = [], [], 0 + for p in paths: + if not p.endswith('.py') or not os.path.exists(p): + continue + pr, un, ck = check_module(p) + problems += pr + unchecked += un + checked += ck + + if unchecked: + print(f'{len(unchecked)} conversation artifact(s) NOT checked, headers are ' + f'not statically resolvable:') + for u in sorted(unchecked): + print(f' {u}') + print() + + if problems: + print(f'{len(problems)} conversation artifact column-order problem(s):') + for p in sorted(problems): + print(f' {p}') + print() + print(STANDARD_NOTE) + return 1 + + print(f'Checked {checked} conversation artifact(s): columns follow the declared-role order.') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/artifacts/discordMessages.py b/scripts/artifacts/discordMessages.py index 09da1e0..8d9d66e 100644 --- a/scripts/artifacts/discordMessages.py +++ b/scripts/artifacts/discordMessages.py @@ -189,10 +189,24 @@ def _reply_reference(message): @artifact_processor def discordMessages(context): data_headers = ( - ("Timestamp", "datetime"), "Direction", "Sender", "Channel", "Message", - ("Attachments", "media"), "Attachment Names", ("Edited", "datetime"), - "Reply To", "Mentions", "Embeds", "Stickers", "Message Type", "Pinned", - "Sender ID", "Channel ID", "Message ID", ("Cached", "datetime"), + ("Timestamp", "datetime"), + ("Edited", "datetime"), + ("Cached", "datetime"), + "Direction", + "Sender", + "Channel", + "Message", + ("Attachments", "media"), + "Attachment Names", + "Reply To", + "Mentions", + "Embeds", + "Stickers", + "Message Type", + "Pinned", + "Sender ID", + "Channel ID", + "Message ID", "Source Cache File", ) @@ -239,13 +253,14 @@ def discordMessages(context): data_list.append(( sent, + discord_api.iso_to_datetime(message.get("edited_timestamp")), + wrapper["cached"], direction, discord_api.user_display(author), _channel_label(scan, str(message.get("channel_id") or "")), message.get("content") or "", media_refs, ", ".join(n for n in names if n), - discord_api.iso_to_datetime(message.get("edited_timestamp")), _reply_reference(message), ", ".join(discord_api.user_display(m) for m in (message.get("mentions") or []) if isinstance(m, dict)), @@ -256,7 +271,6 @@ def discordMessages(context): author_id, str(message.get("channel_id") or ""), message_id, - wrapper["cached"], context.get_relative_path(wrapper["source"]), )) diff --git a/scripts/artifacts/signalMessages.py b/scripts/artifacts/signalMessages.py index ed99e84..53acf97 100644 --- a/scripts/artifacts/signalMessages.py +++ b/scripts/artifacts/signalMessages.py @@ -167,11 +167,24 @@ def _recovery_failure(root, relative_path, local_key): @artifact_processor def signalMessages(context): data_headers = ( - ("Sent", "datetime"), "Direction", "Sender", "Conversation", "Message", - ("Attachments", "media"), "Attachment Names", "Message Type", - ("Received", "datetime"), ("Server Timestamp", "datetime"), - "Read Status", "View Once", "Erased", "Expires In (s)", - ("Expires At", "datetime"), "Conversation ID", "Message ID", "Source File", + ("Sent", "datetime"), + ("Received", "datetime"), + ("Server Timestamp", "datetime"), + ("Expires At", "datetime"), + "Direction", + "Sender", + "Conversation", + "Message", + ("Attachments", "media"), + "Attachment Names", + "Message Type", + "Read Status", + "View Once", + "Erased", + "Expires In (s)", + "Conversation ID", + "Message ID", + "Source File", ) files_found = [str(f) for f in context.get_files_found()] @@ -240,6 +253,9 @@ def signalMessages(context): data_list.append(( signal_desktop.js_ms_to_datetime(sent_at), + signal_desktop.js_ms_to_datetime(received_at), + signal_desktop.js_ms_to_datetime(server_ts), + signal_desktop.js_ms_to_datetime(expires_at), direction, sender, labels.get(conversation_id, conversation_id or ""), @@ -247,13 +263,10 @@ def signalMessages(context): media_refs, ", ".join(names), message_type or "", - signal_desktop.js_ms_to_datetime(received_at), - signal_desktop.js_ms_to_datetime(server_ts), read_status if read_status is not None else "", "Yes" if view_once else "", "Yes" if erased else "", expire_timer if expire_timer else "", - signal_desktop.js_ms_to_datetime(expires_at), conversation_id or "", message_id or "", context.get_relative_path(next(iter(signal_desktop.database_files(files_found)), "")), diff --git a/scripts/artifacts/whatsappMessages.py b/scripts/artifacts/whatsappMessages.py index ba8ce85..91862ef 100644 --- a/scripts/artifacts/whatsappMessages.py +++ b/scripts/artifacts/whatsappMessages.py @@ -103,10 +103,21 @@ def _sender(is_from_me, chat_jid, partner_name, contact_jid, from_jid, @artifact_processor def whatsappMessages(context): data_headers = ( - ("Message Date", "datetime"), "Direction", "Sender", "Chat", - ("Media", "media"), "Message", "Media Filename", "Type Code", "Starred", - ("Sent Date", "datetime"), "Chat Type", "Sender JID", "Chat JID", - "Message ID", "Source File", + ("Message Date", "datetime"), + ("Sent Date", "datetime"), + "Direction", + "Sender", + "Chat", + "Message", + ("Media", "media"), + "Media Filename", + "Type Code", + "Starred", + "Chat Type", + "Sender JID", + "Chat JID", + "Message ID", + "Source File", ) files_found = [str(f) for f in context.get_files_found()] @@ -146,15 +157,15 @@ def whatsappMessages(context): data_list.append(( whatsapp.cocoa_to_datetime(message_date), + whatsapp.cocoa_to_datetime(sent_date), direction, sender, partner_name or contact_jid or "", - media_ref, text or "", + media_ref, media_name, message_type if message_type is not None else "", "Yes" if starred else "", - whatsapp.cocoa_to_datetime(sent_date), whatsapp.jid_kind(contact_jid), sender_jid, contact_jid or "", diff --git a/scripts/artifacts/wireIndexedDb.py b/scripts/artifacts/wireIndexedDb.py index cf7bcb4..61c1fa1 100644 --- a/scripts/artifacts/wireIndexedDb.py +++ b/scripts/artifacts/wireIndexedDb.py @@ -810,13 +810,13 @@ def wireMessages(context): outgoing = 1 if sender_id in self_ids else 0 rows.append(( _iso_to_dt(v.get("time")), - _account_label(users, self_ids, rec.get("db_name")), - conv_names.get(cid, cid), - _display_name(users, sender_id, self_ids), outgoing, + _display_name(users, sender_id, self_ids), + conv_names.get(cid, cid), + text or "", media_for(d) if etype == "conversation.asset-add" else "", + _account_label(users, self_ids, rec.get("db_name")), kind, - text or "", attachment, v.get("id", ""), sender_id, @@ -828,8 +828,8 @@ def wireMessages(context): else datetime.min.replace(tzinfo=timezone.utc))) data_headers = ( - ("Timestamp", "datetime"), "Account", "Conversation", "Sender", - "Outgoing", ("Media", "media"), "Message Type", "Message", "Attachment", + ("Timestamp", "datetime"), "Outgoing", "Sender", "Conversation", + "Message", ("Media", "media"), "Account", "Message Type", "Attachment", "Message ID", "Sender ID", "Conversation ID", "Status", ) return data_headers, rows, _source(context, dirs)