Repository navigation
Expand file tree
/
Copy pathlogging.py
More file actions
268 lines (220 loc) · 10.5 KB
/
Copy pathlogging.py
File metadata and controls
268 lines (220 loc) · 10.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
"""Structured JSON logging with secrets redaction.
Logs are written as newline-delimited JSON objects to a rotating file.
A logging filter redacts known credential patterns before anything is written.
"""
from __future__ import annotations
import json
import logging
import logging.handlers
import os
import re
import sys
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
# Rotating daemon log. ``--data-dir`` relocates this with the rest of that
# tree (TD-4842); the default data dir keeps today's platform path.
LOG_FILE_NAME = "tstd.log"
#: What each secret-shaped match becomes in every output path.
REDACTED = "[REDACTED]"
# A run after "sk-" that reads as words, not a key (TD-5160): segments
# split by "-" or "_", each a short lowercase or Title-case word with up
# to four digits after it ("v2", "api03"), a short acronym ("API") or a
# short number ("2026"), up to the end of the run. A key's random part has
# a long segment or mixed case inside one, so it never fits.
_SK_WORD = r"(?:[A-Z]?[a-z]{1,24}[0-9]{0,4}|[A-Z]{1,4}[0-9]{0,4}|[0-9]{1,8})"
_SK_WORDS = rf"{_SK_WORD}(?:[-_]+{_SK_WORD})*[-_]*(?![a-zA-Z0-9_-])"
# Patterns that must be redacted in log output (from tst-cua policy.py).
SECRET_PATTERNS: list[re.Pattern[str]] = [
# OpenAI / OpenRouter / Anthropic key. Dashed forms (sk-or-v1-…,
# sk-proj-…, sk-ant-…) are the shapes the shipped presets actually
# hold. Straight after a lowercase letter (TD-5160) a match is let go
# only when the run after "sk-" reads as words (_SK_WORDS): that is
# "task-tool-availability" or "tst-desk-gap-audit-2026-10-14". A key
# glued to a letter is still caught, as after an ANSI colour code
# ("\x1b[31msk-…"), a JSON escape ("\u003esk-…") or "\nsk-…", because a
# key's random part never reads as words. The UI belt in
# ui/src/lib/redact.ts uses the same sk- rule (TD-5161): it also cleans
# OS notification banners, which a person reads.
re.compile(rf"((?:(?<![a-z])|(?!sk-{_SK_WORDS}))sk-[a-zA-Z0-9][a-zA-Z0-9_-]{{15,}})"),
re.compile(r"(github_pat_[a-zA-Z0-9_]{36,})"), # GitHub fine-grained PAT
re.compile(r"(ghp_[a-zA-Z0-9]{36,})"), # GitHub classic PAT
re.compile(r"(ghu_[A-Za-z0-9_]{20,})"), # GitHub user-to-server token
re.compile(r"(ghr_[A-Za-z0-9_]{20,})"), # GitHub refresh token
re.compile(r"(gho_[A-Za-z0-9_]{20,})"), # GitHub OAuth token
re.compile(r"(ghs_[A-Za-z0-9_]{20,})"), # GitHub server token
re.compile(r"(AKIA[0-9A-Z]{16})"), # AWS access key
re.compile(r"(-----BEGIN\s+(RSA |EC |DSA |OPENSSH )?PRIVATE KEY)"), # Private key
]
# XDG leaf. The Tauri identifier is ``com.thatsimpletech.tstdesk``; a
# host that joined that name onto ``dirs::data_dir()`` split Linux GUI
# state from the CLI. Canonical name matches ``docs/configuration.md``.
LINUX_DATA_DIR_NAME = "tst-desk"
LINUX_LEGACY_DATA_DIR_NAME = "com.thatsimpletech.tstdesk"
def linux_user_data_dir(base: Path) -> Path:
"""Resolve the Linux / BSD data dir under *base*.
If only the reverse-DNS leftover exists, rename it once. When both
exist, the documented ``tst-desk`` tree wins and the leftover is left
alone — do not merge two live stores.
"""
canonical = base / LINUX_DATA_DIR_NAME
legacy = base / LINUX_LEGACY_DATA_DIR_NAME
if canonical.exists() or not legacy.exists():
return canonical
try:
legacy.rename(canonical)
except OSError:
return legacy
return canonical
def user_data_dir() -> Path:
"""Return the platform-appropriate user data directory for TST Desk."""
system = sys.platform
if system == "darwin":
return Path.home() / "Library" / "Application Support" / "com.thatsimpletech.tstdesk"
if system == "win32":
appdata = os.environ.get("APPDATA")
if appdata:
return Path(appdata) / "com.thatsimpletech.tstdesk"
return Path.home() / "AppData" / "Roaming" / "com.thatsimpletech.tstdesk"
# Linux / BSD
xdg = os.environ.get("XDG_DATA_HOME")
base = Path(xdg) if xdg else Path.home() / ".local" / "share"
return linux_user_data_dir(base)
def log_directory(data_dir: Path) -> Path:
"""Directory that holds ``tstd.log`` for a daemon data directory."""
return data_dir / "logs"
#: Exact secret texts this process has read or stored (TD-5110), longest
#: first. A token in an MCP server's env or header can have any shape, so
#: the patterns above may miss it; its exact text never will. Rebound as a
#: whole tuple, so a reader on another thread sees the old or the new one.
_KNOWN_SECRETS: tuple[str, ...] = ()
#: Shorter texts would also match ordinary words.
MIN_KNOWN_SECRET_CHARS = 8
def register_secret_value(value: str) -> None:
"""Scrub *value* from every output path for the life of the process.
A header such as ``Bearer <token>`` also registers each long
whitespace-separated part, so the bare token is caught too. Nothing is
ever unregistered: a rotated token is still worth hiding.
"""
global _KNOWN_SECRETS
parts = {value.strip(), *value.split()}
fresh = {p for p in parts if len(p) >= MIN_KNOWN_SECRET_CHARS} - set(_KNOWN_SECRETS)
if fresh:
_KNOWN_SECRETS = tuple(sorted({*_KNOWN_SECRETS, *fresh}, key=len, reverse=True))
def redact_secrets(text: str) -> str:
"""Replace known credential patterns in *text* with ``[REDACTED]``.
This is the single redaction implementation shared by every output
path — logs (SecretsRedactionFilter), the audit store (TD-901), and
the event pipeline (TD-1405) — so a pattern added here protects all
of them at once. Secrets registered with :func:`register_secret_value`
are replaced by their exact text first.
"""
for known in _KNOWN_SECRETS:
if known in text:
text = text.replace(known, REDACTED)
for pattern in SECRET_PATTERNS:
text = pattern.sub(REDACTED, text)
return text
def redact_structure(value: Any) -> Any:
"""Recursively redact secrets in every string of a JSON-able structure.
Keys are scrubbed as well as values — a credential used as a mapping
key is unusual but must not reach disk either. Shared by the audit
store and the event-log chokepoint (TD-1405).
"""
if isinstance(value, str):
return redact_secrets(value)
if isinstance(value, Mapping):
return {redact_structure(k): redact_structure(v) for k, v in value.items()}
if isinstance(value, Sequence):
return [redact_structure(item) for item in value]
return value
class SecretsRedactionFilter(logging.Filter):
"""Redact known credential patterns from log records.
Operates on the formatted message text so it catches secrets in any
log field regardless of how the record was constructed.
"""
def filter(self, record: logging.LogRecord) -> bool:
if record.msg:
record.msg = redact_secrets(record.msg)
# Also redact args that are strings
if record.args:
args = list(record.args)
for i, arg in enumerate(args):
if isinstance(arg, str):
args[i] = redact_secrets(arg)
record.args = tuple(args)
return True
class JSONFormatter(logging.Formatter):
"""Format log records as JSON lines."""
def format(self, record: logging.LogRecord) -> str:
obj: dict[str, Any] = {
"ts": f"{record.created:.3f}",
"level": record.levelname,
"logger": record.name,
# The filter scrubs msg and string args; an arg whose str() holds
# a secret only shows up once the message is built (TD-5110).
"message": redact_secrets(record.getMessage()),
}
if record.exc_info and record.exc_info[0]:
# Tracebacks can carry secrets in exception messages and local
# variable reprs; the filter runs on msg/args before formatting,
# so the last mile scrubs here (TD-4802).
obj["exception"] = redact_secrets(self.formatException(record.exc_info))
# Merge extra fields passed via extra={} — scrubbed for the same
# reason: the filter never sees these values.
for key, value in getattr(record, "extra_fields", {}).items():
obj[key] = redact_structure(value)
return json.dumps(obj, default=str, ensure_ascii=False)
def setup_logging(
*,
level: str | int = "INFO",
log_dir: Path | None = None,
log_to_stdout: bool = True,
max_bytes: int = 10 * 1024 * 1024, # 10 MB
backup_count: int = 5,
) -> logging.Logger:
"""Configure the root logger with JSON output and secrets redaction.
Args:
level: Log level (name or int).
log_dir: Directory for rotating log files. Defaults to
``<user data dir>/logs``. The daemon passes ``<data-dir>/logs``
so ``--data-dir`` does not leave lines in the real data dir.
log_to_stdout: Also emit JSON lines to stdout.
max_bytes: Max size per log file before rotation.
backup_count: Number of rotated log files to keep.
Returns:
The root logger, already configured.
"""
if isinstance(level, str):
level = getattr(logging, level.upper(), logging.INFO)
# TD-1102: websockets logs raw frame contents at DEBUG, and frames can
# carry credentials (set_api_key payloads) and chat content. Redaction
# is pattern-based, so an unusually-shaped key would survive it — cap
# the frame logger at INFO no matter what --log-level was requested.
logging.getLogger("websockets").setLevel(max(level, logging.INFO))
log_dir = log_dir or log_directory(user_data_dir())
log_dir.mkdir(parents=True, exist_ok=True)
log_file = log_dir / LOG_FILE_NAME
root = logging.getLogger()
root.setLevel(level)
# Remove any pre-existing handlers (idempotent)
root.handlers.clear()
# Rotating file handler
file_handler = logging.handlers.RotatingFileHandler(
log_file,
maxBytes=max_bytes,
backupCount=backup_count,
)
file_handler.setFormatter(JSONFormatter())
file_handler.addFilter(SecretsRedactionFilter())
root.addHandler(file_handler)
# Optional stdout handler
if log_to_stdout:
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setFormatter(JSONFormatter())
stdout_handler.addFilter(SecretsRedactionFilter())
root.addHandler(stdout_handler)
return root
def get_logger(name: str) -> logging.Logger:
"""Get a logger with the given name, child of the root."""
return logging.getLogger(name)