From 69c8571d988a6f3b2578256253ad413c80b94cd8 Mon Sep 17 00:00:00 2001 From: abruption Date: Sun, 1 Nov 2026 08:37:53 +0900 Subject: [PATCH] ops: prepare explicit 24-claim recovery operating budget --- docs/experimental-recovery.ko.md | 2 +- docs/experimental-recovery.md | 2 +- .../new_incident_policy_deployment.py | 27 +++- research/mcp-events/recovery-budget.ko.md | 49 ++++++ research/mcp-events/recovery-budget.md | 49 ++++++ research/mcp-events/recovery-cooldown.ko.md | 2 + research/mcp-events/recovery-cooldown.md | 2 + .../test_new_incident_policy_deployment.py | 152 ++++++++++++++++-- .../test_operating_recovery_budget.py | 104 ++++++++++++ 9 files changed, 369 insertions(+), 20 deletions(-) create mode 100644 research/mcp-events/recovery-budget.ko.md create mode 100644 research/mcp-events/recovery-budget.md create mode 100644 research/mcp-events/test_operating_recovery_budget.py diff --git a/docs/experimental-recovery.ko.md b/docs/experimental-recovery.ko.md index 6d36c3b..36ad041 100644 --- a/docs/experimental-recovery.ko.md +++ b/docs/experimental-recovery.ko.md @@ -43,7 +43,7 @@ MCP 서버가 원격 셸을 실행하지는 않습니다. claim 만료 시각은 ## 정책과 운영 상태 확인 -기본값은 **최근 24시간 최대 3회, 실행 허가 사이 최소 3,600초**입니다. 관리자 시험은 간격을 유지하면서 최대 12회로 명시적으로 설정했습니다. 성공한 시도도 횟수에 포함됩니다. 설정 키는 `recovery_max_claims_per_24h`, `recovery_cooldown_seconds`이며, 정책 변경은 서버 설정 검토를 거쳐야 합니다. Dots에게 제한을 무시하라고 요청하는 방식으로 바꾸지 마세요. +기본값은 **최근 24시간 최대 3회, 실행 허가 사이 최소 3,600초**입니다. 관리자 시험은 간격을 유지하면서 최대 12회로 명시적으로 설정했습니다. 성공한 시도도 횟수에 포함됩니다. 별도로 검토할 [운영 한도 24회 변경안](../research/mcp-events/recovery-budget.ko.md)이 있으며, 패키지 기본값 변경이나 운영 반영 완료를 뜻하지는 않습니다. 설정 키는 `recovery_max_claims_per_24h`, `recovery_cooldown_seconds`이며, 정책 변경은 서버 설정 검토를 거쳐야 합니다. Dots에게 제한을 무시하라고 요청하는 방식으로 바꾸지 마세요. MCP의 `get_current_recovery`로 `observation_fresh`, `claimable`, `eligibility_reason`, `retry_at`, `policy`를 확인합니다. `retry_at`은 참고 시각입니다. 그 시간이 지나도 최신 상태를 다시 조회하고 새 claim을 받아야 실행할 수 있습니다. 같은 incident ID로 알림이 다시 올 수 있지만, 새 실행 권한을 뜻하지 않습니다. 원래 장애·claim·전송 이력은 보존됩니다. 이미 claim이 있는 장애는 결과가 불명확해도 다시 claim하거나 실행하지 않습니다. diff --git a/docs/experimental-recovery.md b/docs/experimental-recovery.md index 46646fc..215419f 100644 --- a/docs/experimental-recovery.md +++ b/docs/experimental-recovery.md @@ -55,7 +55,7 @@ The broker does not execute a remote shell. The claim's expiry must be respected ## Policy and operating status -The default is **three claims per rolling 24 hours, at least 3,600 seconds apart**. The maintainer's trial uses an explicit 12-claim limit with the same interval. Successful attempts also count. Operator configuration keys are `recovery_max_claims_per_24h` and `recovery_cooldown_seconds`; changes require reviewed server configuration, not a prompt asking Dots to bypass policy. +The default is **three claims per rolling 24 hours, at least 3,600 seconds apart**. The maintainer's trial uses an explicit 12-claim limit with the same interval. Successful attempts also count. A separate [24-claim operating change](../research/mcp-events/recovery-budget.md) is now proposed; it is not a changed package default or evidence of deployment. Operator configuration keys are `recovery_max_claims_per_24h` and `recovery_cooldown_seconds`; changes require reviewed server configuration, not a prompt asking Dots to bypass policy. Use the MCP `get_current_recovery` tool to inspect `observation_fresh`, `claimable`, `eligibility_reason`, `retry_at`, and `policy`. A future `retry_at` is advisory. Even after that time, Dots must reread fresh state and obtain an accepted claim. A reminder can carry the same incident ID; it does not create new execution authority. The original incident, claims and delivery history remain intact. An already claimed incident must not be claimed or executed again, including when its result is unknown. diff --git a/research/mcp-events/new_incident_policy_deployment.py b/research/mcp-events/new_incident_policy_deployment.py index 6902fa5..8d5a1b8 100644 --- a/research/mcp-events/new_incident_policy_deployment.py +++ b/research/mcp-events/new_incident_policy_deployment.py @@ -141,8 +141,15 @@ def patch_server(body): 'max_claims_per_24h': "cfg.get('recovery_max_claims_per_24h', 3)", 'cooldown_seconds': "cfg.get('recovery_cooldown_seconds', 3600)", 'new_incident_cooldown_seconds': "cfg.get('recovery_new_incident_cooldown_seconds')", + 'local_coordination': "cfg.get('recovery_local_coordination', False)", } - if set(names) not in (set(expected) - {'new_incident_cooldown_seconds'}, set(expected)): + required = {'max_claims_per_24h', 'cooldown_seconds'} + supported_shapes = { + frozenset(required | optional) + for optional in (set(), {'new_incident_cooldown_seconds'}, {'local_coordination'}, + {'new_incident_cooldown_seconds', 'local_coordination'}) + } + if frozenset(names) not in supported_shapes: raise ValueError('unexpected recovery policy call shape') for keyword in call.keywords: if _expression(keyword.value) != _expression(ast.parse(expected[keyword.arg], mode='eval').body): @@ -168,7 +175,7 @@ def offset(line, column): return result -def config_candidate(body, seconds): +def config_candidate(body, seconds, max_claims_per_24h=None): def unique(pairs): result = {} for key, value in pairs: @@ -187,10 +194,16 @@ def unique(pairs): cooldown = value.get('recovery_cooldown_seconds', 3600) if type(budget) is not int or not 1 <= budget <= 24: raise ValueError('invalid existing claim budget') + if max_claims_per_24h is not None and ( + type(max_claims_per_24h) is not int or not 1 <= max_claims_per_24h <= 24): + raise ValueError('new claim budget must be an integer from 1 to 24') if type(cooldown) is not int or not 300 <= cooldown <= 86400: raise ValueError('invalid existing global cooldown') if type(seconds) is not int or not 300 <= seconds <= cooldown: raise ValueError('new incident interval must be from 300 to the existing cooldown') + if max_claims_per_24h is not None: + value['recovery_max_claims_per_24h'] = max_claims_per_24h + budget = max_claims_per_24h value['recovery_new_incident_cooldown_seconds'] = seconds return value, {'max_claims_per_24h': budget, 'cooldown_seconds': cooldown, 'new_incident_cooldown_seconds': seconds} @@ -223,6 +236,8 @@ def main(): parser.add_argument('--expected-server-sha256', required=True) parser.add_argument('--candidate-core-sha256', required=True) parser.add_argument('--new-incident-cooldown-seconds', type=int, required=True) + parser.add_argument('--max-claims-per-24h', type=int, + help='explicitly set the rolling claim cap (integer 1..24); omission preserves the current value') args = parser.parse_args() if os.geteuid() != 0: parser.error('root required') @@ -238,7 +253,8 @@ def main(): initial_config, initial_meta = read_regular(CONFIG, account.pw_uid) if stat.S_IMODE(initial_meta.st_mode) != 0o600: raise ValueError('private configuration permissions required') - config_candidate(initial_config, args.new_incident_cooldown_seconds) + config_candidate(initial_config, args.new_incident_cooldown_seconds, + args.max_claims_per_24h) locks = [] try: # The release updater owns the first lock; earlier policy helpers own the @@ -250,7 +266,8 @@ def main(): original, config_meta = read_regular(CONFIG, account.pw_uid) if stat.S_IMODE(config_meta.st_mode) != 0o600: raise ValueError('private configuration permissions required') - cfg, policy = config_candidate(original, args.new_incident_cooldown_seconds) + cfg, policy = config_candidate(original, args.new_incident_cooldown_seconds, + args.max_claims_per_24h) patched = patch_server(server) parsed_origin = urlsplit(cfg['origin']) if (parsed_origin.scheme != 'https' or not parsed_origin.hostname @@ -333,7 +350,7 @@ def health(): raise ValueError('new policy not exposed by installed server') for key in ('max_claims_per_24h', 'cooldown_seconds'): if observed['policy'].get(key) != policy[key]: - raise ValueError('existing policy changed') + raise ValueError('requested policy not exposed by installed server') phase = 'start_broker' run(['systemctl', 'start', *UNITS]) health() diff --git a/research/mcp-events/recovery-budget.ko.md b/research/mcp-events/recovery-budget.ko.md new file mode 100644 index 0000000..dc6dac8 --- /dev/null +++ b/research/mcp-events/recovery-budget.ko.md @@ -0,0 +1,49 @@ +# 운영 복구 한도 24회 변경안 + +[English](recovery-budget.md) + +[#80](https://github.com/abruption/dots-tailscale/issues/80)의 운영 정책 변경안입니다. 명시적으로 설정한 **12회** 한도를 **최근 86,400초 동안 claim 최대 24회**로 높입니다. 이 문서가 병합되거나 dots-tailscale을 설치하는 것만으로 운영 서버의 설정이 바뀌지는 않습니다. + +패키지 기본값은 계속 **최근 24시간 최대 3회**입니다. 이번 운영 변경안은 전역 실행 간격 **3,600초**, 복구가 확인된 뒤 새 장애에 적용하는 간격 **600초**를 유지합니다. 성공·실패·결과 불명·미보고 여부와 관계없이 발급된 claim은 계속 집계합니다. 집계 기간을 초기화하거나 횟수를 돌려주지 않으며, 로컬 복구 조정 기능이나 watchdog의 별도 한도도 바꾸지 않습니다. + +## 변경 준비 + +기존 비공개 서버 설정에서 다음 키 하나를 바꾸는 안을 검토합니다. + +```json +{"recovery_max_claims_per_24h": 24} +``` + +`recovery_cooldown_seconds=3600`, `recovery_new_incident_cooldown_seconds=600`, 인증·메일 연동·구독 설정 등 다른 값은 유지합니다. 예제 파일로 비공개 설정 전체를 덮어쓰지 마세요. 일반 `config.example.json`은 복구가 꺼진 상태를 유지하며 이번 운영 한도를 기본값으로 넣지 않습니다. + +검토된 `new_incident_policy_deployment.py`에 선택적인 `--max-claims-per-24h` 옵션을 지원합니다. 생략하면 기존 한도나 암묵적 기본값을 보존하고, 지정할 때는 1~24의 정수만 받습니다. 기존 설정이 잘못됐다면 새 값으로 조용히 고치는 대신 거부합니다. + +도구의 보호된 준비 경로, 소스 해시 고정, 배포 잠금, 최신 정상 관측과 진행 중인 작업이 없어야 한다는 조건은 유지합니다. **열린 장애나 전송 중인 이벤트가 있으면 적용을 거부합니다.** 독립 복구 터미널을 준비하고 이 조건을 충족하는 점검 시점을 확보하세요. 진행 중인 장애의 실행 제한을 우회하는 도구가 아닙니다. + +이번 한도 변경에서는 이미 설치된 코어를 독립적으로 검증해 그대로 준비하고, 두 코어 인자에 같은 해시를 사용합니다. 기존 통합 서버도 확인해 해시를 고정하세요. 연동 코드를 바꿀 필요가 없도록 이미 지원되는 정책 인자를 전달하는 서버여야 합니다. 지원되는 `local_coordination` 인자가 있으면 그대로 보존합니다. 통합 서버를 공개 연구용 서버로 교체하지 마세요. + +키 하나만 바꾸는 이 절차를 쓰기 전에 최신 MCP `policy`와 비공개 설정을 읽어 기존 정책이 정확히 12/3600/600이고, 검토한 서버 패치 결과가 원본과 바이트 단위로 같은지 확인하세요. 다르면 중단합니다. 범용 도구는 명시적으로 요청한 다른 변경도 지원하므로, 아래 옵션만으로 이번 절차의 전제까지 강제되지는 않습니다. + +정확한 해시를 확인한 뒤 root 인증으로 실행하는 명령 형태는 다음과 같습니다. 대문자는 실제 해시로 바꿔야 하는 자리표시자입니다. + +```sh +python3 /root/reviewed/new_incident_policy_deployment.py \ + --source-dir /root/reviewed/source \ + --expected-core-sha256 INSTALLED_CORE_SHA256 \ + --expected-server-sha256 INSTALLED_INTEGRATED_SERVER_SHA256 \ + --candidate-core-sha256 INSTALLED_CORE_SHA256 \ + --new-incident-cooldown-seconds 600 \ + --max-claims-per-24h 24 +``` + +이 명령은 별도로 승인된 운영 배포에서만 실행합니다. root 인증이 필요하고 broker 소켓과 서비스를 잠시 중지했다가 시작하며, 코드·설정의 비공개 백업을 남깁니다. 도구가 claim을 만들거나 콜백을 재전송하거나 시작 명령을 실행하지는 않습니다. 재시작 뒤 정상 감시 경로가 현재 조건에 따라 정책 제한 해제 이벤트를 전달할 수는 있지만, 그 이벤트도 실행하려면 새 claim이 필요합니다. + +## 확인과 원복 + +도구의 반환 정책과 최신 읽기 전용 MCP 상태에서 `max_claims_per_24h=24`, `cooldown_seconds=3600`, `new_incident_cooldown_seconds=600`을 확인합니다. 짧은 간격을 적용할 새 장애가 없으면 실제 적용 간격이 3,600초로 나오는 것도 정상입니다. 기존 claim 집계는 유지합니다. 같은 집계 기간의 25번째 claim 거부는 격리 시험으로 확인하며, 운영 서버에 시험 claim을 만들지는 마세요. `retry_at`은 참고 시각이며 시간이 지났다는 이유만으로 실행할 수는 없습니다. + +설정은 JSON으로 읽은 값끼리 비교하세요. 도구가 키 순서와 서식을 다시 작성합니다. 통합 코어·서버의 내용, 한도 키 이외의 비공개 설정, 구독과 실행 승인 이력이 보존됐는지 확인합니다. 로컬 복구 정책도 유지하세요. 이후 실제 복구를 확정하려면 외부의 새 SSH·ping 관측이 필요합니다. + +12회로 되돌릴 때는 시작 정책이 24/3600/600인지 확인하고, 같은 소스 보존 절차에서 `--max-claims-per-24h 12`를 사용합니다. 원복에도 최신 정상 관측과 진행 중인 작업이 없어야 한다는 조건이 적용됩니다. 장애 중 긴급 중지가 필요하면 이 조건을 우회하지 말고 기존 [중지와 원복 절차](../../docs/experimental-recovery.ko.md#중지와-원복)를 따르세요. 이미 12회를 넘게 사용했다면 이전 claim이 충분히 집계 대상에서 빠질 때까지 새 실행이 제한될 수 있습니다. 낮아진 한도에 맞추려고 과거 DB를 복원하거나 기록을 지우지 마세요. 배포 중 새 claim이 생기면 운영자 확인을 요구하는 기존의 보수적인 원복 절차를 유지하며, 원장은 복원하지 않습니다. + +이번 증설로 반복 단절의 원인, #79의 이벤트 수신 후 미실행 문제, #80의 나머지 진단·사전 경고·임시 추가 예산 과제가 해결되는 것은 아닙니다. 이 작업의 격리 시험이 운영 반영이나 무인 복구 성공을 입증하지는 않습니다. diff --git a/research/mcp-events/recovery-budget.md b/research/mcp-events/recovery-budget.md new file mode 100644 index 0000000..bf4d557 --- /dev/null +++ b/research/mcp-events/recovery-budget.md @@ -0,0 +1,49 @@ +# Proposed operating recovery budget: 24 claims per day + +[한국어](recovery-budget.ko.md) + +This is the proposed operating-policy change for [#80](https://github.com/abruption/dots-tailscale/issues/80). It raises an explicitly configured **12-claim** allowance to **24 claims per rolling 86,400 seconds**. Merging this document or installing dots-tailscale does not change a running broker. + +The package default remains **3 claims per rolling 24 hours**. The operating proposal keeps the global interval at **3,600 seconds** and the confirmed-new-incident interval at **600 seconds**. An existing claim still counts whether its result is success, failure, unknown or unreported. The change does not reset the rolling window, refund claims, change local coordination or change the local watchdog budget. + +## Prepare the change + +Review a one-key change in the existing private broker configuration: + +```json +{"recovery_max_claims_per_24h": 24} +``` + +Keep all other values, including `recovery_cooldown_seconds=3600`, `recovery_new_incident_cooldown_seconds=600`, authentication, mail integration and subscription settings. Do not replace the full private configuration with an example file. The general `config.example.json` remains disabled and does not adopt this operating allowance. + +The reviewed `new_incident_policy_deployment.py` helper accepts an optional `--max-claims-per-24h`. Omitting it preserves the current budget or implicit default; an explicit value must be an integer from 1 to 24. Invalid existing values are rejected rather than repaired implicitly. + +The helper retains its protected staging, source pins, exclusive deployment locks and fresh/healthy/quiescent checks. **It refuses to apply during an open outage or in-flight delivery.** Prepare an independent recovery terminal and a maintenance point meeting those checks; the helper cannot be used to bypass an active incident's refusal. + +For this budget-only operating change, stage the already installed, independently verified core unchanged and use its same digest for both core arguments. Inspect and pin the existing integrated server; it must already pass the supported policy arguments so no connector code needs editing. The helper preserves the supported `local_coordination` argument if present. Never replace the integrated server with the public research server. + +Before using this one-key recipe, read the current MCP `policy` and inspect the private configuration to confirm the existing policy is exactly 12/3600/600 and that the reviewed server patch would leave its bytes unchanged; stop if either precondition differs. The general helper also supports other explicitly requested changes, so those flags alone do not enforce this recipe. + +After inspecting the exact hashes, the root-authenticated invocation has this shape. Uppercase values are placeholders, not usable hashes: + +```sh +python3 /root/reviewed/new_incident_policy_deployment.py \ + --source-dir /root/reviewed/source \ + --expected-core-sha256 INSTALLED_CORE_SHA256 \ + --expected-server-sha256 INSTALLED_INTEGRATED_SERVER_SHA256 \ + --candidate-core-sha256 INSTALLED_CORE_SHA256 \ + --new-incident-cooldown-seconds 600 \ + --max-claims-per-24h 24 +``` + +Only a separately approved operating deployment runs that command. It needs root authentication, briefly stops/restarts the broker socket and service, and retains private code/configuration backups. No claim, callback replay or startup command is issued by the helper. Normal monitoring may deliver a policy-unblocked event after restart if current conditions warrant it; that event still requires a fresh claim before execution. + +## Verify and roll back + +Check the returned policy and a fresh read-only MCP status: `max_claims_per_24h=24`, `cooldown_seconds=3600`, `new_incident_cooldown_seconds=600`. An effective interval of 3,600 seconds remains valid when there is no eligible confirmed-new-incident transition. Existing claim counts must be retained. The isolated tests check that the 25th claim within the same rolling window is blocked; do not create test claims on the operating broker. `retry_at` is advisory; time passing alone does not authorize execution. + +Compare configuration values as parsed JSON; the helper rewrites key ordering and formatting. Verify that the integrated core/server content, private configuration outside the budget key, subscriptions and authorization history are preserved. Keep local recovery policy unchanged. Independent external SSH and ping remain necessary to confirm any later recovery. + +To return to 12, confirm the starting policy is 24/3600/600 and use the same source-preserving process with `--max-claims-per-24h 12`. Rollback also requires fresh healthy observations and no open work. For an emergency stop during an outage, follow the existing [stop and rollback procedure](../../docs/experimental-recovery.md#stop-and-rollback) instead of bypassing those guards. If usage is already above 12, that policy may block new claims until enough old entries expire. Never restore an old DB or delete entries to make the lower limit fit. The helper keeps its existing conservative rollback behavior, including operator attention if a new claim appears during deployment; it never restores a ledger. + +This finite increase does not resolve repeated disconnects, the received-but-unclaimed reminder gap in #79, or the remaining diagnostics/early-warning/temporary-escalation work in #80. Those items remain separate. No operating rollout or unattended recovery success is established by the isolated tests for this change. diff --git a/research/mcp-events/recovery-cooldown.ko.md b/research/mcp-events/recovery-cooldown.ko.md index dd91502..5449d45 100644 --- a/research/mcp-events/recovery-cooldown.ko.md +++ b/research/mcp-events/recovery-cooldown.ko.md @@ -134,3 +134,5 @@ dots-tailscale 일반 설치는 이 연구 서버를 배포하지 않습니다. claim만 있다면 증거가 소급 생성되지 않는 것이 정상입니다. 이후 실제 장애 복구는 별도의 실측입니다. 짧은 간격을 끄려면 통제된 설정 변경으로 새 항목을 삭제하거나 `null`로 돌리고, 기존 예산과 이력은 유지하세요. + +이후 제안된 운영 한도 12회→24회 변경은 [복구 예산 안내](recovery-budget.ko.md)를 참고하세요. 두 실행 간격과 기존 claim 이력을 유지하는 별도 변경안이며, 이 문서의 기본값을 바꾸지는 않습니다. diff --git a/research/mcp-events/recovery-cooldown.md b/research/mcp-events/recovery-cooldown.md index 67d80bb..f42980a 100644 --- a/research/mcp-events/recovery-cooldown.md +++ b/research/mcp-events/recovery-cooldown.md @@ -148,3 +148,5 @@ only legacy claims, confirmation evidence is deliberately not backfilled. Actual a separate end-to-end observation. To disable the short path, remove/set the new configuration key to `null` through a controlled operator change; leave all budgets and histories intact. + +For the later proposed operating increase from 12 to 24 claims, see the [recovery budget guide](recovery-budget.md). That proposal preserves both intervals and existing claim history; it does not change the defaults described here. diff --git a/research/mcp-events/test_new_incident_policy_deployment.py b/research/mcp-events/test_new_incident_policy_deployment.py index 3f6c53a..f5ba4bf 100644 --- a/research/mcp-events/test_new_incident_policy_deployment.py +++ b/research/mcp-events/test_new_incident_policy_deployment.py @@ -1,5 +1,6 @@ """Isolated deployment fixtures; never access production files or services.""" import contextlib +import ast import hashlib import io import json @@ -70,6 +71,40 @@ def test_accepts_existing_trailing_comma_without_duplicate_comma(self): compile(result, '', 'exec') self.assertNotIn(b',,', result) + def test_adds_cooldown_without_changing_local_coordination_setting(self): + body = SERVER.replace( + b"cooldown_seconds=cfg.get('recovery_cooldown_seconds', 3600))", + b"cooldown_seconds=cfg.get('recovery_cooldown_seconds', 3600),\n" + b" local_coordination=cfg.get('recovery_local_coordination', False))") + result = deploy.patch_server(body) + insertion = (b",\n new_incident_cooldown_seconds=" + b"cfg.get('recovery_new_incident_cooldown_seconds')") + self.assertEqual(result.replace(insertion, b'', 1), body) + self.assertIn(b"local_coordination=cfg.get('recovery_local_coordination', False)", result) + compile(result, '', 'exec') + + def test_current_patched_server_with_local_coordination_is_byte_preserved(self): + for local_coordination in (True, False): + with self.subTest(local_coordination=local_coordination): + body = deploy.patch_server(SERVER).replace( + b"new_incident_cooldown_seconds=cfg.get('recovery_new_incident_cooldown_seconds'))", + b"new_incident_cooldown_seconds=cfg.get('recovery_new_incident_cooldown_seconds'),\n" + b" local_coordination=cfg.get('recovery_local_coordination', False))") + self.assertEqual(deploy.patch_server(body), body) + tree = ast.parse(body.decode('utf-8')) + function = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name == 'load_recovery') + call = next(node for node in ast.walk(function) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == 'RecoveryCore') + expression = next(keyword.value for keyword in call.keywords + if keyword.arg == 'local_coordination') + value = eval(compile(ast.Expression(expression), '', 'eval'), + {}, {'cfg': {'recovery_local_coordination': local_coordination}}) + self.assertIs(value, local_coordination) + class ConfigTests(unittest.TestCase): def config(self): @@ -92,6 +127,26 @@ def test_defaults_are_not_written_as_unrequested_changes(self): self.assertEqual(policy['max_claims_per_24h'], 3) self.assertEqual(policy['cooldown_seconds'], 3600) + def test_explicit_budget_override_changes_only_budget_and_interval(self): + before = self.config() + before['recovery_local_coordination'] = True + before['unrelated_extension'] = {'preserve': 'yes'} + candidate, policy = deploy.config_candidate(json.dumps(before).encode(), 600, 24) + self.assertEqual(candidate.pop('recovery_new_incident_cooldown_seconds'), 600) + self.assertEqual(candidate['recovery_max_claims_per_24h'], 24) + candidate['recovery_max_claims_per_24h'] = before['recovery_max_claims_per_24h'] + self.assertEqual(candidate, before) + self.assertEqual(policy, {'max_claims_per_24h': 24, 'cooldown_seconds': 3600, + 'new_incident_cooldown_seconds': 600}) + self.assertNotIn('DUMMY_TEST_ONLY', json.dumps(policy)) + + def test_omitting_budget_override_preserves_existing_value(self): + before = self.config() + before['recovery_max_claims_per_24h'] = 12 + candidate, policy = deploy.config_candidate(json.dumps(before).encode(), 600) + self.assertEqual(candidate['recovery_max_claims_per_24h'], 12) + self.assertEqual(policy['max_claims_per_24h'], 12) + def test_rejects_invalid_new_or_existing_policy_values(self): for seconds in (True, '600', None, 299, 3601, -1): with self.subTest(seconds=seconds), self.assertRaises(ValueError): @@ -106,6 +161,16 @@ def test_rejects_invalid_new_or_existing_policy_values(self): with self.subTest(key=key, value=value), self.assertRaises(ValueError): deploy.config_candidate(json.dumps(cfg).encode(), 600) + def test_explicit_budget_override_is_a_strict_integer_from_one_to_twenty_four(self): + for value in (True, False, '24', 24.0, 0, 25, -1): + with self.subTest(value=value), self.assertRaises(ValueError): + deploy.config_candidate(json.dumps(self.config()).encode(), 600, value) + + def test_explicit_override_does_not_heal_an_invalid_existing_budget(self): + cfg = self.config(); cfg['recovery_max_claims_per_24h'] = 25 + with self.assertRaisesRegex(ValueError, 'existing claim budget'): + deploy.config_candidate(json.dumps(cfg).encode(), 600, 24) + def test_runtime_dir_must_match_exact_canonical_data_directory(self): for value in (None, False, '/var/lib/other', str(deploy.DATA) + '/', '/var/lib/../lib/dots-events', '/tmp/dots-data-alias'): @@ -170,19 +235,31 @@ def test_quiescent_preflight_refuses_each_unsafe_condition(self): class OrchestrationTests(unittest.TestCase): def invoke(self, *, schema_failure=False, inactive_unit=None, wrong_runtime=False, - post_start_failure=False, new_claims_after_start=False): + post_start_failure=False, new_claims_after_start=False, + max_claims_per_24h=None, budget_only=False, initial_budget=12): with tempfile.TemporaryDirectory() as folder: root = Path(folder); app = root / 'app'; app.mkdir() data = root / 'data'; data.mkdir(); records = root / 'records'; records.mkdir() stage = root / 'stage'; stage.mkdir() - core = b'OLD_CORE = True\n'; candidate = b'NEW_CORE = True\n' + core = b'OLD_CORE = True\n'; candidate = core if budget_only else b'NEW_CORE = True\n' + server = SERVER + if budget_only: + server = deploy.patch_server(SERVER).replace( + b"new_incident_cooldown_seconds=cfg.get('recovery_new_incident_cooldown_seconds'))", + b"new_incident_cooldown_seconds=cfg.get('recovery_new_incident_cooldown_seconds'),\n" + b" local_coordination=cfg.get('recovery_local_coordination', False))") cfg = json.dumps({'enable_recovery': True, 'origin': 'https://recovery.example.com', 'runtime_dir': '/var/lib/other' if wrong_runtime else str(data), - 'recovery_max_claims_per_24h': 12, + 'recovery_max_claims_per_24h': initial_budget, 'recovery_cooldown_seconds': 3600, 'mail': {'settings': 'DUMMY_KEEP_ME'}}).encode() + if budget_only: + existing = json.loads(cfg) + existing['recovery_new_incident_cooldown_seconds'] = 600 + existing['recovery_local_coordination'] = False + cfg = json.dumps(existing).encode() config = data / 'config.json' - for path, body in [(app / 'recovery_core.py', core), (app / 'server.py', SERVER), + for path, body in [(app / 'recovery_core.py', core), (app / 'server.py', server), (stage / 'recovery_core.py', candidate), (config, cfg)]: path.write_bytes(body); path.chmod(0o600) ledger = data / 'recovery.sqlite' @@ -216,14 +293,18 @@ def run(argv, **kwargs): if script == deploy.ADOPT: self.assertEqual((app / 'recovery_core.py').read_bytes(), candidate) self.assertTrue(any(cmd[:2] == ['systemctl', 'stop'] for cmd in commands)) - if schema_failure: - raise RuntimeError('simulated schema adoption failure') written = json.loads(config.read_bytes()) self.assertEqual(written['runtime_dir'], str(data)) self.assertEqual(written['recovery_new_incident_cooldown_seconds'], 600) + expected_budget = (initial_budget if max_claims_per_24h is None + else max_claims_per_24h) + self.assertEqual(written['recovery_max_claims_per_24h'], expected_budget) + if schema_failure: + raise RuntimeError('simulated schema adoption failure') return subprocess.CompletedProcess(argv, 0, json.dumps({'policy': { 'new_incident_cooldown_seconds': 600, - 'max_claims_per_24h': 12, 'cooldown_seconds': 3600}}), '') + 'max_claims_per_24h': expected_budget, + 'cooldown_seconds': 3600}}), '') return subprocess.CompletedProcess(argv, 0, '', '') def lock(path): @@ -233,9 +314,11 @@ def lock(path): actual_read = deploy.read_regular actual_pin = deploy.pinned_body argv = ['helper', '--source-dir', str(stage), '--expected-core-sha256', deploy.sha256(core), - '--expected-server-sha256', deploy.sha256(SERVER), + '--expected-server-sha256', deploy.sha256(server), '--candidate-core-sha256', deploy.sha256(candidate), '--new-incident-cooldown-seconds', '600'] + if max_claims_per_24h is not None: + argv.extend(['--max-claims-per-24h', str(max_claims_per_24h)]) output = io.StringIO(); previous_umask = os.umask(0o077) error = None try: @@ -261,16 +344,56 @@ def lock(path): 'result': json.loads(output.getvalue()) if output.getvalue() else None, 'core': (app / 'recovery_core.py').read_bytes(), 'original_core': core, 'candidate': candidate, 'server': (app / 'server.py').read_bytes(), + 'original_server': server, 'config': config.read_bytes(), 'original_config': cfg, 'ledger': ledger.read_bytes()} + def test_budget_only_operating_recipe_changes_one_config_key(self): + state = self.invoke(max_claims_per_24h=24, budget_only=True) + self.assertIsNone(state['error']) + self.assertTrue(state['result']['applied']) + self.assertEqual(state['core'], state['original_core']) + self.assertEqual(state['server'], state['original_server']) + expected = {**json.loads(state['original_config']), 'recovery_max_claims_per_24h': 24} + self.assertEqual(json.loads(state['config']), expected) + self.assertEqual(state['ledger'], b'Never rewind authorization ledger') + self.assertEqual(state['result']['policy'], { + 'max_claims_per_24h': 24, 'cooldown_seconds': 3600, + 'new_incident_cooldown_seconds': 600}) + + def test_omitted_cap_success_retains_twelve(self): + state = self.invoke() + self.assertIsNone(state['error']) + self.assertEqual(state['result']['policy']['max_claims_per_24h'], 12) + self.assertEqual(json.loads(state['config'])['recovery_max_claims_per_24h'], 12) + + def test_already_twenty_four_operating_recipe_never_stops_or_rewrites(self): + state = self.invoke(max_claims_per_24h=24, budget_only=True, initial_budget=24) + self.assertIsNone(state['error']) + self.assertTrue(state['result']['already_current']) + self.assertFalse(state['result']['applied']) + self.assertEqual(state['config'], state['original_config']) + self.assertEqual(state['server'], state['original_server']) + self.assertEqual(state['core'], state['original_core']) + self.assertFalse(any(cmd[:2] == ['systemctl', 'stop'] for cmd in state['commands'])) + + def test_invalid_explicit_cap_stops_before_locks_or_service_calls(self): + for maximum in (0, 25): + with self.subTest(maximum=maximum): + state = self.invoke(max_claims_per_24h=maximum) + self.assertIsInstance(state['error'], ValueError) + self.assertEqual(state['commands'], []) + self.assertEqual(state['locks'], []) + self.assertEqual(state['config'], state['original_config']) + def test_schema_failure_restores_only_code_and_config_not_ledger(self): - state = self.invoke(schema_failure=True) + state = self.invoke(schema_failure=True, max_claims_per_24h=24) self.assertIsInstance(state['error'], RuntimeError) self.assertIn('root-private record', str(state['error'])) self.assertEqual(state['core'], state['original_core']) self.assertEqual(state['server'], SERVER) self.assertEqual(state['config'], state['original_config']) + self.assertEqual(json.loads(state['config'])['recovery_max_claims_per_24h'], 12) self.assertEqual(state['ledger'], b'Never rewind authorization ledger') self.assertEqual(state['result']['rollback'], 'code_and_config_restored') self.assertFalse(state['result']['ledger_restored']) @@ -278,14 +401,16 @@ def test_schema_failure_restores_only_code_and_config_not_ledger(self): self.assertEqual(sum(cmd[:2] == ['systemctl', 'start'] for cmd in state['commands']), 1) def test_success_preserves_mail_configuration_and_checks_every_unit(self): - state = self.invoke() + state = self.invoke(max_claims_per_24h=24) self.assertIsNone(state['error']) self.assertTrue(state['result']['applied']) self.assertEqual(state['core'], state['candidate']) self.assertEqual(state['server'], deploy.patch_server(SERVER)) after = json.loads(state['config']); before = json.loads(state['original_config']) self.assertEqual(after.pop('recovery_new_incident_cooldown_seconds'), 600) - self.assertEqual(after, before) + self.assertEqual(after['recovery_max_claims_per_24h'], 24) + expected = {**before, 'recovery_max_claims_per_24h': 24} + self.assertEqual(after, expected) self.assertEqual(state['ledger'], b'Never rewind authorization ledger') for unit in (*deploy.UNITS, 'caddy.service'): self.assertEqual(sum(cmd == ['systemctl', 'is-active', '--quiet', unit] @@ -321,7 +446,7 @@ def test_wrong_runtime_directory_blocks_before_locks_services_or_mutations(self) self.assertEqual(state['ledger'], b'Never rewind authorization ledger') def test_new_claim_after_start_blocks_older_code_rollback_and_leaves_broker_stopped(self): - state = self.invoke(post_start_failure=True, new_claims_after_start=True) + state = self.invoke(post_start_failure=True, new_claims_after_start=True, max_claims_per_24h=24) self.assertIsInstance(state['error'], RuntimeError) self.assertEqual(state['result']['failed_phase'], 'verify') self.assertEqual(state['result']['rollback'], 'operator_attention') @@ -331,6 +456,7 @@ def test_new_claim_after_start_blocks_older_code_rollback_and_leaves_broker_stop self.assertEqual(state['server'], deploy.patch_server(SERVER)) cfg = json.loads(state['config']) self.assertEqual(cfg['recovery_new_incident_cooldown_seconds'], 600) + self.assertEqual(cfg['recovery_max_claims_per_24h'], 24) self.assertEqual(cfg['mail'], json.loads(state['original_config'])['mail']) self.assertEqual(state['ledger'], b'Never rewind authorization ledger') changes = [cmd[1] for cmd in state['commands'] if cmd[:1] == ['systemctl'] @@ -338,7 +464,7 @@ def test_new_claim_after_start_blocks_older_code_rollback_and_leaves_broker_stop self.assertEqual(changes, ['stop', 'start', 'stop']) def test_post_start_failure_without_new_claim_can_restore_previous_code(self): - state = self.invoke(post_start_failure=True) + state = self.invoke(post_start_failure=True, max_claims_per_24h=24) self.assertIsInstance(state['error'], RuntimeError) self.assertEqual(state['result']['failed_phase'], 'verify') self.assertEqual(state['result']['rollback'], 'code_and_config_restored') diff --git a/research/mcp-events/test_operating_recovery_budget.py b/research/mcp-events/test_operating_recovery_budget.py new file mode 100644 index 0000000..4c6bab0 --- /dev/null +++ b/research/mcp-events/test_operating_recovery_budget.py @@ -0,0 +1,104 @@ +"""Operating cap changes preserve grants and retain a finite rolling limit.""" +import unittest +from collections import Counter + +from recovery_core import RecoveryIneligible +from server import load_recovery +import test_recovery_core as fixtures + + +class OperatingRecoveryBudgetTests(unittest.TestCase): + def setUp(self): + self.fixture = fixtures.RecoveryTests(methodName='test_three_distinct_samples_and_duplicate') + self.fixture.setUp() + self.addCleanup(self.fixture.tearDown) + self.f = self.fixture + + def configured(self, maximum): + return self.f.configured_core(max_claims_per_24h=maximum, cooldown_seconds=3600, + new_incident_cooldown_seconds=600) + + def rows(self): + with self.f.core._db() as db: + return [tuple(row) for row in db.execute( + 'SELECT * FROM recovery_incidents ORDER BY id')] + + def test_server_loader_exposes_twenty_four_and_preserves_execution_mode(self): + config = {'enable_recovery': True, 'runtime_dir': self.f.tmp.name, + 'allowed_callback_origins': [], 'allowed_user': 'operator', + 'recovery_max_claims_per_24h': 24, + 'recovery_cooldown_seconds': 3600, + 'recovery_new_incident_cooldown_seconds': 600} + for enabled in (False, True): + state = load_recovery({**config, 'recovery_local_coordination': enabled}).get_current_recovery('operator') + self.assertEqual(state['policy']['max_claims_per_24h'], 24) + self.assertEqual(state['policy']['cooldown_seconds'], 3600) + self.assertEqual(state['policy']['new_incident_cooldown_seconds'], 600) + self.assertEqual(state['execution_mode'], 'coordinated' if enabled else 'legacy_start') + self.assertFalse(state['claimable']) + with self.assertRaises(ValueError): + load_recovery({**config, 'recovery_max_claims_per_24h': 25}) + + def test_raise_from_twelve_preserves_all_outcomes_and_requires_new_claim(self): + iid = self.f.incident() + self.f.seed_claims([self.f.now - 12000 + n * 60 for n in range(12)]) + with self.f.core._db() as db: + for n in range(12): + # A grant without a report still consumes budget. + changed = db.execute('UPDATE recovery_incidents SET outcome=? WHERE id=?', + (('success', 'failed', 'unknown', None)[n % 4], 'historic_' + str(n))) + self.assertEqual(changed.rowcount, 1) + distribution = Counter(row[0] for row in db.execute( + 'SELECT outcome FROM recovery_incidents WHERE claim_id IS NOT NULL')) + self.assertEqual(distribution, {'success': 3, 'failed': 3, 'unknown': 3, None: 3}) + self.f.core = self.configured(12) + denied = self.f.fresh_state() + self.assertEqual(denied['eligibility_reason'], 'daily_limit') + with self.assertRaises(RecoveryIneligible): + self.f.core.claim_recovery('operator', iid) + before = self.rows() + self.f.core = self.configured(24) # Same ledger, as after a reviewed restart. + state = self.f.fresh_state() + self.assertEqual(self.rows(), before) + self.assertEqual(state['policy']['claims_in_window'], 12) + self.assertEqual(state['policy']['max_claims_per_24h'], 24) + self.assertEqual(state['policy']['cooldown_seconds'], 3600) + self.assertEqual(state['policy']['new_incident_cooldown_seconds'], 600) + self.assertTrue(state['claimable']) + self.assertIsNone(state['incident']['outcome']) + grant = self.f.core.claim_recovery('operator', iid) + self.assertEqual(grant['execution_mode'], 'legacy_start') + self.assertEqual(self.f.core.get_current_recovery('operator')['policy']['claims_in_window'], 13) + with self.assertRaises(RecoveryIneligible): + self.f.core.claim_recovery('operator', iid) + # Lowering the cap again does not roll back the newly issued grant. + self.f.core = self.configured(12) + self.assertEqual(self.f.core.get_current_recovery('operator')['eligibility_reason'], 'already_claimed') + self.assertEqual(self.f.core.get_current_recovery('operator')['policy']['claims_in_window'], 13) + + def test_twenty_fourth_grant_blocks_next_outage_until_oldest_expires(self): + iid = self.f.incident() + start = self.f.now + self.f.seed_claims([start - 80000] + [start - 6000 + n * 60 for n in range(22)]) + self.f.core = self.configured(24) + self.assertEqual(self.f.fresh_state()['policy']['claims_in_window'], 23) + self.f.core.claim_recovery('operator', iid) + self.f.now += 60 + self.f.core.ingest(self.f.observation(healthy=True)) + self.f.fail(); self.f.fail() + blocked = self.f.fail() + self.assertEqual(blocked['policy']['claims_in_window'], 24) + self.assertEqual(blocked['eligibility_reason'], 'daily_limit') + self.assertEqual(blocked['retry_at'], start + 6400) + new_iid = blocked['incident']['incident_id'] + with self.assertRaises(RecoveryIneligible): + self.f.core.claim_recovery('operator', new_iid) + self.f.now = start + 6400 - 0.001 + self.assertEqual(self.f.fresh_state()['eligibility_reason'], 'daily_limit') + self.f.now = start + 6400 + released = self.f.fresh_state() + self.assertEqual(released['policy']['claims_in_window'], 23) + self.assertTrue(released['claimable']) + # Expiry releases eligibility; it never issues a grant on its own. + self.assertEqual(released['incident']['state'], 'pending') + self.assertIsNone(released['incident']['outcome'])