fix(admission): close the non-UTF-8 shortlist hole and pin the claims #176 made

keel/cli.py

@@ -848,7 +848,11 @@ def assets_propose(ctx: click.Context, from_file: str, as_json: bool) -> None:
     repo = _open_repo(ctx)
     try:
         raw = json.loads(Path(from_file).read_text())
-    except (OSError, json.JSONDecodeError) as exc:
+    # `UnicodeDecodeError` subclasses ValueError, NOT OSError, so it needs naming here or a
+    # non-UTF-8 shortlist (a scout run on Windows writes UTF-16LE+BOM -- valid JSON, undecodable
+    # as UTF-8) crashes out with a raw traceback instead of this message. Same fix, same reason,
+    # as `keel.commands.admission.build_propose_view`'s own read.
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
         raise click.ClickException(f"could not read/parse {from_file}: {exc}") from exc
     try:
         parsed = parse_proposal(raw)

keel/commands/admission.py

@@ -217,7 +217,8 @@ def build_propose_view(
     `Path(config.proposals_dir).expanduser()`.
 
     Every failure mode is FAIL-SOFT: a missing directory, an empty one, an unreadable file
-    (`OSError` -- e.g. a permissions problem, or the file vanishing mid-read), invalid JSON
+    (`OSError` -- a permissions problem, the file vanishing mid-read -- or `UnicodeDecodeError`,
+    which is NOT an `OSError` and needs naming separately; see the read below), invalid JSON
     (`json.JSONDecodeError`), or a malformed top-level shortlist (`ProposalError` from
     `parse_proposal`, e.g. `{"candidates": "nope"}`) each produce a `ProposeView` carrying the
     matching `status` and a plain-English `detail` -- never an exception. This function backs a
@@ -266,7 +267,14 @@ def build_propose_view(
 
     try:
         raw_text = source.read_text()
-    except OSError as exc:
+    except (OSError, UnicodeDecodeError) as exc:
+        # `UnicodeDecodeError` subclasses **ValueError, not OSError** -- so `except OSError` alone
+        # let a non-UTF-8 shortlist escape this function entirely, breaking the "Never raises"
+        # contract above. It is not a hypothetical: a scout run on Windows writes UTF-16LE+BOM,
+        # which is perfectly valid JSON and unreadable here. Uncaught, the TUI's propose overlay
+        # repainted `'utf-8' codec can't decode byte 0xff...` every poll forever, naming neither
+        # the file nor a next step, and `keel assets propose --from` exited on a raw traceback.
+        # Both now get the same calm, file-naming `unreadable` report a permissions error gets.
         return ProposeView(
             source=source,
             status="unreadable",

keel/commands/tui.py

@@ -37,14 +37,17 @@
   shortlist file in `config.proposals_dir` (or names why there is none). OFFLINE, DB + local
   filesystem reads only.
 - `d` **discover** -- `build_discover_overlay` over `build_discover_report`: proposes NEW
-  candidates from the venue's own product list. This is the one of the three that needs the
-  network, and it is the SECOND deliberate network exception in this dashboard (the first is `f`
-  fetch): opening the overlay makes no call at all (it renders an ARMED, not-yet-run
-  explanation), and only an explicit Enter keypress *inside* the overlay triggers
-  `_do_discover_report`'s one `_build_broker(config).list_products()` call. The result is then
-  HELD -- every following poll while the overlay stays open repaints the same cached result (or
-  error) rather than re-fetching, and closing the overlay discards it, so reopening is armed but
-  not yet run again.
+  candidates from the venue's own product list. This is the one of the three overlays that needs
+  the network, and it is the THIRD deliberate network exception in this dashboard -- the other
+  two being the automatic ~30s live-balance refresh (`_refresh_balance`, a real `get_accounts`
+  call that has been firing on its own cadence since v3) and `f` fetch. Counting only fetch, as
+  this docstring used to, understates by one and tells an operator the dashboard is offline
+  between keypresses when it is not. Opening the overlay makes no call at all (it renders an
+  ARMED, not-yet-run explanation), and only an explicit Enter keypress *inside* the overlay
+  triggers `_do_discover_report`'s one `_build_broker(config).list_products()` call. The result
+  is then HELD -- every following poll while the overlay stays open repaints the same cached
+  result (or error) rather than re-fetching, and closing the overlay discards it, so reopening is
+  armed but not yet run again.
 
 None of the three attests, admits, or trades -- `attest` (the human judgment the whole gate rests
 on) stays deliberately CLI-only, `keel assets attest`. `screen`/`propose`/`discover` only ever
@@ -390,6 +393,9 @@ def _note(text: str) -> None:
     _row("Live balance")
     _note("  'live account' shows the REAL account's spendable quote balance (e.g. USDC),")
     _note("  refreshed every ~30s and immediately on 'r' or 'f' -- so a deposit or sell shows up.")
+    _note("  Each refresh is a LIVE call to the venue (get_accounts) -- one of the three network")
+    _note("  touches this dashboard makes, and the only one that happens without a keypress. It")
+    _note("  is a read: it places no orders and changes nothing.")
     _note("  In paper mode, paper buys spend paper_cash_usdc instead -- not this balance.")
     lines.append(_blank())
     _row("Help mode (this screen)")
@@ -457,11 +463,12 @@ def _note(text: str) -> None:
     _note("  reads only. Neither ever constructs a broker or touches the network.")
     lines.append(_blank())
     _note(
-        "  discover is the SECOND deliberate network exception in this dashboard (after fetch):"
+        "  discover is the THIRD deliberate network exception in this dashboard. The other two"
     )
-    _note("  it never fires on opening the overlay and never fires again on its own while the")
-    _note("  overlay stays open -- only an explicit Enter, pressed inside it, runs the one live")
-    _note("  venue call it ever makes.")
+    _note("  are the automatic ~30s live-balance refresh above and [f] fetch -- three in total,")
+    _note("  and nothing else here ever leaves this machine. Discover never fires on opening the")
+    _note("  overlay and never fires again on its own while the overlay stays open -- only an")
+    _note("  explicit Enter, pressed inside it, runs the one live venue call it ever makes.")
     lines.append(_blank())
     _note(
         "  NONE of screen, propose or discover attests, admits, or trades. They can only PROPOSE"
@@ -669,9 +676,10 @@ def build_discover_overlay(
         )
         lines.append(
             ScreenLine(
-                "This is the second deliberate network exception in this dashboard (the first "
-                "is [f] fetch): it never fires just from opening this overlay, and it never "
-                "fires again on its own while this overlay stays open.",
+                "This is the third deliberate network exception in this dashboard -- the other "
+                "two are the automatic ~30s live-balance refresh and [f] fetch. It never fires "
+                "just from opening this overlay, and it never fires again on its own while this "
+                "overlay stays open.",
                 "normal",
             )
         )
@@ -987,9 +995,17 @@ def _do_propose_view(open_state: OpenState) -> ProposeView:
     """Build a fresh `ProposeView` over the newest shortlist in `config.proposals_dir` -- OFFLINE
     (DB + local filesystem reads only), rebuilt every poll while the propose overlay is open.
     `_screen_product` is lazy-imported for the identical reason `_do_screen_report` lazy-imports
-    it. `build_propose_view` itself never raises (see its docstring) -- any exception reaching
-    this function's caller can only come from `open_state()` (e.g. a locked DB), never from the
-    shortlist read itself."""
+    it.
+
+    `build_propose_view` is fail-soft about the shortlist FILE -- a missing directory, a
+    permissions error, a non-UTF-8 file, invalid JSON, a malformed top-level shape all come back
+    as a `status`/`detail` pair. That is not the same as "this function cannot raise", and the
+    earlier claim that any exception here could only come from `open_state()` was simply wrong:
+    it once let a `UnicodeDecodeError` from the read itself through (that specific hole is now
+    closed -- see `build_propose_view`'s own `except (OSError, UnicodeDecodeError)`), and
+    `build_propose_view` still SCREENS every parsed candidate afterwards, which means real DB
+    reads through `_screen_product`. A locked DB, mid-screen, surfaces here exactly like one from
+    `open_state()` does. The caller's `try/except` is load-bearing for both."""
     from keel.cli import _screen_product
 
     repo, config = open_state()
@@ -1026,7 +1042,7 @@ def run_live(open_state: OpenState, now_fn: NowFn, interval: float) -> None:
     process -- gathers a fresh report, paints it, then waits up to `interval` seconds for a
     keypress.
 
-    Seven modes: `normal` (the dashboard, plus a transient one-line `message` toast from the last
+    Six modes: `normal` (the dashboard, plus a transient one-line `message` toast from the last
     action), `help` (a scrolled window of `build_help_screen()`), `insights` (a scrolled window of
     `build_insights_screen()` -- a READ-ONLY overlay over `build_insights_report`/
     `build_journal_report`, rebuilt fresh each poll while open, fail-soft exactly like the
@@ -1179,9 +1195,15 @@ def _loop(stdscr: Any) -> None:
                 continue
 
             if mode == "propose":
-                # OFFLINE + fail-soft, same shape as screen above. `_do_propose_view` itself
-                # never raises (`build_propose_view`'s own docstring) -- an exception here can
-                # only come from `open_state()` (e.g. a locked DB), never from the shortlist read.
+                # OFFLINE + fail-soft, same shape as screen above. `build_propose_view` turns
+                # every SHORTLIST-FILE problem into a calm `status`/`detail` pair rather than an
+                # exception, so this handler is not what renders "no shortlist yet" or "not valid
+                # JSON" -- the overlay does. It catches what is left: a locked DB, either from
+                # `open_state()` or from the per-candidate screening `build_propose_view` runs
+                # after parsing. It is also the net that caught a `UnicodeDecodeError` escaping
+                # the read for a non-UTF-8 shortlist -- as an unreadable `'utf-8' codec can't
+                # decode byte 0xff...` toast, repainted every poll, naming no file. That hole is
+                # fixed at the source now; this stays as the backstop, not as the explanation.
                 try:
                     propose_lines = build_propose_overlay(_do_propose_view(open_state))
                 except Exception as exc:

keel/compliance/screen.py

@@ -269,12 +269,27 @@ def split_failures(facts: MarketFacts, result: ScreenResult) -> tuple[list[str],
     there is exactly one place either can change.
 
     With `facts.daily_bars > 0` every failure is returned as `about_the_asset` and
-    `about_our_cache` is empty, UNSPLIT -- including a genuine `history` shortfall, because an
-    asset with some cached history that still falls short of the floor really is too young, and
-    that verdict must not be silenced. The split only ever happens at EXACTLY zero bars, which is
-    the one condition where `history` (and `liquidity`) measure our cache instead of the asset.
-    Original ordering is preserved within each returned list, so a caller that renders them in
-    order does not see failures reshuffled relative to how `screen_asset` produced them.
+    `about_our_cache` is empty, UNSPLIT -- including a `history` shortfall.
+
+    Be clear about what that does and does not mean. It does NOT mean a shallow cache proves the
+    asset is young. `MarketFacts` carries a bar COUNT and no first-bar timestamp, so this function
+    cannot distinguish "listed 18 months ago" from "we fetched an 18-month window": `keel fetch
+    --years 2`, a fetch that aborted partway, or a venue that simply does not serve the full
+    window all leave an OLD asset shallow (`keel/cli.py`'s `fetch` prints a note about exactly
+    that -- "some series are still short... an asset younger than the requested window"). Every
+    surface nonetheless renders `✗ history: 730 daily bars < 1460 required` as a verdict, because
+    a partial cache is genuinely ambiguous and the gate must fail closed on ambiguity rather than
+    admit on it.
+
+    So the operator reading a non-zero `history` failure should CHECK THE FETCH WINDOW before
+    concluding the asset is too young: `keel fetch --products <id> --years 5`, then re-screen. If
+    the count does not move, it is the asset.
+
+    The split is confined to EXACTLY zero bars because that is the only count where there is no
+    ambiguity to resolve: with no candles at all, `history` and `liquidity` are reporting the
+    emptiness of our cache and nothing whatsoever about the asset. Original ordering is preserved
+    within each returned list, so a caller that renders them in order does not see failures
+    reshuffled relative to how `screen_asset` produced them.
     """
     if facts.daily_bars > 0:
         return list(result.failures), []
@@ -304,11 +319,19 @@ def missing_history_lines(product_id: str, not_assessable: Sequence[str]) -> lis
     after the colon. Tags are deduplicated and sorted so two failures sharing a tag collapse to
     one mention and the order is stable regardless of how `screen_asset` happened to emit them.
     """
+    # The second line is the SEMANTIC one -- it says what a zero-bar report means -- and it is
+    # deliberately agnostic about the asset. It used to read "it is not too young, we have simply
+    # never fetched candles for it", which asserts something we cannot know: at exactly zero bars
+    # a listing three days old and one three years old are the same input, since `MarketFacts`
+    # carries no first-bar timestamp. Refusing to rule is the honest answer, and it is also the
+    # useful one -- it points at the fetch, which is the only thing that can resolve it.
+    # `test_missing_history_lines_claim_nothing_about_the_assets_age` pins this.
     lines = [
         f"no local history for {product_id} -- run `keel fetch --products {product_id}` first, "
         "then re-screen.",
-        "This is a MISSING-DATA verdict, not a verdict about the asset: it is not too young, "
-        "we have simply never fetched candles for it.",
+        "This is a MISSING-DATA verdict, not a verdict about the asset: with no candles at all "
+        "we cannot tell a genuinely young asset from one we have simply never fetched, so this "
+        "says nothing about its age either way.",
     ]
     if not_assessable:
         tags = sorted({f.split(":")[0] for f in not_assessable})

keel/proposer.py

@@ -225,8 +225,27 @@ def render_proposal_report(report: ProposalReport) -> list[str]:
 
 
 def report_to_jsonable(report: ProposalReport) -> dict[str, Any]:
-    return {
-        "screened": [
+    """The `--json` shape -- and it must tell the SAME story `render_proposal_report` tells.
+
+    It used to emit `sc.result.failures` raw, so at zero cached bars the payload carried
+    `history: 0 daily bars < 1460 required` as a plain failure while every human surface
+    (`render_proposal_report` two functions up, `assets holdings --screen`, `assets screen`, the
+    TUI's screen/propose overlays) suppressed that exact line and printed the MISSING-DATA
+    explanation instead. One report, two surfaces, two different answers to "is this asset too
+    young?" -- the drift `screen.split_failures` exists to end, surviving in the one surface a
+    script reads and cannot argue with.
+
+    So the same `split_failures` the renderers call decides this too, and the split is made
+    EXPLICIT rather than merely applied: `failures` is what the human report prints with a `✗`
+    (verdicts about the asset), `not_assessable` is what it withholds at zero bars (statements
+    about the emptiness of OUR cache -- suppressed, never dropped, so a consumer that wants them
+    can still read them correctly labelled), and `missing_history` is the flag that says which
+    regime the row is in without the consumer having to infer it from `daily_bars == 0`.
+    """
+    rows: list[dict[str, Any]] = []
+    for sc in report.screened:
+        failures, not_assessable = split_failures(sc.facts, sc.result)
+        rows.append(
             {
                 "asset": sc.candidate.asset,
                 "product": sc.product,
@@ -238,11 +257,14 @@ def report_to_jsonable(report: ProposalReport) -> dict[str, Any]:
                 "admitted": sc.result.admitted,
                 "summary": sc.result.summary,
                 "daily_bars": sc.facts.daily_bars,
-                "failures": sc.result.failures,
+                "missing_history": sc.facts.daily_bars == 0,
+                "failures": failures,
+                "not_assessable": not_assessable,
                 "warnings": sc.result.warnings,
             }
-            for sc in report.screened
-        ],
+        )
+    return {
+        "screened": rows,
         "invalid": [{"reason": e.reason, "raw": e.raw} for e in report.invalid],
         "admitted_count": report.admitted_count,
     }