semantic_search: warn when the index predates the current commit

src/embed.lex

@@ -54,7 +54,14 @@ import "std.int" as int
 
 import "lex-schema/json_value" as jv
 
-type Config = { base_url :: Str, model :: Str, dims :: Int }
+# `built_at_commit` is the git HEAD sha the index was built from ("" when
+# not built inside a git checkout, or by an index written before this
+# field existed). It is a staleness HINT, not a correctness guarantee — a
+# commit that never touched a function's signature would not actually
+# need reindexing, and this cannot tell the difference — but "the index
+# might be stale" said once is worth more than silently querying against
+# whatever the corpus looked like at some unstated point in the past.
+type Config = { base_url :: Str, model :: Str, dims :: Int, built_at_commit :: Str }
 
 type Entry = { sig :: Str, name :: Str, file :: Str, text :: Str, vec :: List[Float] }
 
@@ -109,7 +116,7 @@ fn default_dims() -> Int
 }
 
 fn default_config() -> Config {
-  { base_url: default_base_url(), model: default_model(), dims: default_dims() }
+  { base_url: default_base_url(), model: default_model(), dims: default_dims(), built_at_commit: "" }
 }
 
 # Keep the first `dims` components and renormalise. A vector shorter than
@@ -329,17 +336,21 @@ fn entry_text(name :: Str, signature :: Str, effects :: List[Str], examples :: L
 # ---- the index file ---------------------------------------------------
 fn encode_header(cfg :: Config) -> Str
   examples {
-    encode_header({ base_url: "http://x", model: "m", dims: 64 }) => "{\"kind\":\"header\",\"base_url\":\"http://x\",\"model\":\"m\",\"dims\":64}"
+    encode_header({ base_url: "http://x", model: "m", dims: 64, built_at_commit: "abc123" }) => "{\"kind\":\"header\",\"base_url\":\"http://x\",\"model\":\"m\",\"dims\":64,\"built_at_commit\":\"abc123\"}"
   }
 {
-  jv.stringify(JObj([("kind", JStr("header")), ("base_url", JStr(cfg.base_url)), ("model", JStr(cfg.model)), ("dims", JInt(cfg.dims))]))
+  jv.stringify(JObj([("kind", JStr("header")), ("base_url", JStr(cfg.base_url)), ("model", JStr(cfg.model)), ("dims", JInt(cfg.dims)), ("built_at_commit", JStr(cfg.built_at_commit))]))
 }
 
+# A header line with no `built_at_commit` (written before this field
+# existed) decodes with "" — the same "cannot vouch for freshness" value
+# a missing git checkout produces, and the right one: an old index is not
+# LESS possibly-stale than one this session cannot place in git history.
 fn decode_header(line :: Str) -> Option[Config] {
   match jv.parse_into_errors(str.trim(line)) {
     Err(_) => None,
     Ok(j) => match jv.get_field(j, "kind") {
-      Some(JStr("header")) => Some({ base_url: str_field(j, "base_url"), model: str_field(j, "model"), dims: int_field(j, "dims") }),
+      Some(JStr("header")) => Some({ base_url: str_field(j, "base_url"), model: str_field(j, "model"), dims: int_field(j, "dims"), built_at_commit: str_field(j, "built_at_commit") }),
       _ => None,
     },
   }

src/index_build.lex

@@ -39,8 +39,24 @@ import "lex-schema/json_value" as jv
 
 import "./embed" as embed
 
-fn config_from_env() -> [env] embed.Config {
-  { base_url: env_or("LITELLM_BASE_URL", embed.default_base_url()), model: env_or("LEX_EMBED_MODEL", embed.default_model()), dims: dims_from_env() }
+fn config_from_env() -> [env, proc] embed.Config {
+  { base_url: env_or("LITELLM_BASE_URL", embed.default_base_url()), model: env_or("LEX_EMBED_MODEL", embed.default_model()), dims: dims_from_env(), built_at_commit: git_head() }
+}
+
+# "" outside a git checkout (or if `git` itself is unavailable) rather
+# than an error — a missing commit hash is a fact `semantic_search`'s
+# staleness check already treats as "cannot vouch for freshness", the
+# same conservative reading `sig_for` gives an unhashable verified.jsonl
+# target elsewhere in this codebase, not a reason to fail the whole build.
+fn git_head() -> [proc] Str {
+  match proc.run("git", ["rev-parse", "HEAD"]) {
+    Err(_) => "",
+    Ok(out) => if out.exit_code == 0 {
+      str.trim(out.stdout)
+    } else {
+      ""
+    },
+  }
 }
 
 fn dims_from_env() -> [env] Int {
@@ -210,11 +226,11 @@ fn reuse_or_embed(cfg :: embed.Config, docs :: List[Doc]) -> [io, net] Result[Li
 # ranking, which produces plausible-looking nonsense rather than an error.
 fn reusable(cfg :: embed.Config, prev :: Option[embed.Config]) -> Bool
   examples {
-    reusable({ base_url: "u", model: "m", dims: 64 }, Some({ base_url: "u", model: "m", dims: 64 })) => true,
-    reusable({ base_url: "u", model: "m", dims: 64 }, Some({ base_url: "u", model: "other", dims: 64 })) => false,
-    reusable({ base_url: "u", model: "m", dims: 64 }, Some({ base_url: "elsewhere", model: "m", dims: 64 })) => false,
-    reusable({ base_url: "u", model: "m", dims: 64 }, Some({ base_url: "u", model: "m", dims: 128 })) => false,
-    reusable({ base_url: "u", model: "m", dims: 64 }, None) => false
+    reusable({ base_url: "u", model: "m", dims: 64, built_at_commit: "c1" }, Some({ base_url: "u", model: "m", dims: 64, built_at_commit: "c2" })) => true,
+    reusable({ base_url: "u", model: "m", dims: 64, built_at_commit: "c1" }, Some({ base_url: "u", model: "other", dims: 64, built_at_commit: "c1" })) => false,
+    reusable({ base_url: "u", model: "m", dims: 64, built_at_commit: "c1" }, Some({ base_url: "elsewhere", model: "m", dims: 64, built_at_commit: "c1" })) => false,
+    reusable({ base_url: "u", model: "m", dims: 64, built_at_commit: "c1" }, Some({ base_url: "u", model: "m", dims: 128, built_at_commit: "c1" })) => false,
+    reusable({ base_url: "u", model: "m", dims: 64, built_at_commit: "c1" }, None) => false
   }
 {
   match prev {

src/tools/semantic_search.lex

@@ -16,6 +16,8 @@ import "std.str" as str
 
 import "std.list" as list
 
+import "std.process" as proc
+
 import "std.int" as int
 
 import "lex-llm/tool" as t
@@ -68,15 +70,15 @@ fn k_of(args :: jv.Json) -> Int {
 # query is embedded with those rather than with anything configured here.
 # A cosine between vectors from two different models is a number with no
 # meaning, so this is the only correct source for that choice.
-fn search(query :: Str, k :: Int) -> [net, io] Result[jv.Json, e.Errors] {
+fn search(query :: Str, k :: Int) -> [net, io, proc] Result[jv.Json, e.Errors] {
   match embed.read_index() {
     (None, _) => Err(e.single("", "no_index", missing_index_msg())),
     (Some(cfg), entries) => if list.is_empty(entries) {
       Err(e.single("", "empty_index", missing_index_msg()))
     } else {
       match embed.embed_one(cfg, query) {
         Err(msg) => Err(e.single("", "embed_failed", msg)),
-        Ok(qvec) => Ok(JStr(render(query, embed.top_k(embed.score_all(entries, embed.truncate(qvec, cfg.dims)), k), cfg))),
+        Ok(qvec) => Ok(JStr(render(query, embed.top_k(embed.score_all(entries, embed.truncate(qvec, cfg.dims)), k), cfg, git_head()))),
       }
     },
   }
@@ -86,15 +88,50 @@ fn missing_index_msg() -> Str {
   str.join(["no semantic index at ", embed.index_path(), ". Build one with:\n\n  lex run --allow-effects env,io,net,proc src/index_build.lex main\n\nIt needs a LiteLLM proxy reachable at LITELLM_BASE_URL (default ", embed.default_base_url(), ") serving an embedding model as LEX_EMBED_MODEL (default ", embed.default_model(), ")."], "")
 }
 
+# "" outside a git checkout or if `git` itself fails — same convention as
+# index_build.lex's own git_head, duplicated rather than shared: two
+# independent tool-facing files each small enough that a shared import
+# would cost more than it saves.
+fn git_head() -> [proc] Str {
+  match proc.run("git", ["rev-parse", "HEAD"]) {
+    Err(_) => "",
+    Ok(out) => if out.exit_code == 0 {
+      str.trim(out.stdout)
+    } else {
+      ""
+    },
+  }
+}
+
+# A heuristic, not a guarantee: a commit that never touched a function
+# signature would not actually make the index stale, and this cannot
+# tell the difference (see embed.Config's own comment on
+# built_at_commit). Empty on either side means "cannot vouch either
+# way" — silence, not a false "up to date."
+fn staleness_note(cfg :: embed.Config, current_commit :: Str) -> Str
+  examples {
+    staleness_note({ base_url: "u", model: "m", dims: 8, built_at_commit: "abc" }, "abc") => "",
+    staleness_note({ base_url: "u", model: "m", dims: 8, built_at_commit: "abc" }, "def") => "\n\nNOTE: this index was built at commit abc, HEAD is now def — it may be missing or misdescribing functions changed since. Rebuild with src/index_build.lex if results look stale.",
+    staleness_note({ base_url: "u", model: "m", dims: 8, built_at_commit: "" }, "def") => "",
+    staleness_note({ base_url: "u", model: "m", dims: 8, built_at_commit: "abc" }, "") => ""
+  }
+{
+  if str.is_empty(cfg.built_at_commit) or str.is_empty(current_commit) or cfg.built_at_commit == current_commit {
+    ""
+  } else {
+    str.join(["\n\nNOTE: this index was built at commit ", cfg.built_at_commit, ", HEAD is now ", current_commit, " — it may be missing or misdescribing functions changed since. Rebuild with src/index_build.lex if results look stale."], "")
+  }
+}
+
 # The scores are cosine similarities, and they are only meaningful next to
 # each other: a top hit at 62% on a small corpus can be the right answer,
 # and one at 88% can be wrong. Saying so is cheaper than having a model
 # treat the number as a confidence and stop reading at the first result.
-fn render(query :: Str, hits :: List[embed.Hit], cfg :: embed.Config) -> Str {
+fn render(query :: Str, hits :: List[embed.Hit], cfg :: embed.Config, current_commit :: Str) -> Str {
   if list.is_empty(hits) {
-    str.join(["no matches for \"", query, "\""], "")
+    str.join(["no matches for \"", query, "\"", staleness_note(cfg, current_commit)], "")
   } else {
-    str.join([str.join([int.to_str(list.len(hits)), " nearest to \"", query, "\" (", cfg.model, "):"], ""), "\n\n", embed.render_hits(hits), "\n\nScores are cosine similarity, useful only relative to each other — read the top few rather than trusting the number. The index covers functions as of the last build; run src/index_build.lex after adding code."], "")
+    str.join([str.join([int.to_str(list.len(hits)), " nearest to \"", query, "\" (", cfg.model, "):"], ""), "\n\n", embed.render_hits(hits), "\n\nScores are cosine similarity, useful only relative to each other — read the top few rather than trusting the number.", staleness_note(cfg, current_commit)], "")
   }
 }