From a0bf154efc907d0ea3f7cb288bc80565fcbde221 Mon Sep 17 00:00:00 2001 From: Anders Robstad Date: Tue, 29 Sep 2026 13:53:17 +0200 Subject: [PATCH] Keep the loader cache the size of the spec, and off every MCP call A spec whose schemas refer to each other in a ring wrote a 2.5 GB loader cache: every endpoint's schemas were stored with each $ref expanded along every path through the ring. Every MCP call read that file first, so the server answered `initialize` and then hung. Schemas now live in .schemas.json as the document wrote them, each reachable definition once, and are bundled per endpoint on request with shared definitions under $defs. .json holds only the endpoint list, which is all listing, searching and access decisions read. A cache from before the split is read only as far as its endpoint list. Request-body skeletons get a node budget, since the depth limit alone still let them branch into hundreds of kilobytes. --- .changeset/loader-cache-stays-small.md | 28 + src-tauri/src/lib.rs | 45 +- src-tauri/src/loader.rs | 312 ++++++-- src-tauri/src/mcp.rs | 183 ++++- src-tauri/src/openapi.rs | 462 ++++++++++-- src-tauri/tests/fixtures/cyclic-union.json | 822 +++++++++++++++++++++ tests/e2e/json-schema.spec.ts | 25 + 7 files changed, 1722 insertions(+), 155 deletions(-) create mode 100644 .changeset/loader-cache-stays-small.md create mode 100644 src-tauri/tests/fixtures/cyclic-union.json diff --git a/.changeset/loader-cache-stays-small.md b/.changeset/loader-cache-stays-small.md new file mode 100644 index 0000000..75b5fa1 --- /dev/null +++ b/.changeset/loader-cache-stays-small.md @@ -0,0 +1,28 @@ +--- +"fiber": patch +--- + +A collection whose OpenAPI schemas refer to each other in a ring no longer +writes a loader cache of gigabytes, and the MCP server no longer hangs on one. + +The cache stored every endpoint's request and response schema with each `$ref` +expanded in place, stopping only where a reference met itself on the current +path. Schemas that refer to each other — an operand that is an object or a +function, each of which holds operands — have a number of such paths that +grows factorially, and every endpoint got its own copy: one real collection's +cache reached 2.5 GB, a single endpoint's schema 252 MB. Every MCP call, down +to `list_sections`, read that file first, so the server answered `initialize` +and then nothing else. + +Schemas now live in a file of their own beside the endpoint list, stored as the +document wrote them, with each definition they reach kept once. An endpoint's +schema is put together when it is opened or asked for by `get_endpoint`, with +anything used more than once written a single time under `$defs` — which the +editor's linting already reads — so it can never be larger than the document it +came from. Listing, searching and deciding whether a call is allowed read only +the endpoint list. Request bodies built from such schemas are bounded too, +since a depth limit alone still let them branch into hundreds of kilobytes. + +A cache written by an earlier version still lists its endpoints straight away: +reading stops at the end of the endpoint list, before the schemas. Its schemas +are left unread until the next refresh writes both files. diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 4815313..986362d 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -73,14 +73,19 @@ mod gui { /// `loader_schema` used to deserialize the whole on-disk file — every /// expanded schema included — on each endpoint click. The first read (or /// the last successful run) lives here so later clicks are a map lookup. + /// + /// Schemas are held apart from endpoints, and read the first time one is + /// asked for: the sidebar never needs them. struct LoaderMem { inner: Mutex>, + schemas: Mutex>>, } impl LoaderMem { fn new() -> Self { Self { inner: Mutex::new(HashMap::new()), + schemas: Mutex::new(HashMap::new()), } } @@ -112,11 +117,20 @@ mod gui { } fn schema(&self, dir: &Path, section_id: &str, endpoint_id: &str) -> EndpointSchemas { - let mut map = self.lock(); - let cache = Self::fill(&mut map, dir, section_id); + let schemas = self + .schemas + .lock() + .unwrap_or_else(|err| err.into_inner()) + .entry(section_id.to_string()) + .or_insert_with(|| { + Arc::new(loader::read_schemas(dir, section_id).unwrap_or_default()) + }) + .clone(); + // Outside the lock: bundling is per endpoint and needs only the + // shared, immutable copy. EndpointSchemas { - request: cache.schemas.get(endpoint_id).cloned(), - response: cache.response_schemas.get(endpoint_id).cloned(), + request: schemas.request_for(endpoint_id), + response: schemas.response_for(endpoint_id), } } @@ -125,12 +139,25 @@ mod gui { Self::fill(&mut map, dir, section_id).endpoints.clone() } - fn remember(&self, section_id: &str, cache: loader::LoaderCache) { + fn remember( + &self, + section_id: &str, + cache: loader::LoaderCache, + schemas: loader::LoaderSchemas, + ) { self.lock().insert(section_id.to_string(), cache); + self.schemas + .lock() + .unwrap_or_else(|err| err.into_inner()) + .insert(section_id.to_string(), Arc::new(schemas)); } fn forget(&self, section_id: &str) { self.lock().remove(section_id); + self.schemas + .lock() + .unwrap_or_else(|err| err.into_inner()) + .remove(section_id); } } @@ -513,7 +540,7 @@ mod gui { §ion, ); - let (endpoints, schemas, response_schemas, pages) = loader::run(&config, fetcher).await?; + let (endpoints, schemas, pages) = loader::run(&config, fetcher).await?; let previous = mem.endpoints(&paths.loaders, §ion_id); let (added, removed) = loader::diff(&previous, &endpoints); @@ -522,13 +549,11 @@ mod gui { let cache = loader::LoaderCache { loaded_at, endpoints: endpoints.clone(), - schemas, - response_schemas, }; - if let Err(err) = loader::write_cache(&paths.loaders, §ion_id, &cache) { + if let Err(err) = loader::write_cache(&paths.loaders, §ion_id, &cache, &schemas) { ::log::warn!("could not cache loader output: {err}"); } - mem.remember(§ion_id, cache); + mem.remember(§ion_id, cache, schemas); Ok(LoaderRun { endpoints, diff --git a/src-tauri/src/loader.rs b/src-tauri/src/loader.rs index 95c3d9d..d65466b 100644 --- a/src-tauri/src/loader.rs +++ b/src-tauri/src/loader.rs @@ -177,22 +177,66 @@ impl LoadedEndpoint { } } +/// What the last run found, as `.json`: everything a listing, a search or +/// an access decision needs, and nothing else. +/// +/// Schemas used to live in this file too, which put them in the path of every +/// MCP call — `list_sections` needs an endpoint count, and got a 2.5 GB file to +/// parse for it. They are in [`LoaderSchemas`] now, read only when one is +/// asked for. A file written before the split still reads: its `schemas` and +/// `responseSchemas` are skipped rather than kept, since what they hold is the +/// unbounded expansion this format exists to avoid. #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct LoaderCache { /// Epoch millis of the last successful run. pub loaded_at: i64, pub endpoints: Vec, - /// Request-body schemas keyed by endpoint id. Kept out of the normal cache - /// response: a large OpenAPI document can share the same component schema - /// hundreds of times, and the editor only needs one schema at a time. +} + +/// Request and response schemas for a loader's endpoints, as +/// `.schemas.json`. +/// +/// Stored the way the document wrote them, `$ref`s and all, with each +/// definition those refer to held once under `definitions`. An endpoint's +/// schema is made self-contained only when it is asked for — see +/// [`crate::openapi::bundle`] — so the file stays the size of the document, +/// not of its expansion. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct LoaderSchemas { + /// Keyed by the `$ref` that names each one, e.g. + /// `#/components/schemas/User`. #[serde(default)] - pub schemas: BTreeMap, - /// Successful-response schemas, keyed the same way. Separate so a collection - /// whose request schemas already fill the cache does not grow a second copy - /// of every component for the response pane that is not yet open. + pub definitions: BTreeMap, + /// Request-body schemas, keyed by endpoint id. #[serde(default)] - pub response_schemas: BTreeMap, + pub request: BTreeMap, + /// Successful-response schemas, keyed the same way. + #[serde(default)] + pub response: BTreeMap, +} + +impl LoaderSchemas { + /// One endpoint's request schema, ready to validate against. + pub fn request_for(&self, key: &str) -> Option { + self.request + .get(key) + .map(|schema| crate::openapi::bundle(schema, &self.definitions)) + } + + /// One endpoint's response schema, ready to validate against. + pub fn response_for(&self, key: &str) -> Option { + self.response + .get(key) + .map(|schema| crate::openapi::bundle(schema, &self.definitions)) + } + + fn extend(&mut self, other: LoaderSchemas) { + self.definitions.extend(other.definitions); + self.request.extend(other.request); + self.response.extend(other.response); + } } /// What a run produced, including what changed since last time. @@ -414,15 +458,7 @@ fn kind_of(value: &serde_json::Value) -> &'static str { pub async fn run( config: &LoaderConfig, fetcher: Fetcher, -) -> Result< - ( - Vec, - BTreeMap, - BTreeMap, - usize, - ), - LoaderError, -> { +) -> Result<(Vec, LoaderSchemas, usize), LoaderError> { if config.url.trim().is_empty() { return Err(LoaderError::NoUrl); } @@ -438,8 +474,7 @@ pub async fn run( let mut url = config.url.trim().to_string(); let mut endpoints = Vec::new(); - let mut schemas = BTreeMap::new(); - let mut response_schemas = BTreeMap::new(); + let mut schemas = LoaderSchemas::default(); let mut identities = HashSet::new(); let mut pages = 0; @@ -450,9 +485,7 @@ pub async fn run( // METHOD/path, so keep the first rather than emitting duplicate // keyed rows into the sidebar and overlay. mapped.retain(|endpoint| identities.insert(endpoint.key())); - let (request, response) = enrich_openapi(&document, &mut mapped); - schemas.extend(request); - response_schemas.extend(response); + schemas.extend(enrich_openapi(&document, &mut mapped)); endpoints.extend(mapped); pages += 1; @@ -466,7 +499,7 @@ pub async fn run( } } - Ok((endpoints, schemas, response_schemas, pages)) + Ok((endpoints, schemas, pages)) }) .await .map_err(|_| LoaderError::Timeout)? @@ -580,19 +613,12 @@ fn clip(text: &str) -> String { /// /// Anything else — a routes array, a bespoke manifest — has no `paths`, so this /// does nothing and every endpoint keeps what the filter produced. -fn enrich_openapi( - document: &serde_json::Value, - endpoints: &mut [LoadedEndpoint], -) -> ( - BTreeMap, - BTreeMap, -) { +fn enrich_openapi(document: &serde_json::Value, endpoints: &mut [LoadedEndpoint]) -> LoaderSchemas { let Some(paths) = document.get("paths").and_then(|paths| paths.as_object()) else { - return (BTreeMap::new(), BTreeMap::new()); + return LoaderSchemas::default(); }; - let mut schemas = BTreeMap::new(); - let mut response_schemas = BTreeMap::new(); + let mut schemas = LoaderSchemas::default(); for endpoint in endpoints { let Some(item) = paths.get(&endpoint.path).and_then(|item| item.as_object()) else { @@ -631,14 +657,16 @@ fn enrich_openapi( endpoint.form = form; } if let Some(schema) = crate::openapi::request_schema(document, operation) { - schemas.insert(endpoint.key(), schema); + crate::openapi::collect_definitions(document, schema, &mut schemas.definitions); + schemas.request.insert(endpoint.key(), schema.clone()); } if let Some(schema) = crate::openapi::response_schema(document, operation) { - response_schemas.insert(endpoint.key(), schema); + crate::openapi::collect_definitions(document, schema, &mut schemas.definitions); + schemas.response.insert(endpoint.key(), schema.clone()); } } - (schemas, response_schemas) + schemas } /// `/loaders` @@ -647,43 +675,121 @@ pub fn loaders_dir(app_data_dir: &std::path::Path) -> std::path::PathBuf { } fn cache_path(dir: &std::path::Path, section_id: &str) -> Option { - // Same guard as section files: an id becomes a file name. + // Same guard as section files: an id becomes a file name. It also keeps + // the two names below apart, since an id cannot contain a dot. crate::store::is_safe_id(section_id).then(|| dir.join(format!("{section_id}.json"))) } +pub fn schemas_path(dir: &std::path::Path, section_id: &str) -> Option { + crate::store::is_safe_id(section_id).then(|| dir.join(format!("{section_id}.schemas.json"))) +} + /// The last successful run. Loader output is a cache, never the source of /// truth, so a missing or unreadable file is simply "nothing loaded yet". +/// +/// Streamed, and stops as soon as it has both fields. A file written before +/// schemas moved out holds its gigabytes *after* the endpoint list — the +/// struct wrote its fields in order — so this reads the few hundred kilobytes +/// in front and never the rest, where parsing the whole file took long enough +/// to look like a hang. pub fn read_cache(dir: &std::path::Path, section_id: &str) -> Option { - let path = cache_path(dir, section_id)?; - let text = std::fs::read_to_string(path).ok()?; - serde_json::from_str(&text).ok() + let file = std::fs::File::open(cache_path(dir, section_id)?).ok()?; + let mut found = None; + let mut reader = + serde_json::Deserializer::from_reader(std::io::BufReader::with_capacity(1 << 16, file)); + // Stopping early leaves the object unclosed, which serde_json reports as + // an error once the visitor returns. That error is expected; whether the + // fields were found is what `found` says. + let _ = serde::Deserializer::deserialize_map(&mut reader, CacheHead(&mut found)); + found } +/// Reads `loadedAt` and `endpoints`, and stops. +struct CacheHead<'a>(&'a mut Option); + +impl<'de> serde::de::Visitor<'de> for CacheHead<'_> { + type Value = (); + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a loader cache") + } + + fn visit_map>(self, mut map: A) -> Result<(), A::Error> { + let (mut loaded_at, mut endpoints) = (None, None); + while let Some(key) = map.next_key::()? { + match key.as_str() { + "loadedAt" => loaded_at = Some(map.next_value()?), + "endpoints" => endpoints = Some(map.next_value()?), + _ => { + map.next_value::()?; + } + } + if loaded_at.is_some() && endpoints.is_some() { + break; + } + } + if let (Some(loaded_at), Some(endpoints)) = (loaded_at, endpoints) { + *self.0 = Some(LoaderCache { + loaded_at, + endpoints, + }); + } + Ok(()) + } +} + +/// The schemas from the last successful run, when there are any. +pub fn read_schemas(dir: &std::path::Path, section_id: &str) -> Option { + let file = std::fs::File::open(schemas_path(dir, section_id)?).ok()?; + serde_json::from_reader(std::io::BufReader::with_capacity(1 << 16, file)).ok() +} + +/// Writes both halves of a run. Schemas first: a reader that sees the new +/// endpoint list then finds schemas at least as new, and a schema for an +/// endpoint that is not listed yet is never asked for. pub fn write_cache( dir: &std::path::Path, section_id: &str, cache: &LoaderCache, + schemas: &LoaderSchemas, ) -> std::io::Result<()> { - let Some(path) = cache_path(dir, section_id) else { + let (Some(path), Some(schemas_path)) = + (cache_path(dir, section_id), schemas_path(dir, section_id)) + else { return Ok(()); }; std::fs::create_dir_all(dir)?; - let encoded = serde_json::to_string_pretty(cache) + + // Compact: this half is never read by a person, and indenting a deeply + // nested schema costs more than the schema. + let encoded = serde_json::to_vec(schemas) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; + write_atomic(&schemas_path, &encoded)?; + + let encoded = serde_json::to_vec_pretty(cache) .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; + write_atomic(&path, &encoded) +} - // Same write discipline as section files — temp, sync, rename — and the - // same per-process temp name, because the app and a headless `fiber mcp` - // can both refresh the same loader. - let temp = dir.join(format!("{section_id}.json.tmp-{}", std::process::id())); +/// Same write discipline as section files — temp, sync, rename — and the same +/// per-process temp name, because the app and a headless `fiber mcp` can both +/// refresh the same loader. +fn write_atomic(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> { + let mut temp = path.as_os_str().to_owned(); + temp.push(format!(".tmp-{}", std::process::id())); + let temp = std::path::PathBuf::from(temp); let mut file = std::fs::File::create(&temp)?; - std::io::Write::write_all(&mut file, encoded.as_bytes())?; + std::io::Write::write_all(&mut file, bytes)?; file.sync_all()?; drop(file); - std::fs::rename(&temp, &path) + std::fs::rename(&temp, path) } pub fn forget_cache(dir: &std::path::Path, section_id: &str) { - if let Some(path) = cache_path(dir, section_id) { + for path in [cache_path(dir, section_id), schemas_path(dir, section_id)] + .into_iter() + .flatten() + { let _ = std::fs::remove_file(path); } } @@ -739,7 +845,7 @@ mod tests { .find(|(name, _)| *name == "Array of routes") .unwrap() .1; - let (endpoints, _schemas, _responses, pages) = run( + let (endpoints, _schemas, pages) = run( &config(routes), answering( r#"{"routes":[ @@ -851,7 +957,7 @@ mod tests { let mut settings = config(".routes | map({method: .verb, path: .url})"); settings.next = ".links.next".to_string(); - let (endpoints, _schemas, _responses, pages) = run(&settings, fetcher).await.unwrap(); + let (endpoints, _schemas, pages) = run(&settings, fetcher).await.unwrap(); assert_eq!(pages, 2); assert_eq!( endpoints @@ -879,7 +985,7 @@ mod tests { let mut settings = config(".routes | map({method: .verb, path: .url})"); settings.next = ".links.next".to_string(); - let (endpoints, _schemas, _responses, pages) = run(&settings, fetcher).await.unwrap(); + let (endpoints, _schemas, pages) = run(&settings, fetcher).await.unwrap(); assert_eq!(pages, MAX_PAGES); // The cap still stops the bad pointer, while overlapping pages no longer // create 50 sidebar rows with the same endpoint identity. @@ -933,8 +1039,7 @@ mod tests { .find(|(name, _)| *name == "OpenAPI") .unwrap() .1; - let (endpoints, schemas, _responses, _) = - run(&config(openapi), answering(manifest)).await.unwrap(); + let (endpoints, schemas, _) = run(&config(openapi), answering(manifest)).await.unwrap(); let post = endpoints.iter().find(|e| e.method == "POST").unwrap(); // Type names rather than empty values — the editor turns these into @@ -948,7 +1053,8 @@ mod tests { assert!(post.body.contains("\"dryRun\": boolean"), "{}", post.body); assert_eq!( schemas - .get("POST /activity/backfill-activity") + .request_for("POST /activity/backfill-activity") + .as_ref() .and_then(|schema| schema.pointer("/properties/dryRun/type")) .and_then(|kind| kind.as_str()), Some("boolean") @@ -986,7 +1092,7 @@ mod tests { .find(|(name, _)| *name == "OpenAPI") .unwrap() .1; - let (endpoints, _, _, _) = run(&config(openapi), answering(manifest)).await.unwrap(); + let (endpoints, _, _) = run(&config(openapi), answering(manifest)).await.unwrap(); let search = endpoints .iter() @@ -1040,19 +1146,107 @@ mod tests { let settings = config( r#".paths | to_entries | map(.key as $path | .value | to_entries | map({method: .key, path: $path, body: "{\"enabled\": true}"})) | flatten"#, ); - let (endpoints, schemas, _, _) = run(&settings, answering(manifest)).await.unwrap(); + let (endpoints, schemas, _) = run(&settings, answering(manifest)).await.unwrap(); let post = endpoints.iter().find(|e| e.method == "POST").unwrap(); assert_eq!(post.body, "{\"enabled\": true}"); assert_eq!( schemas - .get("POST /flags") + .request_for("POST /flags") + .as_ref() .and_then(|schema| schema.pointer("/properties/enabled/type")) .and_then(|kind| kind.as_str()), Some("boolean") ); } + fn scratch(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("fiber-loader-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + dir + } + + /// The regression this file layout exists for. Schemas that refer to each + /// other in a ring, shared by many endpoints, used to be expanded along + /// every path through the ring and stored once per endpoint: this 20 KB + /// document made a cache of over 200 MB, and a real one made 2.5 GB that + /// every MCP call then had to parse. + #[tokio::test] + async fn a_cyclic_spec_makes_a_cache_the_size_of_the_spec() { + let spec = include_str!("../tests/fixtures/cyclic-union.json"); + let openapi = TEMPLATES + .iter() + .find(|(name, _)| *name == "OpenAPI") + .unwrap() + .1; + let (endpoints, schemas, _) = run(&config(openapi), answering(spec)).await.unwrap(); + assert_eq!(endpoints.len(), 21); + + let dir = scratch("cyclic"); + let cache = LoaderCache { + loaded_at: 1, + endpoints, + }; + write_cache(&dir, "sec-1", &cache, &schemas).unwrap(); + + let size = |name: &str| std::fs::metadata(dir.join(name)).unwrap().len(); + // The endpoint list is all a listing reads. It holds no schema, and + // each body built from one is bounded — 440 KB apiece here, before. + assert!(size("sec-1.json") < 1_000_000, "{}", size("sec-1.json")); + // Every definition once, plus one `$ref` per endpoint and direction. + assert!( + size("sec-1.schemas.json") < spec.len() as u64, + "{}", + size("sec-1.schemas.json") + ); + + let read = read_cache(&dir, "sec-1").unwrap(); + assert_eq!(read.endpoints.len(), 21); + let schemas = read_schemas(&dir, "sec-1").unwrap(); + let key = "POST /data-points/concept7/define"; + for bundled in [schemas.request_for(key), schemas.response_for(key)] { + let bundled = bundled.expect("a schema for both directions"); + assert!(bundled["$defs"]["Operand"]["oneOf"].is_array(), "{bundled}"); + assert!(serde_json::to_string(&bundled).unwrap().len() < 8_000); + } + // A body still comes out of the same schema. + let post = read.endpoints.iter().find(|e| e.key() == key).unwrap(); + assert!(post.body.contains("\"kind\": \"step0\""), "{}", post.body); + assert!(post.body.len() < 48_000, "{}", post.body.len()); + + forget_cache(&dir, "sec-1"); + assert!(read_cache(&dir, "sec-1").is_none()); + assert!(read_schemas(&dir, "sec-1").is_none()); + let _ = std::fs::remove_dir_all(dir); + } + + /// A cache written before schemas moved to their own file still lists its + /// endpoints. What it held under `schemas` is the unbounded expansion, so + /// it is not served — and not even read: the endpoint list comes first, + /// and reading stops there. + #[test] + fn a_cache_from_before_the_split_is_read_only_as_far_as_its_endpoints() { + let dir = scratch("legacy"); + std::fs::create_dir_all(&dir).unwrap(); + // Everything after the endpoint list stands in for 2.5 GB: if it were + // parsed, it would fail. + let legacy = r#"{ + "loadedAt": 7, + "endpoints": [{ "method": "POST", "path": "/define", "name": "define" }], + "schemas": { "POST /define": { "oneOf": [ this is never read"#; + std::fs::write(dir.join("sec-1.json"), legacy).unwrap(); + + let read = read_cache(&dir, "sec-1").unwrap(); + assert_eq!(read.loaded_at, 7); + assert_eq!(read.endpoints[0].key(), "POST /define"); + assert!(read_schemas(&dir, "sec-1").is_none()); + + // Missing either field is still "nothing loaded", not half a cache. + std::fs::write(dir.join("sec-1.json"), r#"{"endpoints": []}"#).unwrap(); + assert!(read_cache(&dir, "sec-1").is_none()); + let _ = std::fs::remove_dir_all(dir); + } + /// A manifest that is not OpenAPI has no `paths`, so nothing is derived and /// nothing breaks. #[tokio::test] @@ -1062,7 +1256,7 @@ mod tests { .find(|(name, _)| *name == "Array of routes") .unwrap() .1; - let (endpoints, _schemas, _, _) = run( + let (endpoints, _schemas, _) = run( &config(routes), answering(r#"{"routes":[{"verb":"post","url":"/user","handler":"createUser"}]}"#), ) diff --git a/src-tauri/src/mcp.rs b/src-tauri/src/mcp.rs index 1e5df01..6f9d820 100644 --- a/src-tauri/src/mcp.rs +++ b/src-tauri/src/mcp.rs @@ -257,6 +257,11 @@ struct CachedLoader { cache: Arc, } +struct CachedSchemas { + stamp: Option<(u128, u64)>, + schemas: Arc, +} + type SectionSnapshot = (Arc>>, Arc>); fn file_stamp(path: &std::path::Path) -> Option<(u128, u64)> { @@ -315,6 +320,10 @@ pub struct FiberMcp { sections: Arc>>, manifests: Arc>>, endpoint_caches: Arc>>, + /// Apart from `endpoint_caches` so that only `get_endpoint` ever reads a + /// schema. Everything else — listing, searching, deciding whether a call is + /// allowed — needs the endpoint list and nothing more. + schema_caches: Arc>>, request_ids: Arc, requests: Arc, loaders: Arc, @@ -333,34 +342,62 @@ impl FiberMcp { ) } + /// A section's loaded endpoints, parsed once per change to the file. + /// + /// The read happens with the lock held, so calls that arrive together wait + /// for one read rather than each starting their own. Reads are small: the + /// file holds endpoints only, and one written before schemas moved out is + /// read no further than its endpoint list. fn loader_cache_of(&self, section_id: &str) -> Arc { let path = self.loaders_dir.join(format!("{section_id}.json")); let stamp = file_stamp(&path); - { - let caches = self - .endpoint_caches - .lock() - .unwrap_or_else(|err| err.into_inner()); - if let Some(cached) = caches.get(section_id) { - if cached.stamp == stamp { - return cached.cache.clone(); - } + let mut caches = self + .endpoint_caches + .lock() + .unwrap_or_else(|err| err.into_inner()); + if let Some(cached) = caches.get(section_id) { + if cached.stamp == stamp { + return cached.cache.clone(); } } let cache = Arc::new(loader::read_cache(&self.loaders_dir, section_id).unwrap_or_default()); - self.endpoint_caches - .lock() - .unwrap_or_else(|err| err.into_inner()) - .insert( - section_id.to_string(), - CachedLoader { - stamp, - cache: cache.clone(), - }, - ); + caches.insert( + section_id.to_string(), + CachedLoader { + stamp, + cache: cache.clone(), + }, + ); cache } + /// A section's loader schemas, read the first time an endpoint's are asked + /// for and again only when the file changes. + fn loader_schemas_of(&self, section_id: &str) -> Arc { + let stamp = loader::schemas_path(&self.loaders_dir, section_id) + .as_deref() + .and_then(file_stamp); + let mut caches = self + .schema_caches + .lock() + .unwrap_or_else(|err| err.into_inner()); + if let Some(cached) = caches.get(section_id) { + if cached.stamp == stamp { + return cached.schemas.clone(); + } + } + let schemas = + Arc::new(loader::read_schemas(&self.loaders_dir, section_id).unwrap_or_default()); + caches.insert( + section_id.to_string(), + CachedSchemas { + stamp, + schemas: schemas.clone(), + }, + ); + schemas + } + fn all_sections(&self) -> Result { let stamp = sections_stamp(&self.sections_dir).map_err(|err| { McpError::internal_error(format!("could not inspect collections: {err}"), None) @@ -888,6 +925,7 @@ impl FiberMcp { None, ) })?; + let schemas = self.loader_schemas_of(§ion.id); ok_json(&serde_json::json!({ "sectionId": section.id, "key": endpoint.key(), @@ -906,8 +944,8 @@ impl FiberMcp { "form": endpoint.form, "pathParams": [], "parameters": endpoint.parameters, - "requestSchema": cache.schemas.get(&args.key), - "responseSchema": cache.response_schemas.get(&args.key), + "requestSchema": schemas.request_for(&args.key), + "responseSchema": schemas.response_for(&args.key), })) } @@ -1183,22 +1221,31 @@ impl FiberMcp { let _permit = self.loaders.acquire().await.map_err(|_| { McpError::internal_error("loader concurrency limiter closed".to_string(), None) })?; - let (endpoints, schemas, response_schemas, pages) = - loader::run(&config, self.fetcher(§ion)) - .await - .map_err(|err| McpError::internal_error(err.to_string(), None))?; + let (endpoints, schemas, pages) = loader::run(&config, self.fetcher(§ion)) + .await + .map_err(|err| McpError::internal_error(err.to_string(), None))?; let previous = self.loader_cache_of(§ion.id); let (added, removed) = loader::diff(&previous.endpoints, &endpoints); let cache = loader::LoaderCache { loaded_at: crate::history::now_millis(), endpoints: endpoints.clone(), - schemas, - response_schemas, }; - loader::write_cache(&self.loaders_dir, §ion.id, &cache).map_err(|err| { + loader::write_cache(&self.loaders_dir, §ion.id, &cache, &schemas).map_err(|err| { McpError::internal_error(format!("could not persist loader cache: {err}"), None) })?; + self.schema_caches + .lock() + .unwrap_or_else(|err| err.into_inner()) + .insert( + section.id.clone(), + CachedSchemas { + stamp: loader::schemas_path(&self.loaders_dir, §ion.id) + .as_deref() + .and_then(file_stamp), + schemas: Arc::new(schemas), + }, + ); self.endpoint_caches .lock() .unwrap_or_else(|err| err.into_inner()) @@ -1672,6 +1719,7 @@ pub async fn serve() -> Result<(), Box> { sections: Arc::new(Mutex::new(None)), manifests: Arc::new(Mutex::new(HashMap::new())), endpoint_caches: Arc::new(Mutex::new(HashMap::new())), + schema_caches: Arc::new(Mutex::new(HashMap::new())), request_ids: Arc::new(AtomicU64::new(0)), requests: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_REQUESTS)), loaders: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_LOADERS)), @@ -1775,6 +1823,7 @@ mod tests { sections: Arc::new(Mutex::new(None)), manifests: Arc::new(Mutex::new(HashMap::new())), endpoint_caches: Arc::new(Mutex::new(HashMap::new())), + schema_caches: Arc::new(Mutex::new(HashMap::new())), request_ids: Arc::new(AtomicU64::new(0)), requests: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_REQUESTS)), loaders: Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_LOADERS)), @@ -1840,9 +1889,8 @@ mod tests { &loader::LoaderCache { loaded_at: 1, endpoints, - schemas: Default::default(), - response_schemas: Default::default(), }, + &Default::default(), ) .unwrap(); } @@ -1998,9 +2046,8 @@ mod tests { body: String::new(), ..Default::default() }], - schemas: Default::default(), - response_schemas: Default::default(), }, + &Default::default(), ) .unwrap(); @@ -2012,6 +2059,78 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + /// Listing, searching and deciding whether a call is allowed need the + /// endpoint list and nothing else. Schemas are read only when an endpoint + /// is asked for — when they were one file, a large one hung every call. + #[tokio::test] + async fn only_get_endpoint_reads_schemas() { + let dir = scratch("schemas-apart"); + let mut acme = section(true, false); + acme.loader = Some(loader::LoaderConfig::default()); + store::save(&dir, &acme).unwrap(); + + let document: serde_json::Value = + serde_json::from_str(include_str!("../tests/fixtures/cyclic-union.json")).unwrap(); + let key = "POST /data-points/concept3/define"; + let root = serde_json::json!({ "$ref": "#/components/schemas/Operand" }); + let mut schemas = loader::LoaderSchemas::default(); + crate::openapi::collect_definitions(&document, &root, &mut schemas.definitions); + schemas.request.insert(key.into(), root); + loader::write_cache( + &dir.join("loaders"), + "sec-1", + &loader::LoaderCache { + loaded_at: 1, + endpoints: vec![loader::LoadedEndpoint { + method: "POST".into(), + path: "/data-points/concept3/define".into(), + ..Default::default() + }], + }, + &schemas, + ) + .unwrap(); + + let mcp = server(&dir); + let listed = mcp + .list_sections() + .await + .unwrap() + .structured_content + .unwrap(); + assert_eq!(listed["sections"][0]["endpoints"], 2); + mcp.search_endpoints(Parameters(SearchArgs { + query: "define".into(), + section_id: None, + method: None, + offset: 0, + limit: 10, + })) + .await + .unwrap(); + mcp.decide_one(&acme, "POST", "/data-points/concept3/define"); + assert!( + mcp.schema_caches.lock().unwrap().is_empty(), + "a call that needs no schema read them" + ); + + let found = mcp + .get_endpoint(Parameters(EndpointArgs { + section_id: "sec-1".into(), + key: key.into(), + })) + .await + .unwrap() + .structured_content + .unwrap(); + // Self-contained, with the ring written once under `$defs`. + assert_eq!(found["requestSchema"]["$ref"], "#/$defs/Operand"); + assert!(found["requestSchema"]["$defs"]["Step6"].is_object()); + assert!(found["responseSchema"].is_null()); + assert_eq!(mcp.schema_caches.lock().unwrap().len(), 1); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn long_bodies_are_cut_short() { let (short, cut) = truncate("hello", 10); diff --git a/src-tauri/src/openapi.rs b/src-tauri/src/openapi.rs index 55d0545..056f099 100644 --- a/src-tauri/src/openapi.rs +++ b/src-tauri/src/openapi.rs @@ -170,6 +170,13 @@ pub fn parse(text: &str) -> Result { /// what keep a skeleton finite. const MAX_DEPTH: usize = 6; +/// How many schema nodes one body may be built from. The depth limit alone +/// still lets schemas that refer to each other branch at every level — seven +/// steps that each refer to the other six made a 440 KB body within it, for +/// every endpoint that takes one. A body over this is built again one level +/// shallower, so it is cut evenly rather than wherever the walk ran out. +const MAX_NODES: usize = 1_000; + /// A JSON body to start from, built from the operation's request schema. /// /// An `example` written into the document always wins: it is what the author @@ -201,7 +208,7 @@ pub(crate) fn request_body( let (value, derived) = match given { Some(value) => (value, false), None => match media.get("schema") { - Some(schema) => (skeleton(document, schema, 0, &mut Vec::new()), true), + Some(schema) => (bounded_skeleton(document, schema), true), None => return String::new(), }, }; @@ -218,27 +225,24 @@ pub(crate) fn request_body( } } -/// The JSON Schema that governs an operation's JSON request body. -/// -/// Local references are expanded before the schema leaves Rust. The editor gets -/// this only for the endpoint being edited, and has no copy of the full OpenAPI -/// document with which to resolve `#/components/...` itself. -pub(crate) fn request_schema( - document: &serde_json::Value, - operation: &serde_json::Map, -) -> Option { - let schema = json_media(document, operation)?.get("schema")?; - Some(resolve_schema(document, schema, &mut Vec::new())) +/// The JSON Schema that governs an operation's JSON request body, as the +/// document wrote it — `$ref`s and all. See [`bundle`] for the form that +/// leaves Rust. +pub(crate) fn request_schema<'a>( + document: &'a serde_json::Value, + operation: &'a serde_json::Map, +) -> Option<&'a serde_json::Value> { + json_media(document, operation)?.get("schema") } /// JSON Schema for the successful JSON response, when the operation has one. /// /// Prefers 200, then any 2xx, then `default`. Used to lint the response the /// same way the request body is linted — after it arrives, not before. -pub(crate) fn response_schema( - document: &serde_json::Value, - operation: &serde_json::Map, -) -> Option { +pub(crate) fn response_schema<'a>( + document: &'a serde_json::Value, + operation: &'a serde_json::Map, +) -> Option<&'a serde_json::Value> { let responses = operation .get("responses") .and_then(|value| value.as_object())?; @@ -261,8 +265,7 @@ pub(crate) fn response_schema( .find(|(kind, _)| kind.contains("json")) .map(|(_, value)| value) })?; - let schema = media.get("schema")?; - Some(resolve_schema(document, schema, &mut Vec::new())) + media.get("schema") } /// Body text, kind, and form fields for an operation — JSON skeleton, or a @@ -478,38 +481,216 @@ pub(crate) fn first_tag(operation: Option<&serde_json::Map(schema: &'a serde_json::Value, named: bool, found: &mut impl FnMut(&'a str, bool)) { + match schema { + serde_json::Value::Array(items) => { + for item in items { + each_ref(item, false, found); + } + } + serde_json::Value::Object(fields) => { + for (key, value) in fields { + if named { + each_ref(value, false, found); + } else if key == "$ref" { + if let Some(reference) = value.as_str() { + found(reference, fields.len() > 1); + } + } else if !DATA_KEYWORDS.contains(&key.as_str()) { + each_ref(value, NAMED_SCHEMAS.contains(&key.as_str()), found); + } + } + } + _ => {} + } +} + +/// Every local definition a schema reaches, directly or through another +/// definition, keyed by the `$ref` that names it. Each is stored once, as the +/// document wrote it. +/// +/// This is what keeps a loader cache the size of the document rather than the +/// size of its expansion. The cache used to hold every endpoint's schema with +/// its references expanded in place, and a reference cycle only stopped where +/// it met itself on the current path. Schemas that refer to each other in a +/// ring — an operand that is an object or a function, each of which holds +/// operands — have a number of such paths that grows factorially with the size +/// of the ring, and one real collection's cache reached 2.5 GB that way. +pub(crate) fn collect_definitions( document: &serde_json::Value, schema: &serde_json::Value, - trail: &mut Vec, + into: &mut std::collections::BTreeMap, +) { + let mut queue = vec![schema]; + while let Some(next) = queue.pop() { + each_ref(next, false, &mut |reference, _| { + if into.contains_key(reference) { + return; + } + if let Some(found) = target(document, reference) { + into.insert(reference.to_string(), found.clone()); + queue.push(found); + } + }); + } +} + +/// One schema made self-contained: its references point into a `$defs` block +/// carried alongside, rather than into a document the reader does not have. +/// +/// A definition used once, by a bare `$ref`, is written in place, which is how +/// most schemas in most documents read best. Anything used more than once is +/// written once under `$defs` and referred to — so the result is never larger +/// than the schema plus the definitions it reaches, however those refer to +/// each other. A cycle always has a member that is used more than once (from +/// outside the cycle and from inside it), so nothing is written in place +/// forever. +/// +/// Ajv, which lints bodies in the editor, reads `$defs` natively. +pub(crate) fn bundle( + schema: &serde_json::Value, + definitions: &std::collections::BTreeMap, ) -> serde_json::Value { - if let Some(reference) = schema.get("$ref").and_then(|value| value.as_str()) { - if reference.starts_with("#/") && !trail.iter().any(|seen| seen == reference) { - trail.push(reference.to_string()); - let resolved = resolve(document, schema); - let expanded = resolve_schema(document, resolved, trail); - trail.pop(); - return expanded; + // How often each reachable definition is referred to, counting every + // reference in the schema and in each reachable definition once. + let mut uses: std::collections::BTreeMap<&str, (usize, bool)> = Default::default(); + let mut queue = vec![schema]; + while let Some(next) = queue.pop() { + each_ref(next, false, &mut |reference, beside| { + let Some((key, found)) = definitions.get_key_value(reference) else { + return; + }; + let entry = uses.entry(key.as_str()).or_insert((0, false)); + entry.0 += 1; + entry.1 |= beside; + if entry.0 == 1 { + queue.push(found); + } + }); + } + + // Names come from the reference's last segment, made safe to use in a + // pointer without escaping. Component names are nearly always safe + // already, so this rarely changes anything. + let mut taken: std::collections::HashSet = schema + .get("$defs") + .and_then(|defs| defs.as_object()) + .map(|defs| defs.keys().cloned().collect()) + .unwrap_or_default(); + let mut shared = std::collections::BTreeMap::new(); + for (reference, (count, beside)) in &uses { + if *count < 2 && !beside { + continue; + } + let base: String = reference + .rsplit('/') + .next() + .unwrap_or_default() + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_') { + c + } else { + '_' + } + }) + .collect(); + let base = if base.is_empty() { + "schema".into() + } else { + base + }; + let mut name = base.clone(); + let mut suffix = 1; + while !taken.insert(name.clone()) { + suffix += 1; + name = format!("{base}_{suffix}"); + } + shared.insert(*reference, name); + } + + let mut root = rewrite(schema, false, definitions, &shared); + if !shared.is_empty() { + if let serde_json::Value::Object(fields) = &mut root { + let defs = fields + .entry("$defs") + .or_insert_with(|| serde_json::Value::Object(Default::default())); + if let serde_json::Value::Object(defs) = defs { + for (reference, name) in &shared { + defs.insert( + name.clone(), + rewrite(&definitions[*reference], false, definitions, &shared), + ); + } + } } - return schema.clone(); } + root +} +/// A copy of `schema` with each reference either pointed into `$defs` or, +/// when it is used only once, replaced by what it names. +fn rewrite( + schema: &serde_json::Value, + named: bool, + definitions: &std::collections::BTreeMap, + shared: &std::collections::BTreeMap<&str, String>, +) -> serde_json::Value { match schema { serde_json::Value::Array(items) => serde_json::Value::Array( items .iter() - .map(|item| resolve_schema(document, item, trail)) - .collect(), - ), - serde_json::Value::Object(fields) => serde_json::Value::Object( - fields - .iter() - .map(|(key, value)| (key.clone(), resolve_schema(document, value, trail))) + .map(|item| rewrite(item, false, definitions, shared)) .collect(), ), + serde_json::Value::Object(fields) => { + if !named { + if let Some(reference) = fields.get("$ref").and_then(|value| value.as_str()) { + if !shared.contains_key(reference) { + if let Some(found) = definitions.get(reference) { + return rewrite(found, false, definitions, shared); + } + } + } + } + serde_json::Value::Object( + fields + .iter() + .map(|(key, value)| { + let value = if named { + rewrite(value, false, definitions, shared) + } else if key == "$ref" { + match value.as_str().and_then(|reference| shared.get(reference)) { + Some(name) => serde_json::Value::String(format!("#/$defs/{name}")), + None => value.clone(), + } + } else if DATA_KEYWORDS.contains(&key.as_str()) { + value.clone() + } else { + let named = NAMED_SCHEMAS.contains(&key.as_str()); + rewrite(value, named, definitions, shared) + }; + (key.clone(), value) + }) + .collect(), + ) + } _ => schema.clone(), } } @@ -594,23 +775,45 @@ fn resolve<'a>( document: &'a serde_json::Value, value: &'a serde_json::Value, ) -> &'a serde_json::Value { - let Some(pointer) = value.get("$ref").and_then(|it| it.as_str()) else { - return value; - }; - let Some(rest) = pointer.strip_prefix("#/") else { - return value; - }; + value + .get("$ref") + .and_then(|it| it.as_str()) + .and_then(|pointer| target(document, pointer)) + .unwrap_or(value) +} +/// What a local `$ref` points at, if it points anywhere in this document. +fn target<'a>(document: &'a serde_json::Value, pointer: &str) -> Option<&'a serde_json::Value> { + let rest = pointer.strip_prefix("#/")?; let mut current = document; for segment in rest.split('/') { // JSON Pointer escapes, in the order the spec requires: `~1` first. let key = segment.replace("~1", "/").replace("~0", "~"); - match current.get(&key) { - Some(next) => current = next, - None => return value, + current = current.get(&key)?; + } + Some(current) +} + +/// The deepest skeleton that fits in [`MAX_NODES`]. +fn bounded_skeleton(document: &serde_json::Value, schema: &serde_json::Value) -> serde_json::Value { + for max_depth in (0..=MAX_DEPTH).rev() { + let mut walk = Walk { + max_depth, + room: MAX_NODES, + }; + let value = skeleton(document, schema, 0, &mut walk, &mut Vec::new()); + if walk.room > 0 { + return value; } } - current + serde_json::Value::Null +} + +/// Limits on one skeleton walk. +struct Walk { + max_depth: usize, + /// Nodes still allowed. Reaching 0 means the walk did not fit. + room: usize, } /// One placeholder value per field, following `$ref`s but never in a circle. @@ -618,11 +821,13 @@ fn skeleton( document: &serde_json::Value, schema: &serde_json::Value, depth: usize, + walk: &mut Walk, trail: &mut Vec, ) -> serde_json::Value { - if depth > MAX_DEPTH { + if depth > walk.max_depth || walk.room == 0 { return serde_json::Value::Null; } + walk.room -= 1; // A `$ref` already on the trail is a cycle — stop rather than follow it. if let Some(pointer) = schema.get("$ref").and_then(|it| it.as_str()) { @@ -630,7 +835,7 @@ fn skeleton( return serde_json::Value::Null; } trail.push(pointer.to_string()); - let value = skeleton(document, resolve(document, schema), depth, trail); + let value = skeleton(document, resolve(document, schema), depth, walk, trail); trail.pop(); return value; } @@ -666,7 +871,8 @@ fn skeleton( if let Some(parts) = object.get("allOf").and_then(|it| it.as_array()) { let mut merged = serde_json::Map::new(); for part in parts { - if let serde_json::Value::Object(fields) = skeleton(document, part, depth, trail) { + if let serde_json::Value::Object(fields) = skeleton(document, part, depth, walk, trail) + { merged.extend(fields); } } @@ -683,7 +889,7 @@ fn skeleton( .find(|branch| !only_null(branch)) .or_else(|| branches.first()); if let Some(branch) = pick { - return skeleton(document, branch, depth, trail); + return skeleton(document, branch, depth, walk, trail); } } } @@ -695,7 +901,10 @@ fn skeleton( let mut fields = serde_json::Map::new(); if let Some(properties) = object.get("properties").and_then(|it| it.as_object()) { for (name, property) in properties { - fields.insert(name.clone(), skeleton(document, property, depth + 1, trail)); + fields.insert( + name.clone(), + skeleton(document, property, depth + 1, walk, trail), + ); } } return serde_json::Value::Object(fields); @@ -704,7 +913,7 @@ fn skeleton( if kind == Some("array") { let item = object .get("items") - .map(|items| skeleton(document, items, depth + 1, trail)) + .map(|items| skeleton(document, items, depth + 1, walk, trail)) .unwrap_or(serde_json::Value::Null); return serde_json::Value::Array(if item.is_null() { vec![] } else { vec![item] }); } @@ -1325,6 +1534,151 @@ paths: assert!(body.contains("\"replies\": []"), "{body}"); } + /// Every `$ref` in a bundle points inside it, so it can be validated + /// without the document it came from. + fn assert_self_contained(bundled: &serde_json::Value) { + each_ref(bundled, false, &mut |reference, _| { + assert!( + reference.starts_with("#/$defs/") && target(bundled, reference).is_some(), + "`{reference}` does not resolve inside {bundled}" + ); + }); + } + + fn definitions_of( + document: &serde_json::Value, + schema: &serde_json::Value, + ) -> std::collections::BTreeMap { + let mut definitions = Default::default(); + collect_definitions(document, schema, &mut definitions); + definitions + } + + /// The smallest shape of the ring that blew a loader cache up to 2.5 GB: + /// a union whose branches each hold the union again. + #[test] + fn a_cyclic_union_is_bundled_once() { + let document = json!({ + "components": { "schemas": { + "A": { "oneOf": [ + { "$ref": "#/components/schemas/B" }, + { "$ref": "#/components/schemas/C" } + ] }, + "B": { "type": "object", "properties": { "a": { "$ref": "#/components/schemas/A" } } }, + "C": { "type": "array", "items": { "$ref": "#/components/schemas/A" } }, + "Unused": { "type": "string" } + } } + }); + let root = json!({ "$ref": "#/components/schemas/A" }); + + let definitions = definitions_of(&document, &root); + // What the schema reaches, and nothing it doesn't. + assert_eq!( + definitions.keys().collect::>(), + vec![ + "#/components/schemas/A", + "#/components/schemas/B", + "#/components/schemas/C" + ] + ); + + let bundled = bundle(&root, &definitions); + assert_self_contained(&bundled); + // A is where the ring is entered and re-entered, so it is the one + // written under `$defs`; B and C are used once each and read inline. + assert_eq!( + bundled, + json!({ + "$ref": "#/$defs/A", + "$defs": { "A": { "oneOf": [ + { "type": "object", "properties": { "a": { "$ref": "#/$defs/A" } } }, + { "type": "array", "items": { "$ref": "#/$defs/A" } } + ] } } + }) + ); + } + + /// Every step refers to every other: the number of paths through that + /// grows factorially, and expanding along each one is what the cache used + /// to do. Bundling writes each definition once, however they connect. + #[test] + fn a_densely_connected_ring_stays_the_size_of_the_document() { + let text = include_str!("../tests/fixtures/cyclic-union.json"); + let document: serde_json::Value = serde_json::from_str(text).unwrap(); + let root = json!({ "$ref": "#/components/schemas/Operand" }); + + let bundled = bundle(&root, &definitions_of(&document, &root)); + assert_self_contained(&bundled); + let size = serde_json::to_string(&bundled).unwrap().len(); + // The whole document is under 20 KB. The old expansion of this one + // schema was 5.3 MB. + assert!(size < 8_000, "{size} bytes"); + // Every step is shared, so each is written exactly once. + let defs = bundled["$defs"].as_object().unwrap(); + assert!((0..7).all(|step| defs.contains_key(&format!("Step{step}")))); + } + + /// A definition used once reads in place, as it did before bundling + /// existed — `$defs` is only for what would otherwise repeat. + #[test] + fn a_definition_used_once_is_written_in_place() { + let document = json!({ + "components": { "schemas": { + "NewUser": { + "type": "object", + "properties": { + "address": { "$ref": "#/components/schemas/Address" }, + "example": { "$ref": "#/components/schemas/Address2" } + }, + // Data, not schema: nothing in here is a reference. + "example": { "$ref": "#/components/schemas/Missing" } + }, + "Address": { "type": "object", "properties": { "city": { "type": "string" } } }, + "Address2": { "type": "string" } + } } + }); + let root = json!({ "$ref": "#/components/schemas/NewUser" }); + + let bundled = bundle(&root, &definitions_of(&document, &root)); + assert!(bundled.get("$defs").is_none(), "{bundled}"); + assert_eq!( + bundled.pointer("/properties/address/properties/city/type"), + Some(&json!("string")) + ); + // A property that happens to be called `example` is still a schema. + assert_eq!( + bundled.pointer("/properties/example/type"), + Some(&json!("string")) + ); + assert_eq!( + bundled.pointer("/example"), + Some(&json!({ "$ref": "#/components/schemas/Missing" })) + ); + } + + /// A `$ref` with a description beside it cannot be replaced by its target + /// without dropping the description, so it stays a reference. + #[test] + fn a_reference_with_siblings_stays_a_reference() { + let document = json!({ + "components": { "schemas": { "Id": { "type": "string" } } } + }); + let root = json!({ + "type": "object", + "properties": { + "id": { "$ref": "#/components/schemas/Id", "description": "Who" } + } + }); + + let bundled = bundle(&root, &definitions_of(&document, &root)); + assert_self_contained(&bundled); + assert_eq!( + bundled.pointer("/properties/id"), + Some(&json!({ "$ref": "#/$defs/Id", "description": "Who" })) + ); + assert_eq!(bundled.pointer("/$defs/Id/type"), Some(&json!("string"))); + } + /// Swagger 2 has no `requestBody` — the schema rides on a parameter. #[test] fn reads_a_swagger_2_body_parameter() { diff --git a/src-tauri/tests/fixtures/cyclic-union.json b/src-tauri/tests/fixtures/cyclic-union.json new file mode 100644 index 0000000..cb021ca --- /dev/null +++ b/src-tauri/tests/fixtures/cyclic-union.json @@ -0,0 +1,822 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Cyclic union", + "version": "1" + }, + "paths": { + "/data-points": { + "get": { + "operationId": "listDataPoints" + } + }, + "/data-points/concept0/define": { + "post": { + "operationId": "defineConcept0", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept1/define": { + "post": { + "operationId": "defineConcept1", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept2/define": { + "post": { + "operationId": "defineConcept2", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept3/define": { + "post": { + "operationId": "defineConcept3", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept4/define": { + "post": { + "operationId": "defineConcept4", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept5/define": { + "post": { + "operationId": "defineConcept5", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept6/define": { + "post": { + "operationId": "defineConcept6", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept7/define": { + "post": { + "operationId": "defineConcept7", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept8/define": { + "post": { + "operationId": "defineConcept8", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept9/define": { + "post": { + "operationId": "defineConcept9", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept10/define": { + "post": { + "operationId": "defineConcept10", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept11/define": { + "post": { + "operationId": "defineConcept11", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept12/define": { + "post": { + "operationId": "defineConcept12", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept13/define": { + "post": { + "operationId": "defineConcept13", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept14/define": { + "post": { + "operationId": "defineConcept14", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept15/define": { + "post": { + "operationId": "defineConcept15", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept16/define": { + "post": { + "operationId": "defineConcept16", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept17/define": { + "post": { + "operationId": "defineConcept17", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept18/define": { + "post": { + "operationId": "defineConcept18", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + }, + "/data-points/concept19/define": { + "post": { + "operationId": "defineConcept19", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + }, + "responses": { + "200": { + "description": "ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Operand" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Operand": { + "oneOf": [ + { + "$ref": "#/components/schemas/Step0" + }, + { + "$ref": "#/components/schemas/Step1" + }, + { + "$ref": "#/components/schemas/Step2" + }, + { + "$ref": "#/components/schemas/Step3" + }, + { + "$ref": "#/components/schemas/Step4" + }, + { + "$ref": "#/components/schemas/Step5" + }, + { + "$ref": "#/components/schemas/Step6" + }, + { + "$ref": "#/components/schemas/Literal" + } + ] + }, + "Literal": { + "type": "object", + "properties": { + "value": { + "type": [ + "string", + "number", + "null" + ] + } + } + }, + "Step0": { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "step0" + }, + "input": { + "$ref": "#/components/schemas/Operand" + }, + "then1": { + "$ref": "#/components/schemas/Step1" + }, + "then2": { + "$ref": "#/components/schemas/Step2" + }, + "then3": { + "$ref": "#/components/schemas/Step3" + }, + "then4": { + "$ref": "#/components/schemas/Step4" + }, + "then5": { + "$ref": "#/components/schemas/Step5" + }, + "then6": { + "$ref": "#/components/schemas/Step6" + } + } + }, + "Step1": { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "step1" + }, + "input": { + "$ref": "#/components/schemas/Operand" + }, + "then0": { + "$ref": "#/components/schemas/Step0" + }, + "then2": { + "$ref": "#/components/schemas/Step2" + }, + "then3": { + "$ref": "#/components/schemas/Step3" + }, + "then4": { + "$ref": "#/components/schemas/Step4" + }, + "then5": { + "$ref": "#/components/schemas/Step5" + }, + "then6": { + "$ref": "#/components/schemas/Step6" + } + } + }, + "Step2": { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "step2" + }, + "input": { + "$ref": "#/components/schemas/Operand" + }, + "then0": { + "$ref": "#/components/schemas/Step0" + }, + "then1": { + "$ref": "#/components/schemas/Step1" + }, + "then3": { + "$ref": "#/components/schemas/Step3" + }, + "then4": { + "$ref": "#/components/schemas/Step4" + }, + "then5": { + "$ref": "#/components/schemas/Step5" + }, + "then6": { + "$ref": "#/components/schemas/Step6" + } + } + }, + "Step3": { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "step3" + }, + "input": { + "$ref": "#/components/schemas/Operand" + }, + "then0": { + "$ref": "#/components/schemas/Step0" + }, + "then1": { + "$ref": "#/components/schemas/Step1" + }, + "then2": { + "$ref": "#/components/schemas/Step2" + }, + "then4": { + "$ref": "#/components/schemas/Step4" + }, + "then5": { + "$ref": "#/components/schemas/Step5" + }, + "then6": { + "$ref": "#/components/schemas/Step6" + } + } + }, + "Step4": { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "step4" + }, + "input": { + "$ref": "#/components/schemas/Operand" + }, + "then0": { + "$ref": "#/components/schemas/Step0" + }, + "then1": { + "$ref": "#/components/schemas/Step1" + }, + "then2": { + "$ref": "#/components/schemas/Step2" + }, + "then3": { + "$ref": "#/components/schemas/Step3" + }, + "then5": { + "$ref": "#/components/schemas/Step5" + }, + "then6": { + "$ref": "#/components/schemas/Step6" + } + } + }, + "Step5": { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "step5" + }, + "input": { + "$ref": "#/components/schemas/Operand" + }, + "then0": { + "$ref": "#/components/schemas/Step0" + }, + "then1": { + "$ref": "#/components/schemas/Step1" + }, + "then2": { + "$ref": "#/components/schemas/Step2" + }, + "then3": { + "$ref": "#/components/schemas/Step3" + }, + "then4": { + "$ref": "#/components/schemas/Step4" + }, + "then6": { + "$ref": "#/components/schemas/Step6" + } + } + }, + "Step6": { + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "step6" + }, + "input": { + "$ref": "#/components/schemas/Operand" + }, + "then0": { + "$ref": "#/components/schemas/Step0" + }, + "then1": { + "$ref": "#/components/schemas/Step1" + }, + "then2": { + "$ref": "#/components/schemas/Step2" + }, + "then3": { + "$ref": "#/components/schemas/Step3" + }, + "then4": { + "$ref": "#/components/schemas/Step4" + }, + "then5": { + "$ref": "#/components/schemas/Step5" + } + } + } + } + } +} diff --git a/tests/e2e/json-schema.spec.ts b/tests/e2e/json-schema.spec.ts index 11aecd7..9f34317 100644 --- a/tests/e2e/json-schema.spec.ts +++ b/tests/e2e/json-schema.spec.ts @@ -107,6 +107,31 @@ test.describe('validateJsonBody', () => { ]); }); + /** + * How a loader schema arrives when its definitions refer to each other: + * the ring written once under `$defs`, and the root a `$ref` into it. + */ + test('a recursive schema bundled under $defs is followed', () => { + const schema = { + $ref: '#/$defs/Operand', + $defs: { + Operand: { + oneOf: [ + { + type: 'object', + required: ['input'], + properties: { input: { $ref: '#/$defs/Operand' } }, + additionalProperties: false + }, + { type: 'string' } + ] + } + } + }; + expect(validateJsonBody(schema, '{ "input": { "input": "leaf" } }')).toEqual([]); + expect(validateJsonBody(schema, '{ "input": { "input": 1 } }')).not.toEqual([]); + }); + /** Keywords the subset never covered at all. */ test('constraints beyond the old subset are enforced', () => { expect(validateJsonBody({ type: 'integer', minimum: 1 }, '0')).toEqual([