diff --git a/CHANGELOG.md b/CHANGELOG.md index 47e1a6b41..e949c4f98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **NeMo Relay native plugin** — a dynamically loaded integration that loads + Switchyard's standard TOML deployment and executes its named routes in + process. Managed calls require NeMo Relay 0.8.0 or newer; unknown models use + Relay's continuation unchanged. + +- **NeMo Relay routing marks** — routing-model usage, measured routing + overhead, and selected-model decisions are emitted as ATOF marks. The final + serving call remains represented only by Relay's outer LLM lifecycle event to + prevent double-counting. + - **Advisor-gate routing** — new `advisor` route type pairing the serving executor with a stronger judge-only advisor that reviews terminal turns: APPROVE releases the buffered turn, REDO discards it and feeds the advisor's diff --git a/Cargo.lock b/Cargo.lock index fde36135f..6b3e33eef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,9 +18,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -106,6 +106,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -130,13 +142,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -153,9 +165,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ "aws-lc-sys", "zeroize", @@ -163,9 +175,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", @@ -274,6 +286,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "borrow-or-share" @@ -301,9 +316,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ "find-msvc-tools", "jobserver", @@ -334,11 +349,21 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "clap" -version = "4.6.2" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -346,9 +371,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -358,14 +383,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -399,6 +424,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -424,6 +458,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "data-encoding" version = "2.11.1" @@ -456,13 +496,13 @@ checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -473,9 +513,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "email_address" @@ -502,11 +542,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fancy-regex" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +checksum = "476de73bddf2ef8490aa4ee8f1cf40b430bf1d56c48c22080e5186952cd580e6" dependencies = [ "bit-set", "regex-automata", @@ -521,9 +581,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "fluent-uri" @@ -585,9 +645,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -600,9 +660,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -610,15 +670,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -627,38 +687,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -714,9 +774,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228" dependencies = [ "atomic-waker", "bytes", @@ -756,9 +816,9 @@ checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -776,9 +836,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -807,9 +867,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -867,9 +927,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -881,9 +941,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -894,9 +954,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -908,16 +968,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -928,15 +989,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -980,9 +1041,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "is_terminal_polyfill" @@ -1017,7 +1078,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -1066,9 +1127,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -1077,9 +1138,9 @@ dependencies = [ [[package]] name = "jsonptr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85019623956752c8dd1f04a7b05d066187e0c9b217454246d8397d2a4893cc83" +checksum = "72c22d99362c7aac7d77efab6b49c2181674b165d7e6e7c755b3432ded64bf51" dependencies = [ "serde", "serde_json", @@ -1087,9 +1148,9 @@ dependencies = [ [[package]] name = "jsonschema" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257549c093d8f3d043337ba0d507e8eeace2447dfae3f0ee37eb0f57d3df7655" +checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" dependencies = [ "ahash", "bytecount", @@ -1116,18 +1177,18 @@ dependencies = [ [[package]] name = "jsonschema-regex" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474790e948498099d61ec85c10dc72d459d61298b7975eda7fb163af1934b134" +checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" dependencies = [ "regex-syntax", ] [[package]] name = "jsonschema-value" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8212315eb8e0bc44959d1af653cd5ec515771a3cdca966cfc0a10999bff144b9" +checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" dependencies = [ "ahash", "bytecount", @@ -1145,9 +1206,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "linux-raw-sys" @@ -1157,9 +1218,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -1172,9 +1233,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru-slab" @@ -1226,6 +1287,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nemo-relay-plugin" +version = "0.8.0" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?rev=ca08901629e6058c2d5d65cc7708ec5264073d7b#ca08901629e6058c2d5d65cc7708ec5264073d7b" +dependencies = [ + "futures", + "nemo-relay-types", + "serde", + "serde_json", + "tokio", + "tokio-util", +] + +[[package]] +name = "nemo-relay-types" +version = "0.8.0" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?rev=ca08901629e6058c2d5d65cc7708ec5264073d7b#ca08901629e6058c2d5d65cc7708ec5264073d7b" +dependencies = [ + "bitflags", + "chrono", + "serde", + "serde_json", + "typed-builder", + "uuid", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1276,9 +1363,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -1351,7 +1438,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -1381,7 +1468,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1421,7 +1508,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -1431,6 +1518,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1468,21 +1561,21 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -1508,9 +1601,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -1527,7 +1620,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1669,7 +1762,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -1677,9 +1770,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -1692,7 +1785,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -1714,9 +1807,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1799,18 +1892,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", @@ -1819,9 +1912,9 @@ dependencies = [ [[package]] name = "referencing" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf1a74e036be2546c0c81cdfad6b2def6a25bf31c4e34a468037058d3bd05c6" +checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" dependencies = [ "ahash", "fluent-uri", @@ -1848,9 +1941,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1948,9 +2041,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "once_cell", @@ -1974,9 +2067,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -2011,9 +2104,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -2088,9 +2181,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2098,29 +2191,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", @@ -2281,7 +2374,7 @@ dependencies = [ "serde", "serde_json", "switchyard-protocol", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -2306,7 +2399,7 @@ dependencies = [ "switchyard-libsy", "switchyard-protocol", "switchyard-translation", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "tracing-opentelemetry", @@ -2314,6 +2407,24 @@ dependencies = [ "wiremock", ] +[[package]] +name = "switchyard-nemo-relay-plugin" +version = "0.2.0" +dependencies = [ + "async-channel", + "async-trait", + "futures-util", + "http", + "nemo-relay-plugin", + "serde", + "serde_json", + "switchyard-libsy", + "switchyard-llm-client", + "switchyard-protocol", + "switchyard-translation", + "tokio", +] + [[package]] name = "switchyard-protocol" version = "0.2.0" @@ -2323,7 +2434,7 @@ dependencies = [ "http", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2355,7 +2466,7 @@ dependencies = [ "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "toml", "tracing", @@ -2403,7 +2514,7 @@ dependencies = [ "async-trait", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -2434,7 +2545,7 @@ dependencies = [ "serde", "serde_json", "switchyard-protocol", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2509,11 +2620,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -2529,13 +2640,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2549,9 +2660,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2574,9 +2685,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.4" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2591,13 +2702,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2612,9 +2723,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -2623,22 +2734,24 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", + "libc", "pin-project-lite", "tokio", ] [[package]] name = "toml" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", @@ -2660,9 +2773,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -2803,6 +2916,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -2845,6 +2978,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "uuid-simd" version = "0.8.0" @@ -2909,9 +3054,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -2922,9 +3067,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -2932,9 +3077,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2942,9 +3087,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -2955,9 +3100,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -2977,9 +3122,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -3138,9 +3283,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yansi" @@ -3173,18 +3318,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -3220,9 +3365,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -3231,9 +3376,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -3242,13 +3387,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c1b8d1a94..77d78f5bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/libsy-llm-client", "crates/switchyard-py", "crates/protocol", + "crates/switchyard-nemo-relay-plugin", "crates/switchyard-runner", "crates/switchyard-server", "crates/switchyard-skill-distillation", @@ -24,6 +25,7 @@ repository = "https://github.com/NVIDIA-NeMo/Switchyard" rust-version = "1.96.1" [workspace.dependencies] +async-channel = "2" async-stream = "0.3" async-trait = "0.1" futures = "0.3" @@ -32,6 +34,7 @@ http = "1" httpdate = "1" jsonschema = { version = "0.49.4", default-features = false } jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] } +nemo-relay-plugin = { git = "https://github.com/NVIDIA/NeMo-Relay.git", rev = "ca08901629e6058c2d5d65cc7708ec5264073d7b" } parking_lot = "0.12" rand = "0.10" regex = "1" diff --git a/README.md b/README.md index 142f80ee7..bc83ad51e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ algorithm you write yourself. - **Protocol Translation**: convert between OpenAI Chat, Anthropic Messages, and OpenAI Responses formats - **Multi-Backend Routing**: random routing, LLM-as-classifier routing, signal-driven stage-router, or your own algorithm - **Operational Metrics**: Prometheus metrics cover requests, errors, latency, tokens, and routing overhead +- **NeMo Relay Plugin**: run random, classifier, escalation, or stage routing in Relay while Switchyard owns provider HTTP dispatch ## Maturity @@ -123,6 +124,7 @@ configured LLM client selects one upstream format. - **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application - **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types - **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation +- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: install Switchyard as a native NeMo Relay plugin ## Community diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml new file mode 100644 index 000000000..f6a4b1d64 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-nemo-relay-plugin" +version.workspace = true +description = "Switchyard-owned HTTP routing plugin for NeMo Relay" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +futures-util.workspace = true +http.workspace = true +nemo-relay-plugin.workspace = true +serde.workspace = true +serde_json.workspace = true +switchyard-llm-client.workspace = true +switchyard-protocol.workspace = true +switchyard-runner.workspace = true +switchyard-translation.workspace = true +tokio.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md new file mode 100644 index 000000000..8ce7093f1 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -0,0 +1,56 @@ +# Switchyard NeMo Relay Plugin + +`switchyard-nemo-relay-plugin` is a native NeMo Relay dynamic plugin. It loads +a standard Switchyard TOML deployment and executes its configured routes in +Relay through `switchyard-runner`. + +The plugin does not define a second routing or target configuration language. +`switchyard-server` and Relay therefore use the same targets, client pooling, +algorithm construction, retry policy, and route validation. + +## Install + +Build the platform bundle with the package script, then configure Relay to load +the generated `relay-plugin.toml` manifest. The plugin requires NeMo Relay +`>=0.8.0,<1.0`. + +## Configure Relay + +Point the dynamic plugin configuration at an existing Switchyard deployment: + +```toml +[[plugins.dynamic]] +plugin_id = "nvidia.switchyard" + +[plugins.dynamic.config] +priority = 0 +deployment_path = "/etc/switchyard/routes.toml" +``` + +`deployment_path` is a Switchyard version-1 TOML deployment, accepted by both +`switchyard-server` and `switchyard-runner`. See the +[server configuration guide](../switchyard-server/CONFIGURATION.md) for the +deployment schema and routing algorithms. + +## Request handling + +For OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages calls, +the plugin decodes the Relay request and checks the requested model against the +deployment's route IDs. + +- A configured route is executed by `switchyard-runner`. +- An unknown model calls Relay's continuation unchanged. +- The returned provider response is encoded back into the caller's wire format. +- Streaming responses are returned as unpolled translated streams; Relay owns + cancellation and the outer serving-call lifecycle. + +The plugin emits a routing request mark, routing-model usage marks, measured +routing-overhead marks, and a selected-model decision mark. Answer-call usage +continues to belong to Relay's outer LLM lifecycle. + +## Failure policy + +`switchyard-llm-client` owns provider retry and route-candidate fallback +behavior. The plugin does not maintain a separate trusted-default target or +rerun routing after an execution failure. Failures outside the shared runner, +including response translation failures, are returned to Relay. diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json new file mode 100644 index 000000000..474f212f7 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Switchyard NeMo Relay Plugin", + "description": "Runs a Switchyard deployment through its shared in-process runner.", + "type": "object", + "additionalProperties": false, + "required": ["deployment_path"], + "properties": { + "priority": { + "type": "integer", + "default": 0, + "description": "NeMo Relay execution-intercept priority." + }, + "deployment_path": { + "type": "string", + "minLength": 1, + "description": "Path to a Switchyard version-1 TOML deployment shared with switchyard-server." + } + } +} diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml new file mode 100644 index 000000000..ba5992bd6 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +manifest_version = 1 + +[plugin] +id = "nvidia.switchyard" +kind = "rust_dynamic" + +[compat] +relay = ">=0.8.0,<1.0" +native_api = "1" + +[defaults] +enabled = false + +[capabilities] +items = ["plugin_native", "config_schema"] + +[config_schema] +path = "config.schema.json" + +[source] +artifact = "" + +[integrity] +sha256 = "sha256:" + +[load] +library = "" +symbol = "nemo_relay_register_plugin" diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py new file mode 100644 index 000000000..c5ea9e946 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Materialize the minimal Relay plugin bundle from a compiled cdylib.""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import tarfile +import zipfile +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = CRATE_ROOT.parents[1] +PACKAGE_NAME = "switchyard-nemo-relay-plugin" + + +def digest(path: Path) -> str: + """Return the lowercase SHA-256 digest for a file.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def archive_bundle(bundle: Path, archive: Path) -> None: + """Archive a materialized bundle under the stable package directory name.""" + archive.parent.mkdir(parents=True, exist_ok=True) + if archive.exists(): + raise ValueError(f"bundle archive already exists: {archive}") + + if archive.name.endswith(".tar.gz"): + with tarfile.open(archive, "w:gz") as stream: + stream.add(bundle, arcname=PACKAGE_NAME) + return + + if archive.suffix == ".zip": + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as stream: + for path in sorted(bundle.rglob("*")): + if path.is_file(): + stream.write(path, Path(PACKAGE_NAME) / path.relative_to(bundle)) + return + + raise ValueError("bundle archive must end in .tar.gz or .zip") + + +def main() -> None: + """Materialize a Relay-loadable plugin bundle in an empty directory.""" + parser = argparse.ArgumentParser() + parser.add_argument("--library", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--archive", type=Path) + args = parser.parse_args() + + library = args.library.resolve() + if not library.is_file(): + parser.error(f"compiled plugin library does not exist: {library}") + + manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8") + placeholders = ("", "") + missing = [placeholder for placeholder in placeholders if placeholder not in manifest] + if missing: + parser.error(f"plugin manifest is missing placeholders: {', '.join(missing)}") + + output = args.output.resolve() + if output.exists() and not output.is_dir(): + parser.error(f"bundle output exists and is not a directory: {output}") + if output.is_dir() and any(output.iterdir()): + parser.error(f"bundle output directory must be empty: {output}") + output.mkdir(parents=True, exist_ok=True) + + artifact = output / library.name + shutil.copy2(library, artifact) + shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json") + for filename in ("LICENSE", "NOTICE"): + shutil.copy2(REPOSITORY_ROOT / filename, output / filename) + + artifact_digest = digest(artifact) + manifest = manifest.replace("", artifact.name) + manifest = manifest.replace("", artifact_digest) + (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") + + if args.archive is None: + print(output) + return + + archive = args.archive.resolve() + try: + archive_bundle(output, archive) + except ValueError as error: + parser.error(str(error)) + print(archive) + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs new file mode 100644 index 000000000..9b235a62d --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; + +use serde::Deserialize; +use switchyard_protocol::WireFormat; +use switchyard_runner::Runner; + +pub(crate) fn protocol_from_call(name: &str) -> Option { + match name { + "openai.chat_completions" => Some(WireFormat::OpenAiChat), + "openai.responses" => Some(WireFormat::OpenAiResponses), + "anthropic.messages" => Some(WireFormat::AnthropicMessages), + _ => None, + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SwitchyardConfig { + #[serde(default)] + pub(crate) priority: i32, + pub(crate) deployment_path: PathBuf, +} + +impl SwitchyardConfig { + pub(crate) fn load_runner(&self) -> Result { + Runner::load(&self.deployment_path).map_err(|error| error.to_string()) + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs new file mode 100644 index 000000000..f115923b3 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod config; +mod runtime; +mod translation; + +use std::sync::Arc; + +use nemo_relay_plugin::{ + ConfigDiagnostic, DiagnosticLevel, Json, LlmJsonAsyncStream, NativePlugin, PluginContext, + PluginRuntime, +}; +use serde_json::Map; + +use crate::config::{SwitchyardConfig, protocol_from_call}; +use crate::runtime::{RoutingMark, SwitchyardRuntime}; + +#[derive(Default)] +struct SwitchyardPlugin; + +impl NativePlugin for SwitchyardPlugin { + fn plugin_kind(&self) -> &str { + "nvidia.switchyard" + } + + fn allows_multiple_components(&self) -> bool { + false + } + + fn validate(&self, plugin_config: &Map) -> Vec { + match parse_config(plugin_config).and_then(SwitchyardRuntime::new) { + Ok(_) => Vec::new(), + Err(message) => vec![ConfigDiagnostic { + level: DiagnosticLevel::Error, + code: "switchyard.invalid_config".into(), + component: Some("nvidia.switchyard".into()), + field: Some("config".into()), + message, + }], + } + } + + fn register( + &mut self, + plugin_config: &Map, + ctx: &mut PluginContext<'_>, + ) -> nemo_relay_plugin::Result<()> { + let config = parse_config(plugin_config)?; + let priority = config.priority; + let runtime = Arc::new(SwitchyardRuntime::new(config)?); + let plugin_runtime = ctx.runtime(); + register_buffered(ctx, priority, Arc::clone(&runtime), plugin_runtime.clone())?; + register_stream(ctx, priority, runtime, plugin_runtime)?; + Ok(()) + } +} + +fn register_buffered( + ctx: &mut PluginContext<'_>, + priority: i32, + runtime: Arc, + plugin_runtime: PluginRuntime, +) -> Result<(), String> { + ctx.register_llm_execution_intercept( + "switchyard.runner.buffered", + priority, + move |name, request, next| { + let runtime = Arc::clone(&runtime); + let plugin_runtime = plugin_runtime.clone(); + async move { + let Some(inbound) = protocol_from_call(&name) else { + return next.call(request).await; + }; + let decoded = runtime.decode_request(inbound, &request, false)?; + if !runtime.manages(&decoded) { + return next.call(request).await; + } + let execution = runtime.execute_buffered(inbound, decoded).await; + emit_marks(&plugin_runtime, execution.marks); + execution.result + } + }, + ) +} + +fn register_stream( + ctx: &mut PluginContext<'_>, + priority: i32, + runtime: Arc, + plugin_runtime: PluginRuntime, +) -> Result<(), String> { + ctx.register_llm_stream_execution_intercept( + "switchyard.runner.streaming", + priority, + move |name, request, next| { + let runtime = Arc::clone(&runtime); + let plugin_runtime = plugin_runtime.clone(); + async move { + let Some(inbound) = protocol_from_call(&name) else { + return next.call(request).await; + }; + let decoded = runtime.decode_request(inbound, &request, true)?; + if !runtime.manages(&decoded) { + return next.call(request).await; + } + let execution = runtime.execute_stream(inbound, decoded).await; + emit_marks(&plugin_runtime, execution.marks); + execution + .result + .map(|stream| Box::pin(stream) as LlmJsonAsyncStream) + } + }, + ) +} + +fn parse_config(plugin_config: &Map) -> Result { + serde_json::from_value(Json::Object(plugin_config.clone())) + .map_err(|error| format!("invalid Switchyard configuration: {error}")) +} + +fn emit_marks(runtime: &PluginRuntime, marks: Vec) { + for mark in marks { + if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { + eprintln!("Switchyard could not emit routing mark {:?}: {error}", mark.name); + } + } +} + +nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default); + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn plugin_configuration_requires_a_deployment_path() { + let config = json!({"priority": 0}); + let error = parse_config(config.as_object().unwrap()).unwrap_err(); + assert!(error.contains("deployment_path")); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs new file mode 100644 index 000000000..06b507fe9 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -0,0 +1,343 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use futures_util::{Stream, StreamExt}; +use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; +use serde_json::{Map, json}; +use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; +use switchyard_protocol::{LlmResponse, Metadata, Request, Response, WireFormat}; +use switchyard_runner::{Route, Runner}; +use switchyard_translation::{TranslationEngine, encode_stream}; + +use crate::config::SwitchyardConfig; +use crate::translation; + +#[derive(Debug)] +pub(crate) struct RoutingMark { + pub(crate) name: String, + pub(crate) data: Json, + pub(crate) metadata: Json, +} + +pub(crate) type ReturnedEventStream = Pin> + Send>>; + +pub(crate) struct Execution { + pub(crate) result: Result, + pub(crate) marks: Vec, +} + +pub(crate) struct SwitchyardRuntime { + runner: Runner, + translation: TranslationEngine, +} + +impl SwitchyardRuntime { + pub(crate) fn new(config: SwitchyardConfig) -> Result { + Ok(Self { + runner: config.load_runner()?, + translation: TranslationEngine::default(), + }) + } + + pub(crate) fn manages(&self, request: &Request) -> bool { + request + .llm_request + .model + .as_deref() + .is_some_and(|model| self.runner.route(model).is_some()) + } + + pub(crate) fn decode_request( + &self, + inbound: WireFormat, + request: &RelayRequest, + streaming: bool, + ) -> Result { + let mut llm_request = translation::decode_request(&self.translation, inbound, request)?; + llm_request.stream = streaming; + let headers = string_headers(&request.headers); + let mut metadata = Metadata::from_headers(&headers); + let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id") + && headers + .get("x-nemo-relay-source") + .and_then(|value| value.to_str().ok()) + == Some("gateway") + && metadata.session_id.as_deref() == Some("gateway-gateway"); + if relay_gateway_placeholder { + metadata.session_id = None; + } + metadata.http_headers = Some(headers); + metadata.wire_format = Some(inbound); + Ok(Request { + llm_request, + raw_request: Some(request.content.clone()), + metadata: Some(metadata), + }) + } + + pub(crate) async fn execute_buffered( + &self, + inbound: WireFormat, + request: Request, + ) -> Execution { + let mut execution = self.execute(request).await; + if let Ok(response) = execution.result { + execution.result = finalize_buffered_response(&self.translation, inbound, response); + if execution.result.is_err() { + self.error_mark(&mut execution.marks, "response_finalization", None); + } + } + execution + } + + pub(crate) async fn execute_stream( + &self, + inbound: WireFormat, + request: Request, + ) -> Execution { + let mut execution = self.execute(request).await; + if let Ok(response) = execution.result { + execution.result = returned_events(response, inbound); + if execution.result.is_err() { + self.error_mark(&mut execution.marks, "response_finalization", None); + } + } + execution + } + + async fn execute(&self, request: Request) -> Execution { + let Some(route) = self.route(&request) else { + return Execution { + result: Err("Switchyard has no route for this request model".into()), + marks: Vec::new(), + }; + }; + let metadata = identity_metadata(request.metadata.as_ref()); + let mut marks = vec![RoutingMark { + name: "switchyard.routing.requested".into(), + data: json!({"algorithm": route.algorithm_name()}), + metadata: metadata.clone(), + }]; + if let Err(error) = route.check_caller_format(metadata_wire_format(&request)) { + self.error_mark(&mut marks, "caller_format", None); + return Execution { + result: Err(format!("Switchyard caller format is incompatible: {error}")), + marks, + }; + } + let observations = Arc::new(Mutex::new(Vec::new())); + let observed = Arc::clone(&observations); + let observer: RunObserver = Arc::new(move |observation| { + observed + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(observation); + }); + match route.execute(request, Some(observer)).await { + Ok(output) => { + self.emit_observations(&mut marks, take_observations(&observations), &metadata); + marks.push(RoutingMark { + name: "switchyard.routing.decision".into(), + data: json!({ + "algorithm": route.algorithm_name(), + "selected_model": output.selected_model, + }), + metadata, + }); + Execution { + result: Ok(output.response), + marks, + } + } + Err(_) => { + self.emit_observations(&mut marks, take_observations(&observations), &metadata); + self.error_mark(&mut marks, "route_execution", None); + Execution { + result: Err("Switchyard route execution failed".into()), + marks, + } + } + } + } + + fn route(&self, request: &Request) -> Option<&Route> { + request + .llm_request + .model + .as_deref() + .and_then(|model| self.runner.route(model)) + } + + fn emit_observations( + &self, + marks: &mut Vec, + observations: Vec, + metadata: &Json, + ) { + let mut call_index = 0; + for observation in observations { + match observation { + RunObservation::LlmCall(call) => { + call_index += 1; + self.routing_call_mark(marks, call, call_index, metadata); + } + RunObservation::RoutingOverhead(duration) => marks.push(RoutingMark { + name: "switchyard.routing.overhead".into(), + data: json!({"latency_ms": duration.as_secs_f64() * 1_000.0}), + metadata: metadata.clone(), + }), + RunObservation::AnswerCall(_) => {} + } + } + } + + fn routing_call_mark( + &self, + marks: &mut Vec, + call: LlmCallObservation, + call_index: usize, + metadata: &Json, + ) { + marks.push(RoutingMark { + name: "switchyard.routing.llm_call".into(), + data: json!({ + "call_index": call_index, + "selected_model": call.selected_model, + "call_role": "routing", + "outcome": if call.is_success { "ok" } else { "error" }, + "latency_ms": call.duration.as_secs_f64() * 1_000.0, + "usage": call.usage, + }), + metadata: metadata.clone(), + }); + } + + fn error_mark(&self, marks: &mut Vec, failure_kind: &str, metadata: Option<&Json>) { + let metadata = metadata.cloned().unwrap_or_else(|| { + marks + .first() + .map(|mark| mark.metadata.clone()) + .unwrap_or_else(|| Json::Object(Map::new())) + }); + marks.push(RoutingMark { + name: "switchyard.routing.error".into(), + data: json!({"failure_kind": failure_kind}), + metadata, + }); + } +} + +fn take_observations(observations: &Mutex>) -> Vec { + std::mem::take( + &mut *observations + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + ) +} + +fn metadata_wire_format(request: &Request) -> WireFormat { + request + .metadata + .as_ref() + .and_then(|metadata| metadata.wire_format) + .expect("decoded Relay requests always carry a wire format") +} + +fn finalize_buffered_response( + translation_engine: &TranslationEngine, + inbound: WireFormat, + response: Response, +) -> Result { + let LlmResponse::Agg(response) = response.llm_response else { + return Err("Switchyard returned a stream for a buffered request".into()); + }; + translation::encode_response(translation_engine, inbound, &response) +} + +fn returned_events(response: Response, inbound: WireFormat) -> Result { + let chunks = match response.llm_response { + LlmResponse::Agg(response) => response.into_stream(), + LlmResponse::Stream(chunks) => chunks, + }; + let events = encode_stream(chunks, inbound, None) + .map_err(|error| format!("Switchyard response stream setup failed: {error}"))?; + Ok(Box::pin(events.map(|item| { + item.map_err(|error| format!("Switchyard response stream failed: {error}")) + }))) +} + +fn string_headers(headers: &Map) -> http::HeaderMap { + let mut parsed = http::HeaderMap::with_capacity(headers.len()); + for (name, value) in headers { + let Some(value) = value.as_str() else { + continue; + }; + let (Ok(name), Ok(value)) = ( + http::HeaderName::from_bytes(name.as_bytes()), + http::HeaderValue::from_str(value), + ) else { + continue; + }; + parsed.insert(name, value); + } + parsed +} + +fn identity_metadata(metadata: Option<&Metadata>) -> Json { + json!({ + "session_id": metadata.and_then(|value| value.session_id.as_deref()), + "agent_id": metadata.and_then(|value| value.agent_id.as_deref()), + "parent_agent_id": metadata.and_then(|value| value.parent_agent_id.as_deref()), + "task_id": metadata.and_then(|value| value.task_id.as_deref()), + "turn_id": metadata.and_then(|value| value.turn_id.as_deref()), + "correlation_id": metadata.and_then(|value| value.correlation_id.as_deref()), + }) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, HashMap}; + + use switchyard_llm_client::ClientRouter; + use switchyard_protocol::{ModelId, text_request}; + use switchyard_runner::{AlgorithmSpec, ModelCapabilities}; + + use super::*; + + fn runtime_for(model: &str) -> SwitchyardRuntime { + let algorithm = AlgorithmSpec::Noop {} + .build("relay", &BTreeMap::new()) + .expect("noop route should build"); + let route = Route::new( + algorithm, + ClientRouter::new(HashMap::new()), + None, + ModelCapabilities::default(), + None, + Vec::new(), + ); + SwitchyardRuntime { + runner: Runner::new(vec![(ModelId::from(model), route)]), + translation: TranslationEngine::default(), + } + } + + #[test] + fn only_configured_route_models_are_managed() { + let runtime = runtime_for("switchyard"); + let configured = Request { + llm_request: text_request(Some("switchyard".into()), "hello"), + ..Request::default() + }; + let other = Request { + llm_request: text_request(Some("other".into()), "hello"), + ..Request::default() + }; + + assert!(runtime.manages(&configured)); + assert!(!runtime.manages(&other)); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs new file mode 100644 index 000000000..6f6f965fe --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use nemo_relay_plugin::LlmRequest as RelayRequest; +use serde_json::Value as Json; +use switchyard_protocol::{AggLlmResponse, LlmRequest, WireFormat}; +use switchyard_translation::{ + DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, + TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy, + UnknownFieldPolicy, +}; + +pub(crate) fn decode_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &RelayRequest, +) -> Result { + let output = engine + .decode_request(protocol, &request.content, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.request) +} + +pub(crate) fn validate_target_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &LlmRequest, +) -> Result<(), String> { + let output = engine + .encode_request(protocol, request, &request_policy(protocol)) + .map_err(error)?; + safe(&output.diagnostics) +} + +pub(crate) fn encode_response( + engine: &TranslationEngine, + protocol: WireFormat, + response: &AggLlmResponse, +) -> Result { + let output = engine + .encode_response(protocol, response, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.body) +} + +fn policy() -> TranslationPolicy { + TranslationPolicy { + unknown_field_policy: UnknownFieldPolicy::Preserve, + lossy_conversion_policy: LossyConversionPolicy::Reject, + deterministic_ids: DeterministicIdPolicy::GenerateStable { + prefix: "relay".into(), + }, + preservation: PreservationPolicy::InMemory, + target_capabilities: TargetCapabilities::default(), + } +} + +fn request_policy(protocol: WireFormat) -> TranslationPolicy { + let mut policy = policy(); + if protocol == WireFormat::AnthropicMessages { + policy + .target_capabilities + .supports_json_schema_response_format = Some(false); + } + policy +} + +fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> { + let unsafe_diagnostics = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info) + .collect::>(); + if unsafe_diagnostics.is_empty() { + Ok(()) + } else { + Err(format!( + "Switchyard translation was not lossless: {unsafe_diagnostics:?}" + )) + } +} + +fn error(error: switchyard_translation::TranslationError) -> String { + format!("Switchyard translation failed: {error}") +} + +#[cfg(test)] +mod tests { + use serde_json::{Map, json}; + + use super::*; + + #[test] + fn same_protocol_request_preserves_unknown_fields() { + let request = RelayRequest { + headers: Map::new(), + content: json!({ + "model": "route", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"exact": true} + }), + }; + let engine = TranslationEngine::default(); + let decoded = decode_request(&engine, WireFormat::OpenAiChat, &request).unwrap(); + validate_target_request(&engine, WireFormat::OpenAiChat, &decoded).unwrap(); + assert_eq!( + decoded + .preservation + .requests + .get(&WireFormat::OpenAiChat.into()) + .and_then(|body| body.get("provider_extension")), + Some(&json!({"exact": true})) + ); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py new file mode 100644 index 000000000..296cbc4b8 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the native Relay plugin bundle packager.""" + +from __future__ import annotations + +import hashlib +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] +PACKAGER = CRATE_ROOT / "scripts" / "package_bundle.py" +PACKAGE_NAME = "switchyard-nemo-relay-plugin" + + +class PackageBundleTest(unittest.TestCase): + """Verify materialized and archived plugin bundle contents.""" + + def test_materializes_and_archives_supported_formats(self) -> None: + for archive_suffix in (".tar.gz", ".zip"): + with self.subTest(archive_suffix=archive_suffix), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + library = root / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"compiled plugin") + output = root / "bundle" + archive = root / f"{PACKAGE_NAME}-0.2.0-linux-x86_64{archive_suffix}" + + subprocess.run( + [ + sys.executable, + str(PACKAGER), + "--library", + str(library), + "--output", + str(output), + "--archive", + str(archive), + ], + check=True, + capture_output=True, + text=True, + ) + + expected = { + "LICENSE", + "NOTICE", + "config.schema.json", + library.name, + "relay-plugin.toml", + } + self.assertEqual({path.name for path in output.iterdir()}, expected) + manifest = (output / "relay-plugin.toml").read_text(encoding="utf-8") + self.assertIn(f'artifact = "{library.name}"', manifest) + self.assertIn(hashlib.sha256(library.read_bytes()).hexdigest(), manifest) + self.assertNotIn("", manifest) + self.assertNotIn("", manifest) + self.assertEqual(self.archive_members(archive), {f"{PACKAGE_NAME}/{name}" for name in expected}) + + @staticmethod + def archive_members(archive: Path) -> set[str]: + """Return regular-file paths from a supported bundle archive.""" + if archive.name.endswith(".tar.gz"): + with tarfile.open(archive) as stream: + return {member.name for member in stream.getmembers() if member.isfile()} + with zipfile.ZipFile(archive) as stream: + return {member.filename for member in stream.infolist() if not member.is_dir()} + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/index.md b/docs/index.md index e68ed477b..9b8fb17f9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,6 +9,7 @@ It supports OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages. |---|---|---| | Run Switchyard as a standalone proxy for API clients | Server Path | [Build and run the Rust server](getting_started.md#server-path) | | Add Switchyard routing to a Rust application | Library Path | [`switchyard-libsy`](../crates/libsy/README.md) | +| Add Switchyard routing to NeMo Relay | Native Plugin Path | [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md) | The Server Path builds and runs the standalone `switchyard-server` binary. @@ -27,3 +28,4 @@ The Server Path builds and runs the standalone `switchyard-server` binary. - [`switchyard-libsy`](reference/rust_api.md#switchyard-libsy): embeddable routing algorithms - [`switchyard-protocol`](reference/rust_api.md#switchyard-protocol): provider-neutral API types - [`switchyard-translation`](../crates/switchyard-translation/README.md): protocol translation +- [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md): native NeMo Relay integration