From b2733a2d1ea9e038b178f940e21104391a2d9f91 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Sat, 31 Oct 2026 12:56:15 +0000 Subject: [PATCH 1/2] [ddm, mg-lower, pt6] underlay membership sweep, admin v5, & mrib origination This is the sixth (and last) part in a series of recut PRs split from https://github.com/oxidecomputer/maghemite/pull/675. After pt5, we had ddm peers exchanging multicast subscriptions, but the imports never made their way to the switch, and mgd couldn't originate groups into ddm yet. This work handles both. Includes: - A multicast member sweep in ddmd (`ddm/src/mcast.rs`), which is tasked with programming imported routes into the DPD underlay group members (rear ports for now, geared toward mcast v1). - This sweep runs every 10s and can get woken up early by the db whenever an `(origin, nexthop, interface)` 3-tuple import is added or removed (via watch notify) - A read-modify-write per group per pass. The write passes back the group tag from the read. Nexus still owns group create/delete, and `External` members are not updated. - Only members programmed by this ddmd process (since startup) get removed (same as unicast + tunnel routes). - QSFP "Underlay" members are skipped for now (TODO), as replicas would cycle between racks without RPF or a designated forwarder. - Multicast imports are structured and keyed per `(nexthop, ifname)` link, and `remove_nexthop_{mcast_}routes` take the interface. Before, the same link-local nexthop on two interfaces collapsed into one entry, where losing either link caused a drop. - A new ddm admin API version (`MULTICAST_INTERFACES`), where `get_multicast_groups` returns `ImportedMulticastRoute` with the interface name. - Transits only redistribute imports that are new or changed (including over path/metric modifications). - `mg-common::tfport` kills a TODO and introduces a parser for `tfport_[.vlan]` names, replacing the prefix/suffix string handling we dealt with previously. - mg-lower MRIB origination uses a separate thread (from unicast/tunnels) that diffs `loc_mrib` against ddm's originated groups and advertises/withdraws accordingly. - This is a watcher-driven task with a 1s fallback resync, like unicast. - *testing*: - We introduce a dual-sidecar topology, with a server with one link to each sidecar (e.g., `sl0` -> sidecar a, `sl1` -> sidecar b), importing from both, dropping and restoring `sl1`, and performing a ddmd restart. - Multicast phases have been added to the trio, quartet (failover), and sextet integrations. - *deps* - For `dual-sidecar testing` ~ SoftNPU purposes, we are pinned against oxidecomputer/dendrite#405 for dendrite. --- .github/buildomat/jobs/test-ddm-dual.sh | 18 + .github/buildomat/jobs/test-ddm-trio.sh | 2 +- .github/buildomat/jobs/test-ddm.sh | 8 +- Cargo.lock | 33 +- Cargo.toml | 4 +- ddm-admin-client/src/lib.rs | 1 + ddm-api-types/versions/src/impls/mcast.rs | 35 +- ddm-api-types/versions/src/latest.rs | 2 + ddm-api-types/versions/src/lib.rs | 2 + .../versions/src/multicast_interfaces/db.rs | 25 + .../versions/src/multicast_interfaces/mod.rs | 10 + ddm-api/src/lib.rs | 31 +- ddm/Cargo.toml | 2 + ddm/src/admin.rs | 14 +- ddm/src/db/mcast.rs | 547 +++--- ddm/src/db/mcast/imported.rs | 252 +++ ddm/src/db/mod.rs | 42 +- ddm/src/exchange/mcast.rs | 9 +- ddm/src/exchange/runtime/mcast.rs | 23 +- ddm/src/exchange/runtime/mod.rs | 108 +- ddm/src/lib.rs | 2 + ddm/src/mcast.rs | 1545 +++++++++++++++++ ddm/src/sm/state.rs | 9 +- ddm/src/sys.rs | 60 +- ddmadm/src/main.rs | 17 +- ddmd/src/main.rs | 45 +- mg-common/Cargo.toml | 4 + mg-common/src/lib.rs | 1 + mg-common/src/tfport.rs | 411 +++++ mg-lower/src/ddm.rs | 61 +- mg-lower/src/dendrite.rs | 99 +- mg-lower/src/lib.rs | 46 +- mg-lower/src/mrib.rs | 420 +++++ mg-lower/src/platform.rs | 113 +- mgd/src/main.rs | 19 +- openapi/ddm-admin/ddm-admin-5.0.0-658a80.json | 965 ++++++++++ openapi/ddm-admin/ddm-admin-latest.json | 2 +- tests/Cargo.toml | 2 + tests/conf/dpd-ports-dual.toml | 17 + tests/conf/dpd-ports-sextet.toml | 31 + tests/conf/softnpu-dual-sidecar_a.dual.toml | 5 + tests/conf/softnpu-dual-sidecar_b.dual.toml | 5 + tests/src/ddm.rs | 1038 +++++++++-- 43 files changed, 5536 insertions(+), 549 deletions(-) create mode 100755 .github/buildomat/jobs/test-ddm-dual.sh create mode 100644 ddm-api-types/versions/src/multicast_interfaces/db.rs create mode 100644 ddm-api-types/versions/src/multicast_interfaces/mod.rs create mode 100644 ddm/src/db/mcast/imported.rs create mode 100644 ddm/src/mcast.rs create mode 100644 mg-common/src/tfport.rs create mode 100644 mg-lower/src/mrib.rs create mode 100644 openapi/ddm-admin/ddm-admin-5.0.0-658a80.json create mode 100644 tests/conf/dpd-ports-dual.toml create mode 100644 tests/conf/dpd-ports-sextet.toml create mode 100644 tests/conf/softnpu-dual-sidecar_a.dual.toml create mode 100644 tests/conf/softnpu-dual-sidecar_b.dual.toml diff --git a/.github/buildomat/jobs/test-ddm-dual.sh b/.github/buildomat/jobs/test-ddm-dual.sh new file mode 100755 index 000000000..023ed9e97 --- /dev/null +++ b/.github/buildomat/jobs/test-ddm-dual.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#: +#: name = "test-ddm-dual" +#: variety = "basic" +#: target = "helios-3.0" +#: rust_toolchain = "stable" +#: output_rules = [ +#: "/work/*.log", +#: ] + +source .github/buildomat/test-ddm-common.sh + +# +# dual sidecar tests +# + +banner "dual" +pfexec cargo test --release -p mg-tests test_dual_sidecar -- --nocapture diff --git a/.github/buildomat/jobs/test-ddm-trio.sh b/.github/buildomat/jobs/test-ddm-trio.sh index ab668eca2..49d40e4bc 100755 --- a/.github/buildomat/jobs/test-ddm-trio.sh +++ b/.github/buildomat/jobs/test-ddm-trio.sh @@ -15,4 +15,4 @@ source .github/buildomat/test-ddm-common.sh # banner "trio" -pfexec cargo test --release -p mg-tests test_trio_v3 -- --nocapture +pfexec cargo test --release -p mg-tests test_trio -- --nocapture diff --git a/.github/buildomat/jobs/test-ddm.sh b/.github/buildomat/jobs/test-ddm.sh index f42ff2dc5..547e49038 100755 --- a/.github/buildomat/jobs/test-ddm.sh +++ b/.github/buildomat/jobs/test-ddm.sh @@ -16,4 +16,10 @@ set -o pipefail source .github/buildomat/test-common.sh banner ddm -cargo nextest run -p ddm -p ddm-api-types -p ddm-api-types-versions --nocapture +cargo nextest run \ + -p ddm \ + -p ddm-api-types \ + -p ddm-api-types-versions \ + -p ddm-protocol \ + -p mg-common \ + --nocapture diff --git a/Cargo.lock b/Cargo.lock index 432359360..009d2e60b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -947,7 +947,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -963,7 +963,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" dependencies = [ "anyhow", "chrono", @@ -1431,6 +1431,7 @@ dependencies = [ "ddm-api-types", "ddm-protocol", "dpd-client", + "dpd-types-versions", "dropshot", "hostname 0.4.2", "http-body-util", @@ -1731,7 +1732,7 @@ dependencies = [ [[package]] name = "dpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" dependencies = [ "async-trait", "chrono", @@ -1756,7 +1757,7 @@ dependencies = [ [[package]] name = "dpd-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" dependencies = [ "dpd-types-versions", ] @@ -1764,7 +1765,7 @@ dependencies = [ [[package]] name = "dpd-types-versions" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" dependencies = [ "chrono", "oxnet", @@ -2037,7 +2038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3407,7 +3408,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3998,11 +3999,13 @@ dependencies = [ "anyhow", "backoff", "clap", + "dpd-client", "libc", "libnet", "oximeter", "oximeter-producer", "oxnet", + "proptest", "slog", "slog-async", "slog-bunyan", @@ -4061,6 +4064,8 @@ dependencies = [ "client-common", "ddm-admin-client", "ddm-api-types-versions", + "dpd-client", + "dpd-types-versions", "oxnet", "rand 0.10.3", "serde_json", @@ -6011,7 +6016,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6474,7 +6479,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6541,7 +6546,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7763,7 +7768,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7772,7 +7777,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7782,7 +7787,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9151,7 +9156,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a75c9f589..251edf4b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,8 +158,8 @@ rev = "44052786cb63ad9ff32ec47bd8d65edb6a184e21" [workspace.dependencies.dpd-client] git = "https://github.com/oxidecomputer/dendrite" -rev = "6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +rev = "1d5c2961981d149d85ea08c9ea06f7510a72adbf" [workspace.dependencies.dpd-types-versions] git = "https://github.com/oxidecomputer/dendrite" -rev = "6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +rev = "1d5c2961981d149d85ea08c9ea06f7510a72adbf" diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index fc60883db..658340769 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -15,6 +15,7 @@ progenitor::generate_api!( ExternalMulticastIp = ddm_api_types_versions::latest::net::OverlayMulticast, UnderlayMulticastIpv6 = ddm_api_types_versions::latest::net::UnderlayMulticastIpv6, MulticastRoute = ddm_api_types_versions::latest::db::MulticastRoute, + ImportedMulticastRoute = ddm_api_types_versions::latest::db::ImportedMulticastRoute, MulticastPathHop = ddm_api_types_versions::latest::exchange::MulticastPathHop, PeerInfo = ddm_api_types_versions::latest::db::PeerInfo, PeerStatus = ddm_api_types_versions::latest::db::PeerStatus, diff --git a/ddm-api-types/versions/src/impls/mcast.rs b/ddm-api-types/versions/src/impls/mcast.rs index ebcc36cbc..bc89a5b5d 100644 --- a/ddm-api-types/versions/src/impls/mcast.rs +++ b/ddm-api-types/versions/src/impls/mcast.rs @@ -36,14 +36,16 @@ impl From for MulticastOrigin { #[cfg(test)] mod tests { + use std::collections::HashSet; use std::hash::{DefaultHasher, Hash, Hasher}; use std::net::Ipv6Addr; - use crate::latest::db::MulticastRoute; - use crate::latest::exchange::MulticastPathHop; - use crate::latest::net::{ + use crate::v4::db::MulticastRoute; + use crate::v4::exchange::MulticastPathHop; + use crate::v4::net::{ MulticastOrigin, OverlayMulticast, UnderlayMulticastIpv6, Vni, }; + use crate::v5::db::ImportedMulticastRoute; fn origin(overlay: &str) -> MulticastOrigin { MulticastOrigin::new( @@ -55,6 +57,33 @@ mod tests { .unwrap() } + #[test] + fn v4_conversion_drops_interface_ident() { + let route = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: "fe80::1".parse().unwrap(), + path: vec![MulticastPathHop::new( + "peer".into(), + "fe80::1".parse().unwrap(), + )], + }; + let imported = HashSet::from([ + ImportedMulticastRoute { + route: route.clone(), + if_name: "tfportrear0_0".into(), + }, + ImportedMulticastRoute { + route: route.clone(), + if_name: "tfportrear1_0".into(), + }, + ]); + assert_eq!(imported.len(), 2); + + let prev: HashSet = + imported.into_iter().map(Into::into).collect(); + assert_eq!(prev, HashSet::from([route])); + } + /// Use the same initial hasher state for each route comparison. fn hash_of(route: &MulticastRoute) -> u64 { let mut hasher = DefaultHasher::new(); diff --git a/ddm-api-types/versions/src/latest.rs b/ddm-api-types/versions/src/latest.rs index a187e983b..798da112b 100644 --- a/ddm-api-types/versions/src/latest.rs +++ b/ddm-api-types/versions/src/latest.rs @@ -18,6 +18,8 @@ pub mod db { pub use crate::v4::db::MulticastRoute; pub use crate::v4::db::PeerInfo; + + pub use crate::v5::db::ImportedMulticastRoute; } pub mod exchange { diff --git a/ddm-api-types/versions/src/lib.rs b/ddm-api-types/versions/src/lib.rs index 4af2b9be6..c6eb560df 100644 --- a/ddm-api-types/versions/src/lib.rs +++ b/ddm-api-types/versions/src/lib.rs @@ -39,3 +39,5 @@ pub mod v2; pub mod v3; #[path = "multicast_support/mod.rs"] pub mod v4; +#[path = "multicast_interfaces/mod.rs"] +pub mod v5; diff --git a/ddm-api-types/versions/src/multicast_interfaces/db.rs b/ddm-api-types/versions/src/multicast_interfaces/db.rs new file mode 100644 index 000000000..87d019523 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_interfaces/db.rs @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v4; + +/// A multicast route with the interface on which its peer was discovered. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, +)] +pub struct ImportedMulticastRoute { + /// The imported multicast route. + pub route: v4::db::MulticastRoute, + /// Local interface on which the advertising peer was discovered. + pub if_name: String, +} + +impl From for v4::db::MulticastRoute { + fn from(imported: ImportedMulticastRoute) -> Self { + imported.route + } +} diff --git a/ddm-api-types/versions/src/multicast_interfaces/mod.rs b/ddm-api-types/versions/src/multicast_interfaces/mod.rs new file mode 100644 index 000000000..8e8e7e533 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_interfaces/mod.rs @@ -0,0 +1,10 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `MULTICAST_INTERFACES` of the DDM Admin API. +//! +//! Imported multicast routes convey the interface that their peer was +//! discovered on. + +pub mod db; diff --git a/ddm-api/src/lib.rs b/ddm-api/src/lib.rs index 0ae6cf6d7..e0730016f 100644 --- a/ddm-api/src/lib.rs +++ b/ddm-api/src/lib.rs @@ -2,9 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use ddm_api_types_versions::latest; -use ddm_api_types_versions::v1; -use ddm_api_types_versions::v2; +use ddm_api_types_versions::{latest, v1, v2, v4}; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -27,6 +25,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (5, MULTICAST_INTERFACES), (4, MULTICAST_SUPPORT), (3, EXTERNAL_PEERS), (2, PEER_DURATIONS), @@ -177,11 +176,33 @@ pub trait DdmAdminApi { #[endpoint { method = GET, path = "/multicast_groups", - versions = VERSION_MULTICAST_SUPPORT.. + versions = VERSION_MULTICAST_INTERFACES.. }] async fn get_multicast_groups( ctx: RequestContext, - ) -> Result>, HttpError>; + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + #[endpoint { + operation_id = "get_multicast_groups", + method = GET, + path = "/multicast_groups", + versions = VERSION_MULTICAST_SUPPORT..VERSION_MULTICAST_INTERFACES + }] + async fn get_multicast_groups_v4( + ctx: RequestContext, + ) -> Result>, HttpError> + { + let resp = Self::get_multicast_groups(ctx).await?; + Ok(HttpResponseOk( + resp.0 + .into_iter() + .map(v4::db::MulticastRoute::from) + .collect(), + )) + } #[endpoint { method = PUT, diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index 0acd6d17b..d2aa6bec9 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" pretty_assertions.workspace = true camino-tempfile.workspace = true serde_urlencoded.workspace = true +tokio = { workspace = true, features = ["test-util"] } [dependencies] slog.workspace = true @@ -38,6 +39,7 @@ oxnet.workspace = true uuid.workspace = true ddm-api.workspace = true dpd-client.workspace = true +dpd-types-versions.workspace = true # illumos-only deps used by the routing state machine and platform sys layer. # Gated by the `backend` feature so stub builds (e.g. Linux test fixtures diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 5dbf60c3f..22b0494f0 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -15,7 +15,9 @@ use camino::Utf8PathBuf; use ddm_api::DdmAdminApi; use ddm_api::ddm_admin_api_mod; use ddm_api_types::admin::{EnableStatsRequest, ExpirePathParams, PrefixMap}; -use ddm_api_types::db::{MulticastRoute, PeerInfo, RouterKind, TunnelRoute}; +use ddm_api_types::db::{ + ImportedMulticastRoute, PeerInfo, RouterKind, TunnelRoute, +}; use ddm_api_types::exchange::{PathVector, RouterId}; use ddm_api_types::external_peers::ExternalPeers; use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; @@ -436,9 +438,15 @@ impl DdmAdminApi for DdmAdminApiImpl { async fn get_multicast_groups( ctx: RequestContext, - ) -> Result>, HttpError> { + ) -> Result>, HttpError> + { let ctx = lock!(ctx.context()); - let imported = ctx.db.imported_mcast(); + let imported = ctx + .db + .imported_mcast_links() + .into_iter() + .map(|(route, if_name)| ImportedMulticastRoute { route, if_name }) + .collect(); Ok(HttpResponseOk(imported)) } diff --git a/ddm/src/db/mcast.rs b/ddm/src/db/mcast.rs index da954e79a..91196dda8 100644 --- a/ddm/src/db/mcast.rs +++ b/ddm/src/db/mcast.rs @@ -16,12 +16,29 @@ use slog::error; use super::{Db, Error, RemovedNexthopRoutes}; +mod imported; + +pub(in crate::db) use imported::ImportedMcast; +use imported::Inserted; + +/// The effect of [`Db::splice_imported_mcast`] on the imported set. +pub struct SplicedMulticast { + /// Imports that added an entry or changed an entry's path or metric. + pub imported: HashSet, + /// Routes that were removed. + pub removed: HashSet, +} + /// Persistent tree for locally originated multicast groups. const MCAST_ORIGINATE: &str = "mcast_originate"; impl Db { - pub fn imported_mcast(&self) -> HashSet { - lock!(self.data).imported_mcast.routes().cloned().collect() + pub fn imported_mcast_links(&self) -> Vec<(MulticastRoute, String)> { + lock!(self.data) + .imported_mcast + .links() + .map(|(route, ifname)| (route.clone(), ifname.to_string())) + .collect() } pub fn shortest_path_mcast( @@ -35,34 +52,44 @@ impl Db { .collect() } - /// Sequence cutoff for routes imported before a pull occurs. + /// Pull sequence number used to identify earlier imports. pub fn next_mcast_seq(&self) -> u64 { lock!(self.data).mcast_seq + 1 } /// Apply multicast imports and removals under a held data lock, returning - /// the routes that were actually removed. - pub fn update_imported_mcast( + /// the imports that affected the RIB and the routes that were actually + /// removed. + pub fn splice_imported_mcast( &self, update: MulticastRibChange<'_>, - ) -> HashSet { + ) -> SplicedMulticast { let mut guard = lock!(self.data); let data = &mut *guard; + let mut edit = data.imported_mcast.edit(&self.mcast_wake); // Path and metric live outside of route identity and may change on // re-import. - for x in update.import { - data.mcast_seq += 1; - data.imported_mcast.insert(x, data.mcast_seq); - } + let imported = update + .import + .iter() + .filter(|x| { + data.mcast_seq += 1; + edit.insert(x, update.ifname, data.mcast_seq) + != Inserted::Unchanged + }) + .cloned() + .collect(); - update + let removed = update .remove .iter() .filter_map(|x| { - data.imported_mcast.remove(x, update.imported_before) + edit.remove(x, update.ifname, update.imported_before) }) - .collect() + .collect(); + + SplicedMulticast { imported, removed } } /// Persist multicast origins to advertise to peers. @@ -144,14 +171,19 @@ impl Db { self.path_snapshot(&origins, HashMap::new()) } - /// Remove multicast routes via `nexthop` on a downgrade below V4. + /// Remove multicast routes via `nexthop` on `ifname` on a downgrade below + /// V4. /// /// Underlay and tunnel routes remain intact. pub fn remove_nexthop_mcast_routes( &self, nexthop: Ipv6Addr, + ifname: &str, ) -> RemovedNexthopRoutes { - let removed = lock!(self.data).imported_mcast.extract_nexthop(nexthop); + let removed = lock!(self.data) + .imported_mcast + .edit(&self.mcast_wake) + .extract_link(nexthop, ifname); RemovedNexthopRoutes { underlay: HashSet::new(), @@ -160,17 +192,18 @@ impl Db { } } - /// Origins imported via `nexthop` before `seq` that are not in + /// Origins imported via `nexthop` on `ifname` before `seq` that are not in /// `announced`. pub fn stale_mcast_by_nexthop( &self, nexthop: Ipv6Addr, + ifname: &str, seq: u64, announced: &HashSet, ) -> Vec { lock!(self.data) .imported_mcast - .stale(nexthop, seq, announced) + .stale(nexthop, ifname, seq, announced) } /// Read local origins to complete the snapshot of `imported`. @@ -213,148 +246,6 @@ impl Db { } } -#[derive(Debug, Clone)] -struct ImportedEntry { - route: MulticastRoute, - seq: u64, -} - -/// A type wrapper over the set of imported multicast routes, keyed by origin -/// and then by nexthop. -/// -/// A path snapshot for an update's origins is a lookup per origin. Each -/// origin's routes are sorted by nexthop, with the lowest-nexthop route -/// coming first. Per-peer work (e.g., stale-route scans, expiry) visits each -/// origin once. -// Note: We use a sorted `Vec` here over a `BTreeMap` because an origin rarely -// has more than a few nexthops. -#[derive(Debug, Default, Clone)] -pub struct ImportedMcast(HashMap>); - -fn find_nexthop( - nexthops: &[ImportedEntry], - nexthop: Ipv6Addr, -) -> Result { - nexthops.binary_search_by_key(&nexthop, |entry| entry.route.nexthop) -} - -impl ImportedMcast { - fn routes(&self) -> impl Iterator { - self.0.values().flatten().map(|entry| &entry.route) - } - - fn shortest_path_routes<'a>( - &'a self, - peer_router_id: &'a RouterId, - ) -> impl Iterator { - self.0.keys().filter_map(move |origin| { - self.shortest_path(origin, peer_router_id) - }) - } - - fn insert(&mut self, route: &MulticastRoute, seq: u64) { - let nexthops = self.0.entry(route.origin.clone()).or_default(); - match find_nexthop(nexthops, route.nexthop) { - Ok(idx) => { - let entry = &mut nexthops[idx]; - if entry.route.path != route.path - || entry.route.origin.metric() != route.origin.metric() - { - entry.route = route.clone(); - } - entry.seq = seq; - } - Err(idx) => { - // Grow one slot at a time. - nexthops.reserve_exact(1); - nexthops.insert( - idx, - ImportedEntry { - route: route.clone(), - seq, - }, - ); - } - } - } - - fn remove( - &mut self, - route: &MulticastRoute, - imported_before: Option, - ) -> Option { - let nexthops = self.0.get_mut(&route.origin)?; - let idx = find_nexthop(nexthops, route.nexthop).ok()?; - if imported_before.is_some_and(|seq| nexthops[idx].seq >= seq) { - return None; - } - let entry = nexthops.remove(idx); - if nexthops.is_empty() { - self.0.remove(&route.origin); - } - Some(entry.route) - } - - pub(super) fn extract_nexthop( - &mut self, - nexthop: Ipv6Addr, - ) -> HashSet { - let mut extracted = HashSet::new(); - self.0.retain(|_, nexthops| { - if let Ok(idx) = find_nexthop(nexthops, nexthop) { - extracted.insert(nexthops.remove(idx).route); - } - !nexthops.is_empty() - }); - extracted - } - - /// Pick the route with the shortest path, breaking ties by the lowest - /// origin metric (first) and then by the lowest nexthop address (e.g., - /// `fe80::1` < `fe80::2`). - /// - /// Routes whose path contains `peer_router_id` (the peer an update gets - /// sent to) are omitted. That peer would discard such a path as a loop - /// and drop its route through us. - fn shortest_path( - &self, - origin: &MulticastOrigin, - peer_router_id: &RouterId, - ) -> Option<&MulticastRoute> { - self.0 - .get(origin)? - .iter() - .map(|entry| &entry.route) - .filter(|route| { - !route - .path - .iter() - .any(|hop| &hop.router_id == peer_router_id) - }) - .min_by_key(|route| { - (route.path.len(), route.origin.metric(), route.nexthop) - }) - } - - fn stale( - &self, - nexthop: Ipv6Addr, - seq: u64, - announced: &HashSet, - ) -> Vec { - self.0 - .values() - .filter_map(|nexthops| { - find_nexthop(nexthops, nexthop).ok().map(|i| &nexthops[i]) - }) - .filter(|entry| { - entry.seq < seq && !announced.contains(&entry.route.origin) - }) - .map(|entry| entry.route.origin.clone()) - .collect() - } -} - /// Persistent origin key, excluding metric as in `PartialEq` and `Hash`. fn mcast_origin_key(origin: &MulticastOrigin) -> Result { Ok(serde_json::to_string(&( @@ -368,20 +259,23 @@ fn mcast_origin_key(origin: &MulticastOrigin) -> Result { /// A batch of multicast imports and removals. #[derive(Debug, Clone, Copy)] pub struct MulticastRibChange<'a> { + pub ifname: &'a str, pub import: &'a HashSet, pub remove: &'a HashSet, /// Remove only routes imported before this sequence number. - /// Pull reconciliation uses this cutoff to preserve concurrent pushes. + /// Pull reconciliation uses this sequence to preserve concurrent pushes. pub imported_before: Option, } impl<'a> MulticastRibChange<'a> { - /// A change without a sequence cutoff, as used for push withdrawals. + /// A change without a pull sequence, as used for push withdrawals. pub fn new( + ifname: &'a str, import: &'a HashSet, remove: &'a HashSet, ) -> Self { Self { + ifname, import, remove, imported_before: None, @@ -422,6 +316,7 @@ pub(crate) mod tests { use ddm_protocol::v4::MulticastPathHop; use pretty_assertions::assert_eq; use slog::Logger; + use tokio::sync::watch; use super::*; @@ -456,7 +351,8 @@ pub(crate) mod tests { } fn import(db: &Db, routes: &HashSet) { - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", routes, &HashSet::new(), )); @@ -488,7 +384,8 @@ pub(crate) mod tests { path: Vec::new(), }; db.originate_mcast(&HashSet::from([good.clone()])).unwrap(); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([route.clone()]), &HashSet::new(), )); @@ -523,7 +420,8 @@ pub(crate) mod tests { db.originate_mcast(&HashSet::from([local_origin.clone()])) .unwrap(); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([route.clone()]), &HashSet::new(), )); @@ -604,7 +502,7 @@ pub(crate) mod tests { db.import_tunnel(&HashSet::from([tunnel])); let removed = - db.remove_nexthop_mcast_routes("fe80::a".parse().unwrap()); + db.remove_nexthop_mcast_routes("fe80::a".parse().unwrap(), "test0"); // A snapshot taken after removal no longer views the removed routes. let paths = db @@ -615,7 +513,7 @@ pub(crate) mod tests { assert_eq!(removed.multicast, HashSet::from([a, a2])); assert!(removed.underlay.is_empty()); assert!(removed.tunnel.is_empty()); - assert_eq!(db.imported_mcast(), HashSet::from([b.clone()])); + assert_eq!(db.imported_mcast_links(), vec![(b, "test0".to_string())]); assert_eq!(db.imported_count(), 1); assert_eq!(db.imported_tunnel(), HashSet::from([tunnel])); } @@ -656,16 +554,11 @@ pub(crate) mod tests { let tunnel_b = tunnel("fe80::b", "fd00:1701::b"); db.import_tunnel(&HashSet::from([tunnel_a, tunnel_b])); - let removed = db.remove_nexthop_routes("fe80::a".parse().unwrap()); + let removed = + db.remove_nexthop_routes("fe80::a".parse().unwrap(), "test0"); assert_eq!(removed.underlay, HashSet::from([underlay_a])); assert_eq!(removed.tunnel, HashSet::from([tunnel_a])); - let paths = db.snapshot_mcast_paths( - [&multicast_a.origin, &multicast_b.origin], - &NO_PEER, - ); - assert!(paths.imported(&multicast_a.origin).is_none()); - assert_eq!(paths.imported(&multicast_b.origin), Some(&multicast_b)); assert_eq!(removed.multicast, HashSet::from([multicast_a])); assert_eq!(db.imported(), HashSet::from([underlay_b])); assert_eq!(db.imported_tunnel(), HashSet::from([tunnel_b])); @@ -692,6 +585,13 @@ pub(crate) mod tests { other_peer, ]), ); + let other_link = + route(mcast_origin("233.252.0.5", "ff04::5"), "fe80::a"); + db.splice_imported_mcast(MulticastRibChange::new( + "test1", + &HashSet::from([other_link.clone()]), + &HashSet::new(), + )); let seq = db.next_mcast_seq(); let late = route(mcast_origin("233.252.0.4", "ff04::4"), "fe80::a"); @@ -699,16 +599,25 @@ pub(crate) mod tests { let stale = db.stale_mcast_by_nexthop( peer, + "test0", seq, - &HashSet::from([announced.origin]), + &HashSet::from([announced.origin.clone()]), ); assert_eq!(stale, vec![unannounced.origin]); + + let stale = db.stale_mcast_by_nexthop( + peer, + "test1", + seq, + &HashSet::from([announced.origin]), + ); + assert_eq!(stale, vec![other_link.origin]); } - /// The cutoff removes an untouched route and preserves imports made - /// after it. + /// Remove an untouched route and preserve imports made at or after the + /// pull sequence. #[test] - fn stale_removal_skips_routes_imported_after_the_cutoff() { + fn stale_removal_skips_routes_imported_at_or_after_the_pull_seq() { let (_dir, db) = test_db(); let prev = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); @@ -716,7 +625,7 @@ pub(crate) mod tests { route(mcast_origin("233.252.0.2", "ff04::2"), "fe80::a"); import(&db, &HashSet::from([prev.clone(), refreshed.clone()])); - let started_at = db.next_mcast_seq(); + let pull_seq = db.next_mcast_seq(); // A concurrent push re-announces `refreshed` and adds a `new` route // while the pull is reading a snapshot that contains neither of these. @@ -736,14 +645,25 @@ pub(crate) mod tests { ..new.clone() }, ]); - let removed = db.update_imported_mcast(MulticastRibChange { - import: &HashSet::new(), - remove: &candidates, - imported_before: Some(started_at), - }); + let removed = db + .splice_imported_mcast(MulticastRibChange { + ifname: "test0", + import: &HashSet::new(), + remove: &candidates, + imported_before: Some(pull_seq), + }) + .removed; assert_eq!(removed, HashSet::from([prev])); - assert_eq!(db.imported_mcast(), HashSet::from([refreshed, new])); + assert_eq!( + db.imported_mcast_links() + .into_iter() + .collect::>(), + HashSet::from([ + (refreshed, "test0".to_string()), + (new, "test0".to_string()), + ]), + ); } #[test] @@ -766,7 +686,8 @@ pub(crate) mod tests { Some((low.nexthop, low.path.clone())) ); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::new(), &HashSet::from([low.clone()]), )); @@ -789,7 +710,7 @@ pub(crate) mod tests { ); import(&db, &HashSet::from([low.clone()])); - let removed = db.remove_nexthop_mcast_routes(low.nexthop); + let removed = db.remove_nexthop_mcast_routes(low.nexthop, "test0"); assert_eq!(removed.multicast.len(), 2); assert_eq!( shortest(&db, &origin), @@ -797,9 +718,9 @@ pub(crate) mod tests { ); assert_eq!(shortest(&db, &other), None); - db.remove_nexthop_mcast_routes(high.nexthop); + db.remove_nexthop_mcast_routes(high.nexthop, "test0"); assert_eq!(shortest(&db, &origin), None); - assert!(db.imported_mcast().is_empty()); + assert!(db.imported_mcast_links().is_empty()); } #[test] @@ -818,9 +739,12 @@ pub(crate) mod tests { let peer_in_path = via("fe80::1", &["peer", "x"]); let long = via("fe80::2", &["a", "b", "c"]); let short = via("fe80::3", &["d", "e"]); - imported.insert(&peer_in_path, 1); - imported.insert(&long, 2); - imported.insert(&short, 3); + let wake = watch::Sender::new(0); + let mut edit = imported.edit(&wake); + edit.insert(&peer_in_path, "test0", 1); + edit.insert(&long, "test0", 2); + edit.insert(&short, "test0", 3); + drop(edit); let picked = imported.shortest_path(&origin, &"peer".into()).unwrap(); assert_eq!(picked.nexthop, short.nexthop); @@ -845,13 +769,16 @@ pub(crate) mod tests { let low_nexthop = via("fe80::1", 20, &["a", "b"]); let low_metric = via("fe80::2", 10, &["c", "d"]); let shorter = via("fe80::3", 30, &["e"]); - imported.insert(&low_nexthop, 1); - imported.insert(&low_metric, 2); + let wake = watch::Sender::new(0); + let mut edit = imported.edit(&wake); + edit.insert(&low_nexthop, "test0", 1); + edit.insert(&low_metric, "test0", 2); + drop(edit); let picked = imported.shortest_path(&origin, &"peer".into()).unwrap(); assert_eq!(picked.nexthop, low_metric.nexthop); - imported.insert(&shorter, 3); + imported.edit(&wake).insert(&shorter, "test0", 3); let picked = imported.shortest_path(&origin, &"peer".into()).unwrap(); assert_eq!(picked.nexthop, shorter.nexthop); } @@ -859,14 +786,234 @@ pub(crate) mod tests { #[test] fn imported_mcast_drops_emptied_origins() { let mut imported = ImportedMcast::default(); + let wake = watch::Sender::new(0); let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::1"); - imported.insert(&a, 1); - assert_eq!(imported.remove(&a, None), Some(a.clone())); - assert!(imported.0.is_empty()); + let mut edit = imported.edit(&wake); + edit.insert(&a, "test0", 1); + assert_eq!(edit.remove(&a, "test0", None), Some(a.clone())); + drop(edit); + assert!(imported.links().next().is_none()); + } + + #[test] + fn imports_are_keyed_per_link() { + let mut imported = ImportedMcast::default(); + let wake = watch::Sender::new(0); + let ifnames = |imported: &ImportedMcast| -> Vec { + imported + .links() + .map(|(_, ifname)| ifname.to_string()) + .collect() + }; + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::1"); + + let mut edit = imported.edit(&wake); + assert_eq!(edit.insert(&a, "tfportrear0_0", 1), Inserted::New); + assert_eq!(edit.insert(&a, "tfportrear1_0", 2), Inserted::New); + drop(edit); + assert_eq!(*wake.borrow(), 1); + + assert_eq!( + imported.edit(&wake).insert(&a, "tfportrear0_0", 3), + Inserted::Unchanged + ); + assert_eq!(*wake.borrow(), 1); + assert_eq!(imported.links().count(), 2); + + assert_eq!( + imported.edit(&wake).remove(&a, "tfportrear0_0", None), + Some(a.clone()) + ); + assert_eq!(ifnames(&imported), vec!["tfportrear1_0"]); + + let mut edit = imported.edit(&wake); + edit.insert(&a, "tfportrear0_0", 4); + assert_eq!( + edit.extract_link(a.nexthop, "tfportrear0_0"), + HashSet::from([a.clone()]), + ); + drop(edit); + assert_eq!(ifnames(&imported), vec!["tfportrear1_0"]); + assert_eq!( + imported + .edit(&wake) + .extract_link(a.nexthop, "tfportrear1_0"), + HashSet::from([a]), + ); + assert!(imported.links().next().is_none()); + assert_eq!(*wake.borrow(), 4); + } + + #[test] + fn splice_reports_only_changed_imports() { + let (_dir, db) = test_db(); + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); + let splice = |routes: &HashSet| { + db.splice_imported_mcast(MulticastRibChange::new( + "test0", + routes, + &HashSet::new(), + )) + .imported + }; + + let routes = HashSet::from([a.clone()]); + assert_eq!(splice(&routes), routes); + assert!(splice(&routes).is_empty()); + + let mut wake = db.subscribe_mcast_sweep(); + wake.borrow_and_update(); + + let remetric = MulticastRoute { + origin: a.origin.clone().with_metric(5), + ..a + }; + let changed = splice(&HashSet::from([remetric.clone()])); + assert_eq!( + changed + .iter() + .map(|r| r.origin.metric()) + .collect::>(), + vec![5], + ); + assert!(splice(&HashSet::from([remetric])).is_empty()); + assert!(!wake.has_changed().unwrap()); + } + + fn check_multicast_removal_by_interface( + remove_routes: impl Fn(&Db, Ipv6Addr, &str) -> RemovedNexthopRoutes, + ) { + let (_dir, db) = test_db(); + let shared = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::1"); + let unique = route(mcast_origin("233.252.0.2", "ff04::2"), "fe80::1"); + let other = route(mcast_origin("233.252.0.3", "ff04::3"), "fe80::2"); + db.splice_imported_mcast(MulticastRibChange::new( + "tfportrear0_0", + &HashSet::from([shared.clone(), unique.clone(), other.clone()]), + &HashSet::new(), + )); + db.splice_imported_mcast(MulticastRibChange::new( + "tfportrear1_0", + &HashSet::from([shared.clone()]), + &HashSet::new(), + )); + + let mut rx = db.subscribe_mcast_sweep(); + let generation = *rx.borrow_and_update(); + let removed = remove_routes(&db, shared.nexthop, "tfportrear2_0"); + assert!(removed.multicast.is_empty()); + assert_eq!(db.imported_mcast_links().len(), 4); + assert_eq!(*rx.borrow_and_update(), generation); + + let removed = remove_routes(&db, shared.nexthop, "tfportrear0_0"); + assert_eq!( + removed.multicast, + HashSet::from([shared.clone(), unique.clone()]), + ); + assert_eq!( + db.imported_mcast_links() + .into_iter() + .collect::>(), + HashSet::from([ + (shared.clone(), "tfportrear1_0".to_string()), + (other.clone(), "tfportrear0_0".to_string()), + ]), + ); + let paths = + db.snapshot_mcast_paths([&shared.origin, &unique.origin], &NO_PEER); + assert_eq!(paths.imported(&shared.origin), Some(&shared)); + assert!(paths.imported(&unique.origin).is_none()); + assert_eq!(*rx.borrow_and_update(), generation + 1); + + let removed = remove_routes(&db, shared.nexthop, "tfportrear0_0"); + assert!(removed.multicast.is_empty()); + assert_eq!(*rx.borrow_and_update(), generation + 1); + + let removed = remove_routes(&db, shared.nexthop, "tfportrear1_0"); + assert_eq!(removed.multicast, HashSet::from([shared.clone()])); + assert_eq!( + db.imported_mcast_links(), + vec![(other, "tfportrear0_0".to_string())], + ); + assert!( + db.snapshot_mcast_paths([&shared.origin], &NO_PEER) + .imported(&shared.origin) + .is_none(), + ); + assert_eq!(*rx.borrow_and_update(), generation + 2); + } + + #[test] + fn expiry_preserves_multicast_on_other_interfaces() { + check_multicast_removal_by_interface(Db::remove_nexthop_routes); + } + + #[test] + fn downgrade_preserves_multicast_on_other_interfaces() { + check_multicast_removal_by_interface(Db::remove_nexthop_mcast_routes); + } + + #[test] + fn membership_changes_wake_subscribers() { + let (_dir, db) = test_db(); + let mut rx = db.subscribe_mcast_sweep(); + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); + let b = route(mcast_origin("233.252.0.2", "ff04::2"), "fe80::a"); + + import(&db, &HashSet::from([a.clone()])); + assert!(rx.has_changed().unwrap()); + rx.borrow_and_update(); + + import(&db, &HashSet::from([b.clone()])); + assert!(rx.has_changed().unwrap()); + rx.borrow_and_update(); + + db.splice_imported_mcast(MulticastRibChange::new( + "test0", + &HashSet::new(), + &HashSet::from([b.clone()]), + )); + assert!(rx.has_changed().unwrap()); + } + + #[test] + fn non_membership_changes_do_not_wake_subscribers() { + let (_dir, db) = test_db(); + let mut rx = db.subscribe_mcast_sweep(); + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); + let absent = route(mcast_origin("233.252.0.9", "ff04::9"), "fe80::b"); + + let pull_seq = db.next_mcast_seq(); + import(&db, &HashSet::from([a.clone()])); + rx.borrow_and_update(); + + import(&db, &HashSet::from([a.clone()])); + let mut rerouted = a.clone(); + rerouted.path.push(MulticastPathHop::new( + "other".into(), + "fe80::3".parse().unwrap(), + )); + import(&db, &HashSet::from([rerouted])); + + import(&db, &HashSet::new()); + db.splice_imported_mcast(MulticastRibChange::new( + "test0", + &HashSet::new(), + &HashSet::from([absent.clone()]), + )); + let removed = db + .splice_imported_mcast(MulticastRibChange { + ifname: "test0", + import: &HashSet::new(), + remove: &HashSet::from([a.clone()]), + imported_before: Some(pull_seq), + }) + .removed; + assert!(removed.is_empty()); + db.remove_nexthop_mcast_routes(absent.nexthop, "test0"); + db.remove_nexthop_routes(absent.nexthop, "test0"); - imported.insert(&a, 2); - assert_eq!(imported.extract_nexthop(a.nexthop), HashSet::from([a])); - assert!(imported.0.is_empty()); + assert!(!rx.has_changed().unwrap()); } } diff --git a/ddm/src/db/mcast/imported.rs b/ddm/src/db/mcast/imported.rs new file mode 100644 index 000000000..183373bc5 --- /dev/null +++ b/ddm/src/db/mcast/imported.rs @@ -0,0 +1,252 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The imported multicast set, made mutable through an [`ImportedMcastEdit`] +//! guard that wakes the multicast sweep when membership gets updated. + +use std::collections::{HashMap, HashSet}; +use std::net::Ipv6Addr; + +use ddm_api_types::db::MulticastRoute; +use ddm_api_types::net::MulticastOrigin; +use ddm_protocol::RouterId; +use tokio::sync::watch; + +#[derive(Debug, Clone)] +struct ImportedEntry { + route: MulticastRoute, + ifname: String, + seq: u64, +} + +/// A type wrapper over the set of imported multicast routes, keyed by origin +/// and then by nexthop and the interface that its peer was discovered on. +/// +/// A lookup per origin produces the path snapshot for an update. Each +/// origin's routes are sorted by nexthop and then by interface, with the +/// lowest-nexthop route first. +/// +/// Per-peer work (e.g., stale-route scans, expiry) visits each origin just +/// once. +// Note: We use a sorted `Vec` here over a `BTreeMap` because an origin rarely +// has more than a few nexthops. +#[derive(Debug, Default, Clone)] +pub struct ImportedMcast(HashMap>); + +fn find_link( + nexthops: &[ImportedEntry], + nexthop: Ipv6Addr, + ifname: &str, +) -> Result { + nexthops.binary_search_by(|entry| { + (entry.route.nexthop, entry.ifname.as_str()).cmp(&(nexthop, ifname)) + }) +} + +impl ImportedMcast { + /// Begin an edit operation. + /// + /// Dropping the returned guard wakes `wake`'s watched subscribers if the + /// edit added or removed an `(origin, nexthop, ifname)` entry. + pub(in crate::db) fn edit<'a>( + &'a mut self, + wake: &'a watch::Sender, + ) -> ImportedMcastEdit<'a> { + ImportedMcastEdit { + imported: self, + wake, + changed: false, + } + } + + /// Each imported route paired with the interface its peer was discovered + /// on. + pub(in crate::db) fn links( + &self, + ) -> impl Iterator { + self.0 + .values() + .flatten() + .map(|entry| (&entry.route, entry.ifname.as_str())) + } + + pub(in crate::db) fn shortest_path_routes<'a>( + &'a self, + peer_router_id: &'a RouterId, + ) -> impl Iterator { + self.0.keys().filter_map(move |origin| { + self.shortest_path(origin, peer_router_id) + }) + } + + /// Pick the route with the shortest path, breaking ties by the lowest + /// origin metric (first), then by the lowest nexthop address (e.g., + /// `fe80::1` < `fe80::2`), and finally by the interface name. + /// + /// Routes whose path contains `peer_router_id` (the peer an update gets + /// sent to) are omitted. That peer would discard such a path as a loop + /// and drop its route through us. + pub(in crate::db) fn shortest_path( + &self, + origin: &MulticastOrigin, + peer_router_id: &RouterId, + ) -> Option<&MulticastRoute> { + let key = |entry: &ImportedEntry| { + ( + entry.route.path.len(), + entry.route.origin.metric(), + entry.route.nexthop, + ) + }; + self.0 + .get(origin)? + .iter() + .filter(|entry| { + !entry + .route + .path + .iter() + .any(|hop| &hop.router_id == peer_router_id) + }) + .min_by(|a, b| { + key(a).cmp(&key(b)).then_with(|| a.ifname.cmp(&b.ifname)) + }) + .map(|entry| &entry.route) + } + + /// Return origins imported via `nexthop` on `ifname` before a `seq` + /// that are not contained in `announced`. + pub(in crate::db) fn stale( + &self, + nexthop: Ipv6Addr, + ifname: &str, + seq: u64, + announced: &HashSet, + ) -> Vec { + self.0 + .values() + .filter_map(|nexthops| { + find_link(nexthops, nexthop, ifname) + .ok() + .map(|i| &nexthops[i]) + }) + .filter(|entry| { + entry.seq < seq && !announced.contains(&entry.route.origin) + }) + .map(|entry| entry.route.origin.clone()) + .collect() + } +} + +/// The effect of [`ImportedMcastEdit::insert`] on the imported set. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(in crate::db) enum Inserted { + /// A new `(origin, nexthop, ifname)` entry. + New, + /// An existing entry whose path or metric changed. + Changed, + /// An existing entry refreshed without change. + Unchanged, +} + +/// An edit guard for [`ImportedMcast`]. +/// +/// Mutations record whether or not they added or removed an entry. If an +/// update occurred, then dropping the guard wakes the multicast sweep process. +pub(in crate::db) struct ImportedMcastEdit<'a> { + imported: &'a mut ImportedMcast, + wake: &'a watch::Sender, + changed: bool, +} + +impl ImportedMcastEdit<'_> { + /// Insert or refresh a `route` at `seq` number, reporting whether it added + /// a new `(origin, nexthop, ifname)` entry, changed an existing entry's + /// path or metric, or left it unchanged. + /// + /// Only a new entry wakes the sweep, because multicast membership depends + /// only on which `(nexthop, ifname)` links import a group. + pub(in crate::db) fn insert( + &mut self, + route: &MulticastRoute, + ifname: &str, + seq: u64, + ) -> Inserted { + let nexthops = self.imported.0.entry(route.origin.clone()).or_default(); + let inserted = match find_link(nexthops, route.nexthop, ifname) { + Ok(idx) => { + let entry = &mut nexthops[idx]; + entry.seq = seq; + if entry.route.path != route.path + || entry.route.origin.metric() != route.origin.metric() + { + entry.route = route.clone(); + Inserted::Changed + } else { + Inserted::Unchanged + } + } + Err(idx) => { + // Grow one slot at a time. + nexthops.reserve_exact(1); + nexthops.insert( + idx, + ImportedEntry { + route: route.clone(), + ifname: ifname.to_string(), + seq, + }, + ); + Inserted::New + } + }; + self.changed |= inserted == Inserted::New; + inserted + } + + pub(in crate::db) fn remove( + &mut self, + route: &MulticastRoute, + ifname: &str, + imported_before: Option, + ) -> Option { + let nexthops = self.imported.0.get_mut(&route.origin)?; + let idx = find_link(nexthops, route.nexthop, ifname).ok()?; + if imported_before.is_some_and(|seq| nexthops[idx].seq >= seq) { + return None; + } + let entry = nexthops.remove(idx); + if nexthops.is_empty() { + self.imported.0.remove(&route.origin); + } + self.changed = true; + Some(entry.route) + } + + pub(in crate::db) fn extract_link( + &mut self, + nexthop: Ipv6Addr, + ifname: &str, + ) -> HashSet { + let mut extracted = HashSet::new(); + self.imported.0.retain(|_, nexthops| { + if let Ok(idx) = find_link(nexthops, nexthop, ifname) { + extracted.insert(nexthops.remove(idx).route); + } + !nexthops.is_empty() + }); + self.changed |= !extracted.is_empty(); + extracted + } +} + +impl Drop for ImportedMcastEdit<'_> { + fn drop(&mut self) { + if self.changed { + self.wake.send_modify(|generation| { + *generation = generation.wrapping_add(1); + }); + } + } +} diff --git a/ddm/src/db/mod.rs b/ddm/src/db/mod.rs index 8b7fa85c0..f47c76087 100644 --- a/ddm/src/db/mod.rs +++ b/ddm/src/db/mod.rs @@ -13,10 +13,12 @@ use slog::{Logger, error}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::net::Ipv6Addr; use std::sync::{Arc, Mutex}; +use tokio::sync::watch; pub(crate) mod mcast; -pub use mcast::{ImportedMcast, McastPathSnapshot, MulticastRibChange}; +use mcast::ImportedMcast; +pub use mcast::{McastPathSnapshot, MulticastRibChange, SplicedMulticast}; /// The handle used to open a persistent key-value tree for originated /// prefixes. @@ -45,6 +47,10 @@ pub enum Error { pub struct Db { data: Arc>, persistent_data: sled::Db, + /// A generation counter incremented once per edit of the imported + /// multicast set that adds or removes an `(origin, nexthop, ifname)` + /// entry, waking [`crate::mcast::run`]. + mcast_wake: Arc>, log: Logger, } @@ -54,7 +60,7 @@ pub struct DbData { pub imported_tunnel: HashSet, /// Imported multicast routes, each paired with the value of /// `mcast_seq` at its most recent import. - pub imported_mcast: ImportedMcast, + pub(in crate::db) imported_mcast: ImportedMcast, pub mcast_seq: u64, pub external_peers: BTreeSet, } @@ -69,6 +75,7 @@ impl Db { Ok(Self { data: Arc::new(Mutex::new(DbData::default())), persistent_data: sled::open(db_path)?, + mcast_wake: Arc::new(watch::Sender::new(0)), log, }) } @@ -76,6 +83,11 @@ impl Db { lock!(self.data).clone() } + /// Subscribe to wakes on modifications made to the imported multicast set. + pub fn subscribe_mcast_sweep(&self) -> watch::Receiver { + self.mcast_wake.subscribe() + } + pub fn imported(&self) -> HashSet { lock!(self.data).imported.clone() } @@ -229,18 +241,28 @@ impl Db { } /// Remove all routes via `nexthop`. + /// + /// Underlay and tunnel routes are removed on every interface. Multicast + /// routes are removed only for the specific link `(nexthop, ifname)` entry. pub fn remove_nexthop_routes( &self, nexthop: Ipv6Addr, + ifname: &str, ) -> RemovedNexthopRoutes { - let mut data = lock!(self.data); - let underlay = - data.imported.extract_if(|x| x.nexthop == nexthop).collect(); - let tunnel = data - .imported_tunnel - .extract_if(|x| x.nexthop == nexthop) - .collect(); - let multicast = data.imported_mcast.extract_nexthop(nexthop); + let (underlay, tunnel, multicast) = { + let mut data = lock!(self.data); + let underlay = + data.imported.extract_if(|x| x.nexthop == nexthop).collect(); + let tunnel = data + .imported_tunnel + .extract_if(|x| x.nexthop == nexthop) + .collect(); + let multicast = data + .imported_mcast + .edit(&self.mcast_wake) + .extract_link(nexthop, ifname); + (underlay, tunnel, multicast) + }; RemovedNexthopRoutes { underlay, diff --git a/ddm/src/exchange/mcast.rs b/ddm/src/exchange/mcast.rs index 0cb9b6a7b..93de390fc 100644 --- a/ddm/src/exchange/mcast.rs +++ b/ddm/src/exchange/mcast.rs @@ -151,7 +151,8 @@ mod tests { if !originated.is_empty() { db.originate_mcast(&originated).unwrap(); } - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &imported, &HashSet::new(), )); @@ -438,7 +439,8 @@ mod tests { let still_imported = origin("233.252.0.1", "ff04::1", 10); let local_origin = origin("233.252.0.2", "ff04::2", 10); let remaining_hop = hop("remaining", 2); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([MulticastRoute { origin: still_imported.clone(), nexthop: "fe80::2".parse().unwrap(), @@ -495,7 +497,8 @@ mod tests { let other_hop = hop("other", 3); let local_hop = hop("local", 2); let withdrawn_hop = hop("withdrawn", 4); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([ MulticastRoute { origin: alternate.clone(), diff --git a/ddm/src/exchange/runtime/mcast.rs b/ddm/src/exchange/runtime/mcast.rs index f34210363..1538e25a8 100644 --- a/ddm/src/exchange/runtime/mcast.rs +++ b/ddm/src/exchange/runtime/mcast.rs @@ -144,15 +144,16 @@ pub(super) fn apply_multicast_update( imported_before: Option, ) -> AppliedMulticastUpdate { let batch = MulticastRibChange { + ifname: &ctx.config.if_name, import: &import, remove, imported_before, }; - let removed = ctx.db.update_imported_mcast(batch); + let spliced = ctx.db.splice_imported_mcast(batch); AppliedMulticastUpdate { - imported: import, - removed, + imported: spliced.imported, + removed: spliced.removed, hop: MulticastPathHop::new(ctx.router_id.clone(), ctx.config.addr), } } @@ -229,8 +230,8 @@ pub(super) fn partition_multicast_update( path: Vec::new(), }); } - // A simultaneous import would receive a sequence number above the pull - // cutoff. + // A simultaneous import would receive a sequence number at or above + // the pull sequence. import.retain(|route| !remove.contains(route)); (import, remove) @@ -359,17 +360,21 @@ mod tests { let log = Logger::root(Discard, o!()); let db = Db::new(dir.path().as_str(), log).unwrap(); let existing = HashSet::from([route(1)]); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &existing, &HashSet::new(), )); let (import, remove) = partition(&update); - let removed = - db.update_imported_mcast(MulticastRibChange::new(&import, &remove)); + let removed = db + .splice_imported_mcast(MulticastRibChange::new( + "test0", &import, &remove, + )) + .removed; assert_eq!(removed, existing); - assert!(db.imported_mcast().is_empty()); + assert!(db.imported_mcast_links().is_empty()); assert!(import.is_empty()); } diff --git a/ddm/src/exchange/runtime/mod.rs b/ddm/src/exchange/runtime/mod.rs index af0264259..d85f40d59 100644 --- a/ddm/src/exchange/runtime/mod.rs +++ b/ddm/src/exchange/runtime/mod.rs @@ -346,8 +346,9 @@ fn fetch_pull_body( /// Withdrawals are synthesized only after the complete snapshot has been /// imported. A failed pull imports nothing and synthesizes no withdrawals. /// -/// `started_at` is the cutoff for this pull. A route updated during the read -/// stays out of stale-route removal, including a refresh after the scan. +/// `pull_seq` is recorded before reading routes from the peer. A route +/// updated during the read stays out of stale-route removal, including a +/// refresh after the scan. /// /// Underlay and tunnel withdrawals remain push-only at every version. pub(crate) fn pull( @@ -359,7 +360,7 @@ pub(crate) fn pull( mode: UpdateMode, ) -> Result<(), ExchangeError> { let version = session.version; - let started_at = ctx.db.next_mcast_seq(); + let pull_seq = ctx.db.next_mcast_seq(); if stop.load(Ordering::Relaxed) { return Ok(()); @@ -390,7 +391,7 @@ pub(crate) fn pull( let withdraw: HashSet = ctx .db - .stale_mcast_by_nexthop(addr, started_at, &announced) + .stale_mcast_by_nexthop(addr, &ctx.config.if_name, pull_seq, &announced) .into_iter() .map(|origin| MulticastPathVector { origin, @@ -410,7 +411,7 @@ pub(crate) fn pull( ctx, addr, mode, - Some(started_at), + Some(pull_seq), session, ); } @@ -877,9 +878,9 @@ fn handle_update( if session.is_canceled() { redistribute = redistribute.withdrawals_only(); - if redistribute.is_empty() { - return; - } + } + if redistribute.is_empty() { + return; } if let Err(e) = ctx.tx.send(Event::Peer(PeerEvent::Redistribute { @@ -1121,8 +1122,8 @@ mod tests { use crate::defaults::EXCHANGE_TIMEOUT; use crate::discovery::Version; use crate::sm::{ - Config, Event, ExchangeSession, InterfaceState, PeerIdentity, - SessionStats, SmContext, + Config, Event, ExchangeSession, InterfaceState, PeerEvent, + PeerIdentity, SessionStats, SmContext, }; use ddm_api_types::db::{MulticastRoute, RouterKind}; use ddm_api_types::net::{OverlayMulticast, UnderlayMulticastIpv6, Vni}; @@ -1274,7 +1275,7 @@ mod tests { struct ExchangeFixture { ctx: SmContext, - _events: Receiver, + events: Receiver, _dir: camino_tempfile::Utf8TempDir, } @@ -1319,7 +1320,7 @@ mod tests { discovery_stop: None, first_run: true, }, - _events: events, + events, _dir: dir, } } @@ -1416,6 +1417,68 @@ mod tests { }); } + fn redistributed_multicast(events: &Receiver) -> Vec { + events + .try_iter() + .filter_map(|event| match event { + Event::Peer(PeerEvent::Redistribute { update, .. }) => Some( + update.multicast.as_ref().map_or(0, |m| m.announce.len()), + ), + _ => None, + }) + .collect() + } + + #[test] + fn transit_redistributes_only_changed_multicast() { + let rt = Runtime::new().unwrap(); + let _guard = rt.enter(); + let mut fixture = ExchangeFixture::new(&rt); + fixture.ctx.config.kind = RouterKind::Transit; + let session = publish(&fixture.ctx, Version::V4); + let server = fixture.server(session); + let (update, route) = multicast_announcement(); + + push_v4(&rt, &server, &update); + push_v4(&rt, &server, &update); + assert_eq!(redistributed_multicast(&fixture.events), vec![1]); + + let rerouted: v4::Update = + v4::MulticastUpdate::announce(HashSet::from([ + v4::MulticastPathVector { + origin: route.origin, + path: vec![ + v4::MulticastPathHop::new("peer".into(), PEER), + v4::MulticastPathHop::new("far".into(), PEER), + ], + }, + ])) + .into(); + push_v4(&rt, &server, &rerouted); + push_v4(&rt, &server, &rerouted); + assert_eq!(redistributed_multicast(&fixture.events), vec![1]); + + let missing = v4::MulticastOrigin::new( + OverlayMulticast::new("233.252.0.9".parse().unwrap()).unwrap(), + UnderlayMulticastIpv6::new("ff04::109".parse().unwrap()).unwrap(), + Vni::new(77).unwrap(), + None, + ) + .unwrap(); + let withdraw: v4::Update = + v4::MulticastUpdate::withdraw(HashSet::from([ + v4::MulticastPathVector { + origin: missing, + path: Vec::new(), + }, + ])) + .into(); + push_v4(&rt, &server, &withdraw); + assert!(redistributed_multicast(&fixture.events).is_empty()); + + rt.block_on(server.teardown()); + } + #[test] fn stale_push_cannot_restore_multicast_after_session_replacement() { let rt = Runtime::new().unwrap(); @@ -1426,26 +1489,29 @@ mod tests { let old_server = fixture.server(old_session); let (update, route) = multicast_announcement(); + let imported = vec![(route.clone(), ctx.config.if_name.clone())]; push_v4(&rt, &old_server, &update); - assert_eq!(ctx.db.imported_mcast(), HashSet::from([route.clone()])); + assert_eq!(ctx.db.imported_mcast_links(), imported); { let mut session = ctx.iface.session(); *session = Some(Arc::new(ExchangeSession::new(Version::V3))); - let removed = ctx.db.remove_nexthop_mcast_routes(PEER); + let removed = ctx + .db + .remove_nexthop_mcast_routes(PEER, &ctx.config.if_name); assert_eq!(removed.multicast, HashSet::from([route.clone()])); } - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); push_v4(&rt, &old_server, &update); - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); let fresh_session = publish(ctx, Version::V4); push_v4(&rt, &old_server, &update); - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); let fresh_server = fixture.server(fresh_session); push_v4(&rt, &fresh_server, &update); - assert_eq!(ctx.db.imported_mcast(), HashSet::from([route])); + assert_eq!(ctx.db.imported_mcast_links(), imported); rt.block_on(fresh_server.teardown()); rt.block_on(old_server.teardown()); @@ -1474,7 +1540,7 @@ mod tests { .unwrap() }); assert_eq!(error.message, "multicast exchange requires a V4 peer"); - assert!(fixture.ctx.db.imported_mcast().is_empty()); + assert!(fixture.ctx.db.imported_mcast_links().is_empty()); rt.block_on(server.teardown()); } @@ -1489,7 +1555,7 @@ mod tests { let server = fixture.server(session); push_v4(&rt, &server, &v4::Update::default()); - assert!(fixture.ctx.db.imported_mcast().is_empty()); + assert!(fixture.ctx.db.imported_mcast_links().is_empty()); rt.block_on(server.teardown()); } @@ -1506,7 +1572,7 @@ mod tests { session.cancel(); push_v4(&rt, &server, &update); - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); rt.block_on(server.teardown()); } diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index 19326ec0a..a1421a583 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -7,6 +7,7 @@ pub mod db; pub mod defaults; pub mod discovery; pub mod exchange; +pub mod mcast; pub mod oxstats; pub mod sm; #[cfg(all(feature = "backend", target_os = "illumos"))] @@ -15,6 +16,7 @@ pub mod sys; pub const COMPONENT_DDM: &str = "ddm"; pub const MOD_ADMIN: &str = "admin"; pub const MOD_EXCHANGE: &str = "exchange"; +pub(crate) const DDM_TAG: &str = "ddmd"; #[macro_export] macro_rules! err { diff --git a/ddm/src/mcast.rs b/ddm/src/mcast.rs new file mode 100644 index 000000000..c35c6a822 --- /dev/null +++ b/ddm/src/mcast.rs @@ -0,0 +1,1545 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Underlay multicast membership programming. +//! +//! Programs multicast routes DDM has imported from peers into the local +//! switch's underlay multicast replication member set operated by the +//! Dendrite (DPD) dataplane. This is the multicast analog of +//! unicast's import-to-DPD setup which `add_underlay_routes` performs +//! in-process. With `ddmd` owning its input locally, i.e., the imported +//! multicast set along with the interface each route's peer was discovered +//! on ([`imported_mcast_links`]), it programs DPD directly. +//! +//! ## Membership Reconciliation +//! +//! Membership reconciliation runs by way of a periodic sweep over all active +//! underlay groups, similar to how the unicast lower-half's resync works. When +//! updates are made to the imported multicast set, whether from the exchange +//! update handler, peer expiry, or protocol downgrade, they all operate through +//! [`Db`], which wakes the sweep mechanism early (see +//! [`Db::subscribe_mcast_sweep`]). Otherwise, the sweep waits out +//! `RECONCILE_INTERVAL` after each pass before starting the next one. +//! +//! On each pass, we recompute the wanted member set for every group containing +//! imports or programmed members (withdrawn groups included), then repair +//! drift, drain withdrawn groups, and program members we could not +//! previously resolve (e.g., the group hadn't existed yet or an interface had +//! not resolved to a switch link). +//! +//! The sweep removes only members recorded as programmed since this `ddmd` +//! process started up, akin to ddmd's handling of unicast and tunnel routes. +//! A matching read or a successful write records the members derived from +//! imports, including those already in DPD. A failed write retains the prior +//! record and adds the wanted members because DPD may have already applied it. +//! Programmed members are removed when no import can derive them, and a group +//! stays in the sweep until it has no imports or programmed members. +//! +//! A member no import has derived since this process started, such as one left +//! by an earlier `ddmd` process, remains in place. +//! +//! DPD replaces the full member set on each write. Unicast and tunnel updates +//! can add or remove entries without reading back the full set. However, inline +//! multicast updates would need serialization per group along with a retry +//! mechanism. The sweep provides both: a single task reconciling groups one at +//! a time, and subsequent passes retrying failures. Each pass consolidates +//! route updates into at most one read-modify-write per group, avoiding +//! unnecessary writes when the computed member set matches DPD. +//! +//! ## DPD Groups and Membership +//! +//! Each imported [`MulticastRoute`] identifies an advertising peer (`nexthop`), +//! and is kept with the interface that peer was discovered on, akin to how +//! unicast routes work. This interface supplies the switch `(PortId, LinkId)` +//! tuple entry, while resolved members are deduplicated per underlay group +//! address (which is DPD's group key); the overlay group is not part of this +//! key. +//! +//! Nexus creates each underlay group and deletes it by tag. Within an existing +//! group, `ddmd` writes the members whose packet copies keep their Geneve +//! encapsulation (`Underlay` in DPD) and preserves the members whose copies +//! are decapsulated before delivery (`External`). +//! Rack-originated traffic already carries Geneve encapsulation addressed to +//! the underlay group and bypasses ingress NAT ([DPD packet paths]). +//! +//! Writes pass back the tag from the same pass's read, which the +//! [DPD update API] checks against the group's tag. So `ddmd` only ever +//! replaces the member list (never the tag, never the group itself). +//! +//! See [RFD 488] for the multicast architecture. +//! +//! [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +//! [`imported_mcast_links`]: crate::db::Db::imported_mcast_links +//! [DPD packet paths]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/dpd/src/mcast/mod.rs#L22-L25 +//! [DPD update API]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/dpd-api/src/lib.rs#L2496-L2512 + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::time::Duration; + +use ddm_api_types::db::MulticastRoute; +use ddm_api_types::net::UnderlayMulticastIpv6; +use dpd_client::types::{ + Direction, MulticastGroupMember, MulticastGroupUnderlayResponse, + MulticastGroupUpdateUnderlayEntry, PortId, +}; +use dpd_client::{Client, ClientState}; +use dpd_types_versions::latest::mcast::MulticastTag; +use hyper::StatusCode; +use slog::{Logger, debug, error, info, warn}; +use tokio::sync::watch; + +use crate::DDM_TAG; +use crate::db::Db; +use crate::sm::DpdConfig; + +/// The interval between the sweep's periodic membership reconciliation passes. +const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); + +/// Run the multicast membership reconciliation sweep until it's explicitly +/// canceled. +/// +/// This reconciles imported and draining groups on wakes from +/// [`Db::subscribe_mcast_sweep`] or via periodic ticks. Switch links are +/// resolved on each pass from the interface each route's peer was discovered +/// on, including dynamically added external peers. +/// +/// Each new `ddmd` process starts without a record of programmed membership, +/// which means that existing members it never derives again could remain +/// in DPD indefinitely. +/// +/// TODO: Recover forwarding state after restart across multicast, unicast, +/// and tunnel routes. We retain existing behavior here: [unicast and +/// tunnel cleanup] also uses only the current imported set. +/// +/// TODO: Count sweep outcomes (passes, writes, failures, missing groups) in +/// [`RouterStats`] once the `ddm-router.toml` schema can export them. +/// +/// [`RouterStats`]: crate::admin::RouterStats +/// [unicast and tunnel cleanup]: https://github.com/oxidecomputer/maghemite/blob/04a64f5b6b80dadbfa6ece123a30304b2f82414e/ddm/src/sm/state.rs#L357-L378 +pub async fn run(db: Db, dpd: DpdConfig, log: Logger) { + let mcast = db.subscribe_mcast_sweep(); + let client = Client::new( + &format!("http://{}:{}", dpd.host, dpd.port), + ClientState { + tag: DDM_TAG.into(), + log: log.clone(), + }, + ); + run_with_client(db, &client, mcast, log).await; +} + +async fn run_with_client( + db: Db, + client: &C, + mut mcast: watch::Receiver, + log: Logger, +) { + let mut programmed: ProgrammedMembers = HashMap::new(); + let mut unresolved = HashSet::new(); + + loop { + // Before reading the imported set, we mark the current generation + // number as seen. If an update advances the generation during this + // pass, another pass starts without waiting for the timer. + mcast.borrow_and_update(); + + // For each group within a pass, the imported set is the same snapshot, + // which means we can read it once here, avoiding a per-group lookup. + let imported = db.imported_mcast_links(); + + reconcile_pass( + &mut programmed, + &mut unresolved, + &imported, + client, + &log, + ) + .await; + + // Wait for a wake cycle or one idle reconcile interval. + // + // Both waits (below) are cancel-safe. + // + // If an update races with selection of the sleep branch, the next + // pass will mark it seen and read the set that includes it. + tokio::select! { + Ok(()) = mcast.changed() => {} + _ = tokio::time::sleep(RECONCILE_INTERVAL) => {} + } + } +} + +/// The per-group `Underlay` members programmed since this sweep started. +type ProgrammedMembers = HashMap; +type Members = HashSet; + +/// Reconcile active groups with imports or programmed members. +/// +/// An empty result drops the group's entry, recorded nothing for it. +/// A group that still contains imports (e.g., a group Nexus has not yet +/// created) is picked back up on the next pass via those imports. +async fn reconcile_pass( + programmed: &mut ProgrammedMembers, + unresolved: &mut HashSet, + imported: &[(MulticastRoute, String)], + client: &C, + log: &Logger, +) { + let mut by_group: HashMap> = + HashMap::new(); + for (route, ifname) in imported { + by_group + .entry(route.origin.underlay_group()) + .or_default() + .push(ifname); + } + let groups: BTreeSet = + by_group.keys().chain(programmed.keys()).copied().collect(); + let links = resolve_links( + imported.iter().map(|(_, ifname)| ifname.as_str()), + unresolved, + log, + ); + + for group_ip in groups { + let ifnames = by_group.get(&group_ip).map_or(&[][..], Vec::as_slice); + let prev = programmed.remove(&group_ip).unwrap_or_default(); + let members = reconcile_group( + group_ip, + ifnames.iter().copied(), + &links, + prev, + client, + log, + ) + .await; + + if !members.is_empty() { + programmed.insert(group_ip, members); + } + } +} + +/// The outcome of writing a group's member list to DPD. +#[derive(Clone, Copy)] +enum WriteOutcome { + /// Members were written. + Updated, + /// The write failed and should be retried on the next pass. + Retry, +} + +/// The outcome of reading a group's state from DPD. +#[derive(Clone)] +enum FetchOutcome { + /// The fetched group exists, and its authorization tag and current members + /// were read without issue. + Found(MulticastTag, Vec), + /// The fetched group does not exist in DPD, maybe because Nexus has not + /// created it yet or because it was deleted before this operation. + Missing, + /// The read operation failed, making the group's state unknown on this + /// pass, for which a retry should ensue. + Retry, +} + +/// Behavior encompassing DPD group operations that the reconciliation relies +/// on. +/// +/// NOTE: tests implement this trait in order to inject failures and pause +/// requests. +trait GroupClient { + /// Read an underlay group's current members along with its authorization + /// tag. + async fn fetch_group( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + ) -> FetchOutcome; + + /// Write `members` to an underlay group, authorized by its current `tag`. + async fn write_members( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + tag: &MulticastTag, + members: Vec, + ) -> WriteOutcome; +} + +impl GroupClient for Client { + /// This distinguishes a group that is genuinely missing + /// (`FetchOutcome::Missing`) from one with state that could not be read + /// (`FetchOutcome::Retry`) at the time of execution. + /// + /// Therefore, a withdrawn group is not dropped from the sweep when + /// encountering a transient read failure before its members are confirmed + /// drained. + async fn fetch_group( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + ) -> FetchOutcome { + match self.multicast_group_get_underlay(&group_ip).await { + Ok(resp) => { + let MulticastGroupUnderlayResponse { tag, members, .. } = + resp.into_inner(); + MulticastTag::try_from(tag) + .map(|tag| FetchOutcome::Found(tag, members)) + .unwrap_or_else(|e| { + error!( + log, + "tag for underlay group {group_ip} is invalid, \ + retrying next pass: {e}" + ); + FetchOutcome::Retry + }) + } + // The creation and deletion of an underlay group is owned by Nexus, + // which creates the group before traffic flows toward it. + // + // We bypass membership programming until the group exists. + Err(e) if e.status() == Some(StatusCode::NOT_FOUND) => { + debug!( + log, + "underlay group {group_ip} does not exist yet, bypassing \ + until Nexus creates it" + ); + FetchOutcome::Missing + } + Err(e) => { + warn!(log, "failed to get underlay group {group_ip}: {e}"); + FetchOutcome::Retry + } + } + } + + /// Any failure, including a 404 or a 400 for a retag since the read, is + /// [`WriteOutcome::Retry`]. DPD also returns 404 for a missing ASIC + /// resource on a group that still exists, so the next pass's read decides + /// whether the group is gone. + async fn write_members( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + tag: &MulticastTag, + members: Vec, + ) -> WriteOutcome { + let body = MulticastGroupUpdateUnderlayEntry { members }; + match self + .multicast_group_update_underlay(&group_ip, tag, &body) + .await + { + Ok(_) => WriteOutcome::Updated, + Err(dpd_client::Error::ErrorResponse(resp)) + if resp.error_code.as_deref() + == Some(dpd_client::MCAST_TAG_MISMATCH_ERROR_CODE) => + { + debug!( + log, + "underlay group {group_ip} tag changed since the read, \ + retrying next pass" + ); + WriteOutcome::Retry + } + Err(e) => { + warn!( + log, + "failed to update underlay group {group_ip} members: {e}" + ); + WriteOutcome::Retry + } + } + } +} + +/// Aggregate one underlay group's wanted replication members. +/// +/// `ifnames` are the interfaces on which the peers behind an underlay group's +/// imports were discovered. +/// Interfaces that don't resolve to a switch link during this pass do not +/// contribute toward membership. +/// +/// Members are derived from each import's interface. All distinct downstream +/// links convey distinct interfaces, making each of these its own member. +/// Subscribers reached through a duplicate downstream link collapse into one +/// member because a single egress port toward that peer is sufficient enough +/// and the next set of hop handlers all fan-out further. +fn group_members<'a>( + ifnames: impl IntoIterator, + links: &HashMap<&str, MulticastGroupMember>, +) -> Members { + ifnames + .into_iter() + .filter_map(|ifname| links.get(ifname).cloned()) + .collect() +} + +/// Resolve each distinct interface in `imported` to an `Underlay` member once +/// per pass. +/// +/// Interfaces that cannot resolve to a switch link, or that resolve to a +/// non-rear port, are not retained. +fn resolve_links<'a>( + imported: impl IntoIterator, + unresolved: &mut HashSet, + log: &Logger, +) -> HashMap<&'a str, MulticastGroupMember> { + let ifnames: BTreeSet<&str> = imported.into_iter().collect(); + unresolved.retain(|ifname| ifnames.contains(ifname.as_str())); + + ifnames + .into_iter() + .filter_map(|ifname| { + let (port_id, link_id) = + mg_common::tfport::port_link_from_untagged_ifname(ifname) + .inspect_err(|e| { + if !unresolved.contains(ifname) { + unresolved.insert(ifname.to_string()); + warn!( + log, + "cannot resolve interface {ifname} to a \ + switch link, omitting as multicast \ + member: {e}" + ) + } + }) + .ok()?; + + // TODO: Allow QSFP members once inter-rack replication has loop + // prevention (e.g., RPF or a designated forwarder). + if !matches!(port_id, PortId::Rear(_)) { + if !unresolved.contains(ifname) { + unresolved.insert(ifname.to_string()); + debug!( + log, + "skipping multicast member on {ifname} ({port_id}), \ + cross-rack replication is not supported" + ); + } + return None; + } + unresolved.remove(ifname); + Some(( + ifname, + MulticastGroupMember { + port_id, + link_id, + direction: Direction::Underlay, + }, + )) + }) + .collect() +} + +/// Reconcile an underlay group's members in DPD against the imported multicast +/// routes DDM contains, returning the members to record as programmed. +/// +/// With wanted members `D`, existing DPD members `E`, and previously +/// programmed members `P`, the write consists of `D ∪ (E \ P)`. +/// +/// Existing members outside of `P` are preserved, including those left over +/// from an earlier run. When no longer wanted, members in `P` are removed. +/// +/// # Notes +/// +/// - A matching read or a successful write records `D`, including any existing +/// member it must now account for. +/// - A failed read preserves `P`, as the next pass may need to remove +/// members no longer wanted. +/// - A failed write records `P ∪ D`, as DPD may have already applied the +/// operation. +/// - A missing group records nothing whatsoever. +async fn reconcile_group<'a, C: GroupClient>( + group_ip: UnderlayMulticastIpv6, + ifnames: impl IntoIterator, + links: &HashMap<&str, MulticastGroupMember>, + programmed: Members, + client: &C, + log: &Logger, +) -> Members { + let wanted = group_members(ifnames, links); + + let (tag, existing) = match client.fetch_group(log, group_ip).await { + FetchOutcome::Found(tag, existing) => { + (tag, existing.into_iter().collect::()) + } + // The group is missing from DPD -> Nexus has not created it or it has + // been deleted, holding 0 members. + // + // We keep the group while it still has imports, giving way for a later + // pass to program it once it exists. + FetchOutcome::Missing => return Members::new(), + // The read state is unknown at this point. + // + // We keep the programmed members around for the next pass to remove + // what was programmed. + FetchOutcome::Retry => return programmed, + }; + + let mut to_write = &existing - &programmed; + to_write.extend(wanted.iter().cloned()); + if to_write == existing { + return wanted; + } + + let members = to_write.into_iter().collect(); + match client.write_members(log, group_ip, &tag, members).await { + WriteOutcome::Updated => { + info!(log, "updated underlay group {group_ip} members"); + wanted + } + WriteOutcome::Retry => &programmed | &wanted, + } +} + +#[cfg(test)] +mod tests { + use std::convert::Infallible; + use std::net::{IpAddr, Ipv6Addr}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use ddm_api_types::net::{MulticastOrigin, OverlayMulticast, Vni}; + use dpd_client::types::LinkId; + use http_body_util::{BodyExt, Full}; + use hyper::body::{Bytes, Incoming}; + use hyper::server::conn::http1; + use hyper::service::service_fn; + use hyper::{Method, Request, Response}; + use hyper_util::rt::TokioIo; + use slog::Drain; + use tokio::net::TcpListener; + use tokio::sync::{Notify, oneshot}; + + use super::*; + + fn underlay(last: u16) -> UnderlayMulticastIpv6 { + UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0, last, + )) + .unwrap() + } + + fn route( + nexthop: Ipv6Addr, + group: UnderlayMulticastIpv6, + ) -> MulticastRoute { + MulticastRoute { + origin: MulticastOrigin::new( + OverlayMulticast::new(IpAddr::V6(Ipv6Addr::new( + 0xff0e, 0, 0, 0, 0, 0, 0, 1, + ))) + .unwrap(), + group, + Vni::new(77).unwrap(), + None, + ) + .unwrap(), + nexthop, + path: Vec::new(), + } + } + + fn learned( + imported: &HashSet, + links: &HashMap, + ) -> Vec<(MulticastRoute, String)> { + imported + .iter() + .map(|route| { + let ifname = links.get(&route.nexthop).map_or_else( + || "vioif0".to_string(), + |(port_id, link_id)| { + format!("tfport{port_id}_{}", link_id.0) + }, + ); + (route.clone(), ifname) + }) + .collect() + } + + fn group_ifnames( + group: UnderlayMulticastIpv6, + learned: &[(MulticastRoute, String)], + ) -> impl Iterator { + learned + .iter() + .filter(move |(route, _)| route.origin.underlay_group() == group) + .map(|(_, ifname)| ifname.as_str()) + } + + fn resolved_members( + group: UnderlayMulticastIpv6, + imported: &HashSet, + links: &HashMap, + ) -> Members { + let learned = learned(imported, links); + let log = Logger::root(slog::Discard, slog::o!()); + group_members( + group_ifnames(group, &learned), + &resolve_links( + learned.iter().map(|(_, ifname)| ifname.as_str()), + &mut HashSet::new(), + &log, + ), + ) + } + + fn rear(port: &str, link: u8) -> (PortId, LinkId) { + ( + PortId::Rear(dpd_client::types::Rear::try_from(port).unwrap()), + LinkId(link), + ) + } + + fn member(port: &str, link: u8) -> MulticastGroupMember { + let (port_id, link_id) = rear(port, link); + MulticastGroupMember { + port_id, + link_id, + direction: Direction::Underlay, + } + } + + #[test] + fn unresolved_interfaces_warn_once_until_removed() { + let warnings = Arc::new(AtomicUsize::new(0)); + let skips = Arc::new(AtomicUsize::new(0)); + let (warned, skipped) = (warnings.clone(), skips.clone()); + let drain = slog::Filter::new(slog::Discard, move |record| { + match record.level() { + slog::Level::Warning => warned.fetch_add(1, Ordering::Relaxed), + slog::Level::Debug => skipped.fetch_add(1, Ordering::Relaxed), + _ => 0, + }; + true + }); + let log = Logger::root(drain.ignore_res(), slog::o!()); + let mut unresolved = HashSet::new(); + + let links = resolve_links( + ["vioif0", "vioif0", "bad0", "tfportqsfp0_0", "tfportrear0_0"], + &mut unresolved, + &log, + ); + assert_eq!( + links, + HashMap::from([("tfportrear0_0", member("rear0", 0))]), + ); + assert_eq!(warnings.load(Ordering::Relaxed), 2); + assert_eq!(skips.load(Ordering::Relaxed), 1); + assert_eq!( + unresolved, + HashSet::from([ + "vioif0".to_string(), + "bad0".to_string(), + "tfportqsfp0_0".to_string(), + ]), + ); + + resolve_links( + ["vioif0", "bad0", "tfportqsfp0_0", "tfportrear0_0"], + &mut unresolved, + &log, + ); + assert_eq!(warnings.load(Ordering::Relaxed), 2); + assert_eq!(skips.load(Ordering::Relaxed), 1); + + resolve_links(["bad0"], &mut unresolved, &log); + assert_eq!(unresolved, HashSet::from(["bad0".to_string()])); + assert_eq!(warnings.load(Ordering::Relaxed), 2); + + resolve_links( + ["vioif0", "bad0", "tfportqsfp0_0"], + &mut unresolved, + &log, + ); + assert_eq!(warnings.load(Ordering::Relaxed), 3); + assert_eq!(skips.load(Ordering::Relaxed), 2); + + resolve_links([], &mut unresolved, &log); + assert!(unresolved.is_empty()); + } + + #[test] + fn membership_follows_links() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let both = vec![ + (route(peer, group), "tfportrear0_0".to_string()), + (route(peer, group), "tfportrear1_0".to_string()), + ]; + let one = vec![(route(peer, group), "tfportrear1_0".to_string())]; + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + let mut programmed = HashMap::new(); + let mut unresolved = HashSet::new(); + rt.block_on(reconcile_pass( + &mut programmed, + &mut unresolved, + &both, + &mock, + &log, + )); + assert_eq!( + programmed, + HashMap::from([( + group, + set(&[member("rear0", 0), member("rear1", 0)]) + )]) + ); + + rt.block_on(reconcile_pass( + &mut programmed, + &mut unresolved, + &one, + &mock, + &log, + )); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear1", 0)]))]) + ); + let writes = mock.writes(); + assert_eq!(writes.len(), 2); + assert_eq!( + set(&writes[0]), + set(&[member("rear0", 0), member("rear1", 0)]) + ); + assert_eq!(writes[1], vec![member("rear1", 0)]); + } + + #[test] + fn same_link_peers_share_membership() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + // Two distinct peers that resolve to the same switch link. Replicating + // twice out one (PortId, LinkId) would duplicate delivery on that link, + // so the members collapse to one. The dedup keys on the resolved link, + // not on the next hop. + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear0", 0)), + ]); + + let members = resolved_members(group, &imported, &peer_links); + assert_eq!(members, set(&[member("rear0", 0)])); + } + + #[test] + fn unresolved_interfaces_are_skipped() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(3); + + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + let members = resolved_members(group, &imported, &peer_links); + assert_eq!(members, set(&[member("rear0", 0)])); + } + + /// The sender signals that a fetch started, and the receiver waits for the + /// test to resume it. + type FetchChannels = (oneshot::Sender<()>, oneshot::Receiver<()>); + + /// Records write operations and makes successful updates visible to + /// fetches. + struct MockDpd { + fetch: std::sync::Mutex, + fetch_by_group: + std::sync::Mutex>, + write_outcome: WriteOutcome, + writes: std::sync::Mutex< + Vec<(UnderlayMulticastIpv6, Vec)>, + >, + fetch_channels: std::sync::Mutex>, + fetch_result_tx: + std::sync::Mutex>>, + write_attempted: Notify, + } + + impl MockDpd { + fn new(fetch: FetchOutcome, write_outcome: WriteOutcome) -> Self { + Self { + fetch: std::sync::Mutex::new(fetch), + fetch_by_group: std::sync::Mutex::new(HashMap::new()), + write_outcome, + writes: std::sync::Mutex::new(Vec::new()), + fetch_channels: std::sync::Mutex::new(None), + fetch_result_tx: std::sync::Mutex::new(None), + write_attempted: Notify::new(), + } + } + + fn writes(&self) -> Vec> { + self.group_writes() + .into_iter() + .map(|(_, members)| members) + .collect() + } + + fn group_writes( + &self, + ) -> Vec<(UnderlayMulticastIpv6, Vec)> { + self.writes.lock().unwrap().clone() + } + } + + impl GroupClient for MockDpd { + async fn fetch_group( + &self, + _log: &Logger, + group_ip: UnderlayMulticastIpv6, + ) -> FetchOutcome { + let channels = self.fetch_channels.lock().unwrap().take(); + if let Some((started, resume)) = channels { + started.send(()).unwrap(); + resume.await.unwrap(); + } + let outcome = + self.fetch_by_group.lock().unwrap().get(&group_ip).cloned(); + let outcome = + outcome.unwrap_or_else(|| self.fetch.lock().unwrap().clone()); + if let Some(tx) = self.fetch_result_tx.lock().unwrap().take() { + assert!(tx.send(outcome.clone()).is_ok()); + } + outcome + } + + async fn write_members( + &self, + _log: &Logger, + group_ip: UnderlayMulticastIpv6, + tag: &MulticastTag, + members: Vec, + ) -> WriteOutcome { + self.writes + .lock() + .unwrap() + .push((group_ip, members.clone())); + + if matches!(self.write_outcome, WriteOutcome::Updated) { + let updated = FetchOutcome::Found(tag.clone(), members); + let overridden = { + let mut by_group = self.fetch_by_group.lock().unwrap(); + match by_group.get_mut(&group_ip) { + Some(outcome) => { + *outcome = updated.clone(); + true + } + None => false, + } + }; + if !overridden { + *self.fetch.lock().unwrap() = updated; + } + } + self.write_attempted.notify_one(); + self.write_outcome + } + } + + fn found(members: Vec) -> FetchOutcome { + FetchOutcome::Found(MulticastTag::try_from("tag").unwrap(), members) + } + + fn reconcile( + group: UnderlayMulticastIpv6, + imported: &HashSet, + peer_links: &HashMap, + programmed: Members, + mock: &MockDpd, + ) -> Members { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let learned = learned(imported, peer_links); + + rt.block_on(reconcile_group( + group, + group_ifnames(group, &learned), + &resolve_links( + learned.iter().map(|(_, ifname)| ifname.as_str()), + &mut HashSet::new(), + &log, + ), + programmed, + mock, + &log, + )) + } + + fn run_sweep( + mut programmed: ProgrammedMembers, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> ProgrammedMembers { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(reconcile_pass( + &mut programmed, + &mut HashSet::new(), + &learned(imported, peer_links), + mock, + &log, + )); + programmed + } + + fn set(members: &[MulticastGroupMember]) -> Members { + members.iter().cloned().collect() + } + + #[tokio::test(start_paused = true)] + async fn mid_sweep_change_wakes_next_sweep() { + let dir = camino_tempfile::Utf8TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().as_str(), log.clone()).unwrap(); + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + db.splice_imported_mcast(crate::db::MulticastRibChange::new( + "tfportrear0_0", + &imported, + &HashSet::new(), + )); + + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + let (started_tx, started_rx) = oneshot::channel(); + let (resume_tx, resume_rx) = oneshot::channel(); + *mock.fetch_channels.lock().unwrap() = Some((started_tx, resume_rx)); + + let sweep = + run_with_client(db.clone(), &mock, db.subscribe_mcast_sweep(), log); + let check = async { + tokio::time::timeout(RECONCILE_INTERVAL / 2, started_rx) + .await + .expect("first sweep did not reach the DPD fetch") + .unwrap(); + db.remove_nexthop_mcast_routes(peer, "tfportrear0_0"); + resume_tx.send(()).unwrap(); + + tokio::time::timeout( + RECONCILE_INTERVAL / 2, + mock.write_attempted.notified(), + ) + .await + .expect("withdrawal during sweep did not wake the next sweep"); + assert_eq!(mock.writes(), vec![Vec::::new()]); + }; + tokio::select! { + _ = sweep => panic!("sweep exited"), + _ = check => {} + } + } + + #[test] + fn group_drains_and_returns_across_sweeps() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let active = HashSet::from([route(peer, group)]); + let withdrawn = HashSet::new(); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // Sweep 1: imported and already in sync, the sweep counts the + // existing member as having been programmed (without an explicit + // write). + let programmed = run_sweep(HashMap::new(), &active, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert!(mock.writes().is_empty()); + + // Sweep 2: the withdrawn group carries over from sweep 1, its + // programmed member is removed, and then it drops out of the sweep. + let programmed = run_sweep(programmed, &withdrawn, &peer_links, &mock); + assert!(programmed.is_empty()); + assert_eq!(mock.writes(), vec![Vec::::new()]); + + // Sweep 3: the dropped group is re-imported and reconfigured (the group + // contains no membership in DPD). + let programmed = run_sweep(programmed, &active, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert_eq!(mock.writes(), vec![Vec::new(), vec![member("rear0", 0)]]); + } + + /// Case: after a restart, the sweep acts as a no-op for members it has not + /// yet programmed. + /// + /// - A leftover member by the previous `ddmd` process is not removed, akin + /// to stale unicast routes. + /// - A leftover member that an import re-derives is counted as programmed + /// and is removed once that import is finally withdrawn. + #[test] + fn a_restart_keeps_leftover_members() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear1", 0))]); + let mock = MockDpd::new( + found(vec![member("rear0", 0), member("rear1", 0)]), + WriteOutcome::Updated, + ); + + let programmed = + run_sweep(HashMap::new(), &imported, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear1", 0)]))]) + ); + assert!(mock.writes().is_empty()); + + let programmed = + run_sweep(programmed, &HashSet::new(), &peer_links, &mock); + assert!(programmed.is_empty()); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn existing_members_are_kept_on_addition() { + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![member("rear1", 0), external("qsfp0", 0)]), + WriteOutcome::Updated, + ); + + let programmed = + reconcile(group, &imported, &peer_links, Members::new(), &mock); + assert_eq!(programmed, set(&[member("rear0", 0)])); + let writes = mock.writes(); + assert_eq!(writes.len(), 1); + assert_eq!( + set(&writes[0]), + set(&[ + member("rear0", 0), + member("rear1", 0), + external("qsfp0", 0) + ]) + ); + + let programmed = + reconcile(group, &imported, &peer_links, programmed.clone(), &mock); + assert_eq!(programmed, set(&[member("rear0", 0)])); + assert_eq!(mock.writes().len(), 1); + } + + #[tokio::test(start_paused = true)] + async fn missing_group_is_retried_on_periodic_sweep() { + let dir = camino_tempfile::Utf8TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().as_str(), log.clone()).unwrap(); + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + db.splice_imported_mcast(crate::db::MulticastRibChange::new( + "tfportrear0_0", + &imported, + &HashSet::new(), + )); + let imports = db.imported_mcast_links(); + let wakes = db.subscribe_mcast_sweep(); + + let mock = MockDpd::new(FetchOutcome::Missing, WriteOutcome::Updated); + let (fetched_tx, fetched_rx) = oneshot::channel(); + *mock.fetch_result_tx.lock().unwrap() = Some(fetched_tx); + let sweep = + run_with_client(db.clone(), &mock, db.subscribe_mcast_sweep(), log); + let check = async { + let result = + tokio::time::timeout(RECONCILE_INTERVAL / 2, fetched_rx) + .await + .expect("first sweep did not fetch the missing group") + .unwrap(); + assert!(matches!(result, FetchOutcome::Missing)); + assert!(mock.writes().is_empty()); + + let missing_at = tokio::time::Instant::now(); + *mock.fetch.lock().unwrap() = found(Vec::new()); + tokio::time::timeout( + RECONCILE_INTERVAL * 2, + mock.write_attempted.notified(), + ) + .await + .expect("periodic sweep did not program the newly created group"); + + assert!(missing_at.elapsed() >= RECONCILE_INTERVAL); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + assert_eq!(db.imported_mcast_links(), imports); + assert!(!wakes.has_changed().unwrap()); + }; + tokio::select! { + _ = sweep => panic!("sweep exited"), + _ = check => {} + } + } + + #[test] + fn deleted_group_drops() { + let group = underlay(1); + let mock = MockDpd::new(FetchOutcome::Missing, WriteOutcome::Updated); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert!(mock.writes().is_empty()); + } + + #[test] + fn read_retry_keeps_withdrawn_group() { + let group = underlay(1); + let mock = MockDpd::new(FetchOutcome::Retry, WriteOutcome::Updated); + + // A withdrawn group must not drop out on a transient read failure, or + // its programmed members would stay with nothing to remove them. + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + set(&[member("rear0", 0)]) + ); + assert!(mock.writes().is_empty()); + } + + #[tokio::test] + async fn transport_failure_retries_withdrawal() { + read_failure_retries_withdrawal(None).await; + } + + #[tokio::test] + async fn http_503_retries_withdrawal() { + read_failure_retries_withdrawal(Some(StatusCode::SERVICE_UNAVAILABLE)) + .await; + } + + async fn read_failure_retries_withdrawal( + failure_status: Option, + ) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let client = Client::new( + &format!("http://{}", listener.local_addr().unwrap()), + ClientState { + tag: DDM_TAG.into(), + log: log.clone(), + }, + ); + let group = underlay(1); + let rear0 = member("rear0", 0); + let preserved = external("qsfp0", 0); + let members = + std::sync::Mutex::new(vec![rear0.clone(), preserved.clone()]); + let unavailable = AtomicBool::new(false); + let failures = AtomicUsize::new(0); + let writes = AtomicUsize::new(0); + + let server = async { + loop { + let (socket, _) = listener.accept().await.unwrap(); + if failure_status.is_none() + && unavailable.load(Ordering::Relaxed) + { + failures.fetch_add(1, Ordering::Relaxed); + drop(socket); + continue; + } + + let service = service_fn(|request: Request| { + let members = &members; + let writes = &writes; + let unavailable = &unavailable; + let failures = &failures; + async move { + assert_eq!( + request.uri().path(), + format!("/multicast/underlay-groups/{group}") + ); + if let Some(status) = failure_status + .filter(|_| unavailable.load(Ordering::Relaxed)) + { + assert_eq!(request.method(), Method::GET); + failures.fetch_add(1, Ordering::Relaxed); + let body = serde_json::json!({ + "message": "multicast group repair failed", + "request_id": "test", + }); + return Ok::<_, Infallible>( + Response::builder() + .status(status) + .header("content-type", "application/json") + .body(Full::new(Bytes::from( + body.to_string(), + ))) + .unwrap(), + ); + } + if request.method() == Method::PUT { + assert_eq!(request.uri().query(), Some("tag=tag")); + let body = request + .into_body() + .collect() + .await + .unwrap() + .to_bytes(); + let update: MulticastGroupUpdateUnderlayEntry = + serde_json::from_slice(&body).unwrap(); + *members.lock().unwrap() = update.members; + writes.fetch_add(1, Ordering::Relaxed); + } else { + assert_eq!(request.method(), Method::GET); + } + let body = serde_json::to_vec( + &MulticastGroupUnderlayResponse { + group_ip: group, + tag: "tag".into(), + external_group_id: 100, + underlay_group_id: 101, + members: members.lock().unwrap().clone(), + }, + ) + .unwrap(); + Ok::<_, Infallible>( + Response::builder() + .header("content-type", "application/json") + .body(Full::new(Bytes::from(body))) + .unwrap(), + ) + } + }); + http1::Builder::new() + .keep_alive(false) + .serve_connection(TokioIo::new(socket), service) + .await + .unwrap(); + } + }; + let check = async { + let peer = "fe80::1".parse().unwrap(); + let imported = + vec![(route(peer, group), "tfportrear0_0".to_string())]; + let mut programmed = HashMap::new(); + let mut unresolved = HashSet::new(); + + reconcile_pass( + &mut programmed, + &mut unresolved, + &imported, + &client, + &log, + ) + .await; + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert_eq!(writes.load(Ordering::Relaxed), 0); + + unavailable.store(true, Ordering::Relaxed); + reconcile_pass( + &mut programmed, + &mut unresolved, + &[], + &client, + &log, + ) + .await; + assert!(failures.load(Ordering::Relaxed) > 0); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert_eq!( + set(&members.lock().unwrap()), + set(&[rear0, preserved.clone()]) + ); + assert_eq!(writes.load(Ordering::Relaxed), 0); + + unavailable.store(false, Ordering::Relaxed); + reconcile_pass( + &mut programmed, + &mut unresolved, + &[], + &client, + &log, + ) + .await; + assert!(programmed.is_empty()); + assert_eq!(set(&members.lock().unwrap()), set(&[preserved])); + assert_eq!(writes.load(Ordering::Relaxed), 1); + }; + tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + _ = server => panic!("test DPD server exited"), + _ = check => {} + } + }) + .await + .expect("withdrawal did not recover after the read failure"); + } + + #[test] + fn withdrawn_group_with_no_members_drops_without_writing() { + let group = underlay(1); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert!(mock.writes().is_empty()); + } + + #[test] + fn empty_write_retry_keeps_withdrawn_group() { + let group = underlay(1); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Retry); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + set(&[member("rear0", 0)]) + ); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn write_retry_retains_member_for_later_withdrawal() { + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mut mock = MockDpd::new(found(Vec::new()), WriteOutcome::Retry); + + let programmed = + reconcile(group, &imported, &peer_links, Members::new(), &mock); + + *mock.fetch.lock().unwrap() = found(vec![member("rear0", 0)]); + mock.write_outcome = WriteOutcome::Updated; + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &peer_links, + programmed.clone(), + &mock + ), + Members::new() + ); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)], Vec::new()]); + } + + #[test] + fn active_group_with_drifted_members_writes_wanted_set() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert_eq!( + reconcile( + group, + &imported, + &peer_links, + set(&[member("rear0", 0)]), + &mock + ), + set(&[member("rear0", 0)]) + ); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + /// A programmed member whose interface no longer resolves to a switch link + /// is removed, which would occur on peer expiration. + #[test] + fn unresolved_interface_removes_programmed_member() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert_eq!( + reconcile( + group, + &imported, + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn only_programmed_members_are_removed() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![ + member("rear0", 0), + member("rear1", 0), + member("rear2", 0), + external("qsfp0", 0), + ]), + WriteOutcome::Updated, + ); + + assert_eq!( + reconcile( + group, + &imported, + &peer_links, + set(&[member("rear0", 0), member("rear1", 0)]), + &mock, + ), + set(&[member("rear0", 0)]) + ); + let writes = mock.writes(); + assert_eq!(writes.len(), 1); + assert_eq!( + set(&writes[0]), + set(&[ + member("rear0", 0), + member("rear2", 0), + external("qsfp0", 0) + ]) + ); + } + + fn external(port: &str, link: u8) -> MulticastGroupMember { + MulticastGroupMember { + port_id: PortId::Qsfp( + dpd_client::types::Qsfp::try_from(port).unwrap(), + ), + link_id: LinkId(link), + direction: Direction::External, + } + } + + #[test] + fn drain_keeps_external_members() { + let group = underlay(1); + let mock = MockDpd::new( + found(vec![member("rear0", 0), external("qsfp0", 0)]), + WriteOutcome::Updated, + ); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert_eq!(mock.writes(), vec![vec![external("qsfp0", 0)]]); + } + + #[test] + fn group_outcomes_are_independent() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let readable_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let retried = underlay(1); + let missing = underlay(2); + let readable = underlay(3); + let imported = HashSet::from([ + route(peer, retried), + route(peer, missing), + route(readable_peer, readable), + ]); + let peer_links = HashMap::from([ + (peer, rear("rear0", 0)), + (readable_peer, rear("rear2", 0)), + ]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + mock.fetch_by_group.lock().unwrap().extend([ + (retried, FetchOutcome::Retry), + (missing, FetchOutcome::Missing), + (readable, found(vec![member("rear1", 0)])), + ]); + + let prev = HashMap::from([ + (retried, set(&[member("rear1", 0)])), + (missing, set(&[member("rear1", 0)])), + ]); + let programmed = run_sweep(prev, &imported, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([ + (retried, set(&[member("rear1", 0)])), + (readable, set(&[member("rear2", 0)])), + ]) + ); + let writes = mock.group_writes(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].0, readable); + assert_eq!( + set(&writes[0].1), + set(&[member("rear1", 0), member("rear2", 0)]) + ); + } +} diff --git a/ddm/src/sm/state.rs b/ddm/src/sm/state.rs index 9b6f99e6f..493920d1d 100644 --- a/ddm/src/sm/state.rs +++ b/ddm/src/sm/state.rs @@ -811,7 +811,9 @@ impl Exchange { self.retired.clear(); *session = None; self.ctx.iface.clear_peer(); - self.ctx.db.remove_nexthop_routes(self.peer) + self.ctx + .db + .remove_nexthop_routes(self.peer, &self.ctx.config.if_name) }; let routes = removed .underlay @@ -1104,7 +1106,10 @@ impl Exchange { self.session = Arc::new(ExchangeSession::new(version)); *session = Some(self.session.clone()); if prior_version >= Version::V4 && version < Version::V4 { - Some(self.ctx.db.remove_nexthop_mcast_routes(self.peer)) + Some(self.ctx.db.remove_nexthop_mcast_routes( + self.peer, + &self.ctx.config.if_name, + )) } else { None } diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index 14a4ee266..14e40d8b5 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::sm::{Config, DpdConfig}; -use crate::{dbg, err, inf, wrn}; +use crate::{DDM_TAG, dbg, err, inf, wrn}; use ddm_api_types::db::TunnelRoute; use dpd_client::Client; use dpd_client::ClientState; @@ -24,8 +24,6 @@ use ::{ std::collections::HashMap, }; -const DDM_DPD_TAG: &str = "ddmd"; - #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct Route { pub dest: IpAddr, @@ -145,7 +143,7 @@ pub fn add_routes_dendrite( dbg!(log, ifname, "sending to dpd host={} port={}", host, port); let client_state = ClientState { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), log: log.clone(), }; let client = Client::new(&format!("http://{host}:{port}"), client_state); @@ -183,43 +181,14 @@ pub fn add_routes_dendrite( } }; - // TODO this is gross, use link type properties rather than futzing - // around with strings. - - let port_id = { - if let Some(egress_port_num) = ifname - .strip_prefix("tfportrear") - .and_then(|x| x.strip_suffix("_0")) - .map(|x| x.trim()) - .and_then(|x| x.parse::().ok()) - { - let port_name = format!("rear{}", egress_port_num); - match types::Rear::try_from(port_name.as_str()) { - Ok(rear) => PortId::Rear(rear), - Err(e) => { - err!(log, ifname, "bad port name ({port_name}): {e}"); - continue; - } - } - } else if let Some(egress_port_num) = ifname - .strip_prefix("tfportqsfp") - .and_then(|x| x.strip_suffix("_0")) - .map(|x| x.trim()) - .and_then(|x| x.parse::().ok()) - { - let port_name = format!("qsfp{}", egress_port_num); - match types::Qsfp::try_from(port_name.as_str()) { - Ok(qsfp) => PortId::Qsfp(qsfp), - Err(e) => { - err!(log, ifname, "bad port name ({port_name}): {e}"); - continue; - } + let (port_id, link_id) = + match mg_common::tfport::port_link_from_untagged_ifname(ifname) { + Ok(port_link) => port_link, + Err(e) => { + err!(log, ifname, "{e}"); + continue; } - } else { - err!(log, ifname, "expected tfportrear or tfportqsfp"); - continue; - } - }; + }; inf!( log, @@ -228,14 +197,11 @@ pub fn add_routes_dendrite( r.dest, r.gw, port_id, - 0, + link_id.0, ); - // TODO breakout considerations - let link_id = types::LinkId(0); - let target = types::Ipv6Route { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), port_id, link_id, tgt_ip: gw, @@ -428,7 +394,7 @@ pub fn remove_routes_dendrite( log: &Logger, ) { let client_state = ClientState { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), log: log.clone(), }; let client = Client::new(&format!("http://{host}:{port}"), client_state); @@ -477,7 +443,7 @@ pub fn get_routes_dendrite( log: Logger, ) -> Result, String> { let client_state = ClientState { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), log: log.clone(), }; let client = Client::new(&format!("http://{host}:{port}"), client_state); diff --git a/ddmadm/src/main.rs b/ddmadm/src/main.rs index c13046271..79e075ddc 100644 --- a/ddmadm/src/main.rs +++ b/ddmadm/src/main.rs @@ -332,22 +332,28 @@ async fn run() -> Result<()> { SubCommand::MulticastImported => { let msg = client.get_multicast_groups().await?; let mut routes: Vec<_> = msg.into_inner().into_iter().collect(); - routes.sort_by_key(|route| { - (origin_sort_key(&route.origin), route.nexthop) + routes.sort_by_key(|imported| { + ( + origin_sort_key(&imported.route.origin), + imported.route.nexthop, + imported.if_name.clone(), + ) }); let mut tw = TabWriter::new(stdout()); writeln!( &mut tw, - "{}\t{}\t{}\t{}\t{}\t{}\t{}", + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", "Overlay Group".dimmed(), "Underlay Group".dimmed(), "VNI".dimmed(), "Metric".dimmed(), "Source".dimmed(), "Nexthop".dimmed(), + "Interface".dimmed(), "Path".dimmed(), )?; - for route in &routes { + for imported in &routes { + let route = &imported.route; let path: Vec<_> = route .path .iter() @@ -356,13 +362,14 @@ async fn run() -> Result<()> { .collect(); writeln!( &mut tw, - "{}\t{}\t{}\t{}\t{}\t{}\t{}", + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", route.origin.overlay_group(), route.origin.underlay_group(), route.origin.vni(), route.origin.metric(), source_column(route.origin.source()), route.nexthop, + imported.if_name, path.join(" "), )?; } diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index 4c9710943..cf7402b5a 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -184,7 +184,7 @@ async fn run() { let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); let context = Arc::new(Mutex::new(HandlerContext { - db, + db: db.clone(), stats: router_stats, peers, stats_handler: Arc::new(Mutex::new(None)), @@ -208,6 +208,8 @@ async fn run() { router_id: router_id.clone(), })); + start_mcast_sweep(arg.api_only, dpd, db, log.clone()); + if arg.with_stats && let (Some(rack_uuid), Some(sled_uuid)) = (arg.rack_uuid, arg.sled_uuid) @@ -344,6 +346,47 @@ fn start_state_machines( Vec::new() } +/// Spawn the multicast membership sweep. +/// +/// This bypasses the sweep under `--api-only` or without `--dendrite`. In +/// API-only mode, there's no routing, and, therefore, no imports to derive +/// membership from. +/// +/// Builds use `panic = "abort"` (see `.cargo/config.toml`), which means that a +/// panic during a sweep terminates `ddmd` rather than only the task, after +/// which SMF restarts the service. +/// +/// The restarted process bootstraps with no record of programmed members. +/// Existing membership remains in place, and only those an import re-derives +/// come back under management, akin to the unicast and tunnel routes left +/// behind by a `ddmd` crash. +#[cfg(all(feature = "backend", target_os = "illumos"))] +fn start_mcast_sweep( + api_only: bool, + dpd: Option, + db: Db, + log: Logger, +) { + let Some(dpd) = dpd else { + return; + }; + if api_only { + return; + } + tokio::spawn(ddm::mcast::run(db, dpd, log)); +} + +/// Non-illumos variant: underlay multicast replicates only on a switch, meaning +/// that a sweep never starts. +#[cfg(not(all(feature = "backend", target_os = "illumos")))] +fn start_mcast_sweep( + _api_only: bool, + _dpd: Option, + _db: Db, + _log: Logger, +) { +} + /// Install a Ctrl-C handler that withdraws ddmd's imported routes from the /// kernel before exiting. On non-illumos builds there are no kernel routes /// to withdraw, so the handler just exits cleanly. diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index 2fc27988b..aaee6c5bb 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -17,6 +17,10 @@ backoff.workspace = true smf.workspace = true uuid.workspace = true libc.workspace = true +dpd-client.workspace = true + +[dev-dependencies] +proptest.workspace = true # We need this on illumos, but must omit it on other platforms [target.'cfg(target_os = "illumos")'.dependencies.libnet] diff --git a/mg-common/src/lib.rs b/mg-common/src/lib.rs index b2af8cd22..abb47e2fb 100644 --- a/mg-common/src/lib.rs +++ b/mg-common/src/lib.rs @@ -8,6 +8,7 @@ pub mod nexus; pub mod smf; pub mod stats; pub mod test; +pub mod tfport; pub mod thread; use std::time::Duration; diff --git a/mg-common/src/tfport.rs b/mg-common/src/tfport.rs new file mode 100644 index 000000000..c826aec57 --- /dev/null +++ b/mg-common/src/tfport.rs @@ -0,0 +1,411 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Parsing of Tofino port (`tfport`) datalink names. +//! +//! Tofino switch ports are surfaced to the host as illumos datalinks named +//! `tfport_`; e.g., `tfportqsfp10_0` (front panel) or +//! `tfportrear0_0` (backplane). This module parses this datalink naming +//! convention into its specific components, allowing callers to easily map a +//! datalink name back to a switch port identifier. +//! +//! Note: VLAN links over an uplink append the VLAN ID after a `.`, e.g., +//! `tfportqsfp0_0.100`, as surfaced by Dendrite's `uplinkd` ([VLAN naming]). +//! +//! [VLAN naming]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/uplinkd/src/main.rs#L251-L256 + +use dpd_client::types; + +/// Prefix shared by every `tfport` datalink name. +const TFPORT_DEVICE_PREFIX: &str = "tfport"; + +/// The switch-port device kind encoded into a `tfport` datalink name. +/// +/// Front-panel links typically appear as `qsfp`. Backplane links to sleds +/// appear as `rear`, which are the only ports ddmd currently programs as +/// underlay multicast members for ingress multicast traffic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TfportKind { + Qsfp, + Rear, +} + +impl TfportKind { + /// The device token as it appears both in a tfport name (`tfportrear0_0`) + /// and in a DPD port name (`rear0`). + fn token(self) -> &'static str { + match self { + TfportKind::Qsfp => "qsfp", + TfportKind::Rear => "rear", + } + } + + /// Parse a device kind from its datalink token (`qsfp`, `rear`). + fn from_token(s: &str) -> Option { + match s { + "qsfp" => Some(TfportKind::Qsfp), + "rear" => Some(TfportKind::Rear), + _ => None, + } + } +} + +/// Components parsed from a `tfport` datalink name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TfportName { + /// Device kind (front-panel vs backplane). + kind: TfportKind, + /// Switch port number within the kind. + port: u8, + /// Link (lane) number within the port. + link: u8, + /// Optional VLAN tag appended after a `.`. + vlan: Option, +} + +/// Parse a `tfport` datalink name into its components. +/// +/// # Errors +/// +/// Returns an error if `name` lacks the specified `tfport` prefix, points to +/// an unrecognized device kind, or has malformed port/link/vlan fields. +fn parse_tfport_name(name: &str) -> Result { + let body = name.strip_prefix(TFPORT_DEVICE_PREFIX).ok_or_else(|| { + format!("{name} missing expected prefix {TFPORT_DEVICE_PREFIX}") + })?; + + // The device kind is the leading alphabetic run (`qsfp`, `rear`), the + // remainder carries the port/link/vlan numbers. + let split = body + .find(|c: char| !c.is_ascii_alphabetic()) + .ok_or_else(|| format!("{name} has no port id"))?; + let (kind_str, rest) = body.split_at(split); + let kind = TfportKind::from_token(kind_str).ok_or_else(|| { + format!("{name} has unsupported device kind {kind_str}") + })?; + + let (port_link, vlan) = rest + .split_once('.') + .map_or((rest, None), |(port_link, vlan)| (port_link, Some(vlan))); + + let (port, link) = port_link + .split_once('_') + .ok_or_else(|| format!("{name} has no link id"))?; + + let port = parse_number::(port) + .ok_or_else(|| format!("{name} has invalid port {port}"))?; + + let link = parse_number::(link) + .ok_or_else(|| format!("{name} has invalid link id {link}"))?; + + // A second `.` (e.g. `tfportqsfp10_0.100.200`) leaves a non-numeric + // remainder, rejected by the parse op below. + let vlan = vlan + .map(|vlan| { + parse_number::(vlan) + .ok_or_else(|| format!("{name} has invalid vlan {vlan}")) + }) + .transpose()?; + + Ok(TfportName { + kind, + port, + link, + vlan, + }) +} + +/// Parse a port, link, or VLAN number made only of ASCII digits. +/// +/// `str::parse` accepts a leading `+`, which no tfport name conveys, +/// so we have to rule out a sign or any other non-digit before parsing. +fn parse_number(s: &str) -> Option { + if !s.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + s.parse().ok() +} + +/// Build a DPD [`PortId`] from a parsed tfport kind and port number. +/// +/// # Errors +/// +/// Returns an error if the synthesized port name is not a valid DPD `qsfp` or +/// `rear` port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +fn tfport_port_id( + kind: TfportKind, + port: u8, +) -> Result { + let port_name = format!("{}{port}", kind.token()); + match kind { + TfportKind::Qsfp => types::Qsfp::try_from(port_name.as_str()) + .map(types::PortId::Qsfp) + .map_err(|e| format!("bad qsfp port name {port_name}: {e}")), + TfportKind::Rear => types::Rear::try_from(port_name.as_str()) + .map(types::PortId::Rear) + .map_err(|e| format!("bad rear port name {port_name}: {e}")), + } +} + +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the DPD +/// ([`PortId`], [`LinkId`]) pair that names the switch port and link. +/// +/// # Errors +/// +/// Returns an error if `ifname` is not a valid tfport datalink name or its +/// kind and port do not form a valid DPD port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +/// [`LinkId`]: dpd_client::types::LinkId +pub fn port_link_from_ifname( + ifname: &str, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + port_link(&parse_tfport_name(ifname)?) +} + +fn port_link( + tfport: &TfportName, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + let port_id = tfport_port_id(tfport.kind, tfport.port)?; + // Breakout lanes surface up as distinct datalinks (`tfportqsfp0_0`, + // `tfportqsfp0_1`, ...), sharing one `PortId`. The parsed link is the + // lane, which is the DPD `LinkId`. + let link_id = dpd_client::types::LinkId(tfport.link); + Ok((port_id, link_id)) +} + +/// Resolve an untagged tfport datalink name to its DPD `(PortId, LinkId)`. +/// +/// Unlike [`port_link_from_ifname`], which ignores a VLAN suffix, this +/// method rejects it. A `(PortId, LinkId)` names the untagged port and link. +/// Resolving a VLAN interface to it would program state with the incorrect +/// tagging. +/// +/// DDM peers operate over untagged links, using this naming convention, while +/// mg-lower keeps the more open convention for next hops on [VLAN uplinks]. +/// +/// # Errors +/// +/// Returns an error if `ifname` carries a VLAN suffix, or for any reason +/// [`port_link_from_ifname`] would. +/// +/// [VLAN uplinks]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/uplinkd/src/main.rs#L251-L256 +pub fn port_link_from_untagged_ifname( + ifname: &str, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + let tfport = parse_tfport_name(ifname)?; + if let Some(vlan) = tfport.vlan { + return Err(format!( + "{ifname} is a vlan {vlan} interface, expected an untagged tfport" + )); + } + port_link(&tfport) +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + fn name( + kind: TfportKind, + port: u8, + link: u8, + vlan: Option, + ) -> TfportName { + TfportName { + kind, + port, + link, + vlan, + } + } + + #[test] + fn test_tfport_parser() { + assert_eq!( + parse_tfport_name("tfportqsfp10_0").unwrap(), + name(TfportKind::Qsfp, 10, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportqsfp10_0.100").unwrap(), + name(TfportKind::Qsfp, 10, 0, Some(100)) + ); + assert_eq!( + parse_tfport_name("tfportqsfp010_00.00100").unwrap(), + name(TfportKind::Qsfp, 10, 0, Some(100)) + ); + assert_eq!( + parse_tfport_name("tfportqsfp1_1").unwrap(), + name(TfportKind::Qsfp, 1, 1, None) + ); + + assert_eq!( + parse_tfport_name("tfportrear0_0").unwrap(), + name(TfportKind::Rear, 0, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportrear31_0.200").unwrap(), + name(TfportKind::Rear, 31, 0, Some(200)) + ); + + // Malformed names. + assert!(parse_tfport_name("fportqsfp10_0").is_err()); + assert!(parse_tfport_name("10_0").is_err()); + assert!(parse_tfport_name("tfportqsfp10").is_err()); + assert!(parse_tfport_name("tfportqsfp_10").is_err()); + assert!(parse_tfport_name("tfportqsfp0_").is_err()); + assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); + assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.100.200").is_err()); + assert!(parse_tfport_name("tfportrear0_0/ll").is_err()); + assert!(parse_tfport_name("tfportrear0_0/v6").is_err()); + + // Unsupported or missing device kind. + assert!(parse_tfport_name("tfportfoo0_0").is_err()); + assert!(parse_tfport_name("tfport0_0").is_err()); + + // Invalid numeric components. + assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); + assert!(parse_tfport_name("tfportqsfp+1_0").is_err()); + assert!(parse_tfport_name("tfportqsfp1_+0").is_err()); + assert!(parse_tfport_name("tfportqsfp1_0.+100").is_err()); + assert!(parse_tfport_name("tfportqsfp1_0.65536").is_err()); + } + + #[test] + fn untagged_resolution_rejects_vlan() { + assert!(port_link_from_untagged_ifname("tfportrear0_0").is_ok()); + assert!(port_link_from_untagged_ifname("tfportrear0_0.100").is_err()); + assert!(port_link_from_ifname("tfportqsfp0_0.100").is_ok()); + } + + #[test] + fn port_and_link_boundaries() { + assert!(port_link_from_untagged_ifname("tfportqsfp31_0").is_ok()); + assert!(port_link_from_untagged_ifname("tfportqsfp32_0").is_err()); + assert!(port_link_from_untagged_ifname("tfportrear31_0").is_ok()); + assert!(port_link_from_untagged_ifname("tfportrear32_0").is_err()); + assert!(parse_tfport_name("tfportqsfp255_0").is_ok()); + assert!(parse_tfport_name("tfportqsfp256_0").is_err()); + assert!(parse_tfport_name("tfportqsfp0_255").is_ok()); + assert!(parse_tfport_name("tfportqsfp0_256").is_err()); + assert!(parse_tfport_name("tfportint0_0").is_err()); + } + + fn near_miss_kind() -> impl Strategy { + prop_oneof![ + Just("qsfp".to_string()), + Just("rear".to_string()), + "[a-z]{0,4}", + ] + } + + fn near_miss_number() -> impl Strategy { + prop_oneof![ + "[0-9]{1,4}", + "\\+[0-9]{1,3}", + Just(String::new()), + "[0-9]{1,2}[a-z/]", + ] + } + + fn near_miss_suffix() -> impl Strategy { + prop_oneof![ + Just(String::new()), + "\\.[0-9]{1,6}", + Just(".".to_string()), + "\\.[0-9]{1,3}\\.[0-9]{0,3}", + "\\.[+-][0-9]{1,3}", + "\\.[\u{0660}-\u{0669}\u{FF10}-\u{FF19}]{1,3}", + "/[a-z0-9]{0,3}", + ] + } + + proptest! { + /// Property: a well-formed name round-trips through the parser. + /// + /// The parser is purely syntactic, exercising the full u8/u16 ranges. + #[test] + fn prop_roundtrip( + is_rear in any::(), + port in any::(), + link in any::(), + vlan in proptest::option::of(any::()), + ) { + let kind = + if is_rear { TfportKind::Rear } else { TfportKind::Qsfp }; + let mut ifname = format!("tfport{}{port}_{link}", kind.token()); + if let Some(vlan) = vlan { + ifname.push_str(&format!(".{vlan}")); + } + prop_assert_eq!( + parse_tfport_name(&ifname).unwrap(), + TfportName { kind, port, link, vlan }, + ); + } + + /// Property: a DPD `PortId` formatted into a tfport name resolves back + /// to the same `(PortId, LinkId)`. + #[test] + fn prop_resolves_dpd_port_names( + is_rear in any::(), + port in 0u8..32, + link in any::(), + ) { + let port_id = if is_rear { + types::PortId::Rear( + types::Rear::try_from(format!("rear{port}").as_str()) + .unwrap(), + ) + } else { + types::PortId::Qsfp( + types::Qsfp::try_from(format!("qsfp{port}").as_str()) + .unwrap(), + ) + }; + let ifname = format!("tfport{port_id}_{link}"); + prop_assert_eq!( + port_link_from_untagged_ifname(&ifname).unwrap(), + (port_id, types::LinkId(link)), + ); + } + + /// Property: parsing arbitrary input never panics. + #[test] + fn prop_never_panics(ifname in ".*") { + let _ = parse_tfport_name(&ifname); + } + + /// Property: canonical formatting preserves the parsed components. + #[test] + fn prop_accepted_names_roundtrip( + kind in near_miss_kind(), + port in near_miss_number(), + sep in prop_oneof![Just(""), Just("_"), Just("__")], + link in near_miss_number(), + suffix in near_miss_suffix(), + ) { + let ifname = format!("tfport{kind}{port}{sep}{link}{suffix}"); + if let Ok(name) = parse_tfport_name(&ifname) { + let mut canonical = format!( + "tfport{}{}_{}", + name.kind.token(), + name.port, + name.link, + ); + if let Some(vlan) = name.vlan { + canonical.push_str(&format!(".{vlan}")); + } + prop_assert_eq!(parse_tfport_name(&canonical), Ok(name)); + } + } + } +} diff --git a/mg-lower/src/ddm.rs b/mg-lower/src/ddm.rs index 8e1744851..d265407f2 100644 --- a/mg-lower/src/ddm.rs +++ b/mg-lower/src/ddm.rs @@ -5,11 +5,12 @@ use crate::log::ddm_log; #[cfg(target_os = "illumos")] use ddm_admin_client::Client; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::Logger; use std::{net::Ipv6Addr, sync::Arc}; +use crate::error::Error; use crate::platform::Ddm; pub(crate) const BOUNDARY_SERVICES_VNI: u32 = 99; @@ -107,6 +108,64 @@ pub(crate) fn remove_tunnel_routes<'a, I: Iterator>( } } +pub(crate) fn add_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) -> Result<(), Error> { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return Ok(()); + } + let resp = rt.block_on(client.advertise_multicast_groups(&routes)); + if let Err(e) = resp { + ddm_log!(log, + error, + "advertise multicast groups error: {e}"; + "error" => format!("{e}"), + "groups" => format!("{routes:#?}") + ); + return Err(e.into()); + } + Ok(()) +} + +pub(crate) fn remove_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) -> Result<(), Error> { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return Ok(()); + } + let resp = rt.block_on(client.withdraw_multicast_groups(&routes)); + match resp { + Err(e) => { + ddm_log!(log, + error, + "withdraw multicast groups error: {e}"; + "groups" => format!("{routes:#?}") + ); + return Err(e.into()); + } + Ok(_) => ddm_log!(log, + debug, + "withdrew multicast groups"; + "groups" => format!("{routes:#?}") + ), + } + Ok(()) +} + #[cfg(target_os = "illumos")] pub fn new_ddm_client(log: &Logger) -> Client { Client::new("http://localhost:8000", log.clone()) diff --git a/mg-lower/src/dendrite.rs b/mg-lower/src/dendrite.rs index 8661ba108..c7ee45f4f 100644 --- a/mg-lower/src/dendrite.rs +++ b/mg-lower/src/dendrite.rs @@ -21,7 +21,6 @@ use std::{ time::Duration, }; -const TFPORT_QSFP_DEVICE_PREFIX: &str = "tfportqsfp"; const UNIT_DPD: &str = "dpd"; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -329,68 +328,17 @@ where Ok(()) } -// Translate a tfport name into the underlying (port, link, vlan) tuple. -// tfportqsfp10_0 would translate to (10, 0, None) -// tfportqsfp10_0.100 would translate to (10, 0, Some(100)) -// TODO this is gross, use link type properties rather than futzing -// around with strings. -fn parse_tfport_name(name: &str) -> Result<(u8, u8, Option), Error> { - let body = - name.strip_prefix(TFPORT_QSFP_DEVICE_PREFIX) - .ok_or(Error::Tfport(format!( - "{} missing expected prefix {}", - name, TFPORT_QSFP_DEVICE_PREFIX - )))?; - let fields: Vec<&str> = body.split('.').collect(); - let (port, link) = fields[0] - .split_once('_') - .ok_or(Error::Tfport(format!("{} has no link id", name)))?; - - let port = port.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid port {}", name, port)) - })?; - - let link = link.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid link id {}", name, link)) - })?; - - let vlan = match fields.len() { - 1 => Ok(None), - 2 => fields[1].parse::().map(Some).map_err(|_| { - Error::Tfport(format!("{} has invalid vlan {}", name, fields[1])) - }), - _ => Err(Error::Tfport(format!( - "{} has multiple vlan deliminators", - name +fn qsfp_port_and_link_from_ifname( + ifname: &str, +) -> Result<(types::PortId, types::LinkId), Error> { + match mg_common::tfport::port_link_from_ifname(ifname) + .map_err(Error::Tfport)? + { + port_link @ (types::PortId::Qsfp(_), _) => Ok(port_link), + (port_id, _) => Err(Error::Tfport(format!( + "{ifname} resolves to {port_id}, expected a qsfp port" ))), - }?; - - Ok((port, link, vlan)) -} - -#[test] -fn test_tfport_parser() { - // Test valid names - assert_eq!(parse_tfport_name("tfportqsfp10_0").unwrap(), (10, 0, None)); - assert_eq!( - parse_tfport_name("tfportqsfp10_0.100").unwrap(), - (10, 0, Some(100)) - ); - assert_eq!(parse_tfport_name("tfportqsfp1_1").unwrap(), (1, 1, None)); - - // test malformed names - assert!(parse_tfport_name("fportqsfp10_0").is_err()); - assert!(parse_tfport_name("10_0").is_err()); - assert!(parse_tfport_name("tfportqsfp10").is_err()); - assert!(parse_tfport_name("tfportqsfp_10").is_err()); - assert!(parse_tfport_name("tfportqsfp0_").is_err()); - assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); - assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); - - // test invalid components - assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); + } } fn get_port_and_link( @@ -402,18 +350,7 @@ fn get_port_and_link( && nh6.is_unicast_link_local() && let Some(ref iface) = path.nexthop_interface { - let (port, link, _vlan) = parse_tfport_name(iface)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(port_name.as_str()) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {iface} port name: {port_name}: {e}", - )) - })?; - // TODO breakout considerations - let link_id = types::LinkId(link); - return Ok((port_id, link_id)); + return qsfp_port_and_link_from_ifname(iface); } // Standard nexthop resolution for numbered peers @@ -437,19 +374,7 @@ fn resolve_port_and_link( } }; - let (port, link, _vlan) = parse_tfport_name(&ifname)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(port_name.as_str()) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {ifname} port name: {port_name}: {e}" - )) - })?; - - // TODO breakout considerations - let link_id = types::LinkId(link); - Ok((port_id, link_id)) + qsfp_port_and_link_from_ifname(&ifname) } pub(crate) fn get_routes_for_prefix( diff --git a/mg-lower/src/lib.rs b/mg-lower/src/lib.rs index 33096b5e1..55153f625 100644 --- a/mg-lower/src/lib.rs +++ b/mg-lower/src/lib.rs @@ -25,7 +25,7 @@ use std::collections::HashSet; use std::net::Ipv6Addr; use std::sync::Arc; use std::sync::mpsc::{RecvTimeoutError, channel}; -use std::thread::sleep; +use std::thread::{Builder, JoinHandle, sleep}; use std::time::Duration; // Re-export production backends so callers (e.g. mgd) can construct them. @@ -40,6 +40,7 @@ mod ddm; mod dendrite; mod error; mod log; +pub mod mrib; mod platform; #[cfg(test)] @@ -51,15 +52,48 @@ const COMPONENT_MG_LOWER: &str = MG_LOWER_TAG; const MOD_SYNC: &str = "sync"; const UNIT_EVENT_LOOP: &str = "event_loop"; +/// Platform backends used for route lookups and synchronization. +pub struct Backends { + /// Programs routes and the tunnel endpoint address into the switch ASIC + /// via DPD. + pub dpd: D, + /// Advertises tunnel endpoints to DDM. + pub ddm: M, + /// Looks up kernel routes in the switch zone in order to resolve + /// next hops to switch ports. + pub sw: S, +} + +/// Run the lower-half loop on a dedicated `mg-lower` thread. +/// +/// # Errors +/// +/// Returns an error if the thread cannot be spawned. +pub fn spawn( + tep: Ipv6Addr, + db: Db, + log: Logger, + stats: Arc, + rt: Arc, + backends: Backends< + impl Dpd + Send + 'static, + impl Ddm + Send + 'static, + impl SwitchZone + Send + 'static, + >, +) -> std::io::Result> { + let Backends { dpd, ddm, sw } = backends; + Builder::new() + .name(MG_LOWER_TAG.to_string()) + .spawn(move || run(tep, db, log, stats, rt, &dpd, &ddm, &sw)) +} + /// This is the primary entry point for the lower half. It loops forever, -/// observing changes in the routing databse and synchronizing them to the +/// observing changes in the routing database and synchronizing them to the /// underlying forwarding platform. The loop sets up a watcher to start /// receiving events, does an initial synchronization, then responds to changes -/// moving foward. The loop runs on the calling thread, so callers are -/// responsible for running this function in a separate thread if asynchronous -/// execution is required. +/// moving forward. #[allow(clippy::too_many_arguments)] -pub fn run( +fn run( tep: Ipv6Addr, //tunnel endpoint address db: Db, log: Logger, diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs new file mode 100644 index 000000000..f3b14229d --- /dev/null +++ b/mg-lower/src/mrib.rs @@ -0,0 +1,420 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! MRIB (Multicast Routing Information Base) lower-half synchronization. +//! +//! Advertises locally originated MRIB multicast groups to the DDM admin API, +//! which distributes them across the underlay to other sleds and racks. This is +//! the multicast analog of the unicast lower-half's tunnel-endpoint +//! origination. +//! +//! Origination reads from the local MRIB (`loc_mrib`) by way of its watcher. On +//! each change notification, or once `MRIB_PERIODIC_SYNC_INTERVAL` elapses +//! without a notification, the loop runs the same full reconciliation +//! of `loc_mrib` against the groups originated from DDM. This works similarly +//! to the unicast lower-half's loop (see [`crate::spawn`]). +//! +//! The inbound membership half (resolving DDM-imported routes into switch +//! replication members in DPD) lives in ddmd (`ddm::mcast`), where its input, +//! the imported set with each route's interface, is owned in-process. +//! +//! ## Data Flow +//! +//! ```text +//! Origination (MRIB -> DDM -> underlay) +//! MRIB (loc_mrib changes) +//! | [MribChangeNotification] +//! v [MulticastOrigin] (diff vs DDM originated) +//! DDM admin API --[DDM exchange]--> other sleds/racks +//! ``` +//! +//! See RFD 488 for the multicast architecture. + +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::mpsc::{Receiver, RecvTimeoutError, channel}; +use std::thread::{Builder, JoinHandle, sleep}; +use std::time::Duration; + +use ddm_api_types_versions::latest::net::{MulticastOrigin, OverlayMulticast}; +use rdb::Db; +use rdb::types::{MribChangeNotification, MulticastRoute}; +use slog::{Logger, error, warn}; + +use crate::ddm::{add_multicast_routes, remove_multicast_routes}; +use crate::platform::Ddm; + +pub(crate) const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; + +/// Interval between periodic MRIB full syncs. +/// +/// This interval covers the case where an MRIB change notification is missed, +/// following from the unicast lower-half's 1s resync cadence (see +/// [`crate::spawn`]). +const MRIB_PERIODIC_SYNC_INTERVAL: Duration = Duration::from_secs(1); + +/// The delay before retrying a failed MRIB sync. +/// +/// This is independent of [`MRIB_PERIODIC_SYNC_INTERVAL`], separating +/// resync configuration from changing the retry rate. +const MRIB_SYNC_RETRY_DELAY: Duration = Duration::from_secs(1); + +/// Convert an MRIB `MulticastRoute` to a DDM `MulticastOrigin`. +/// +/// # Errors +/// +/// Returns an error if the route's group or source fails DDM origin +/// validation. +fn ddm_origin(route: &MulticastRoute) -> Result { + let overlay_group = OverlayMulticast::new(route.key.group().ip()) + .map_err(|e| format!("invalid overlay group: {e}"))?; + + MulticastOrigin::new( + overlay_group, + route.underlay_group, + route.key.vni(), + route.key.source(), + ) + .map_err(|e| format!("invalid multicast origin: {e}")) +} + +/// Run the MRIB origination loop on a dedicated `mg-lower-mrib` thread. +/// +/// # Errors +/// +/// Returns an error if the thread cannot be spawned. +pub fn spawn( + db: Db, + log: Logger, + rt: Arc, + ddm: impl Ddm + Send + 'static, +) -> std::io::Result> { + Builder::new() + .name(MG_LOWER_MRIB_TAG.to_string()) + .spawn(move || run(db, log, rt, &ddm)) +} + +/// Run the MRIB origination loop. +/// +/// This function loops ad infinitum, watching for MRIB changes and advertising +/// locally originated multicast groups to DDM. +fn run(db: Db, log: Logger, rt: Arc, ddm: &impl Ddm) { + let mut rx = watch(&db); + loop { + if let Err(e) = sync_pending(&db, ddm, &log, &rt, &rx) { + error!(log, "MRIB sync failed: {e}"); + sleep(MRIB_SYNC_RETRY_DELAY); + continue; + } + + match rx.recv_timeout(MRIB_PERIODIC_SYNC_INTERVAL) { + Ok(_) | Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => { + error!(log, "MRIB watcher disconnected"); + rx = watch(&db); + } + } + } +} + +fn watch(db: &Db) -> Receiver { + let (tx, rx) = channel(); + // Registering under the same tag replaces any watcher left over from a + // previous registration. + db.watch_mrib(MG_LOWER_MRIB_TAG.into(), tx); + rx +} + +/// Drain all pending notifications and reconcile the current MRIB with DDM. +/// +/// `sync` reads the fully selected MRIB, discarding individual notifications. +/// +/// # Errors +/// +/// Propagates errors from `sync`. +fn sync_pending( + db: &Db, + ddm: &impl Ddm, + log: &Logger, + rt: &Arc, + rx: &Receiver, +) -> Result<(), String> { + rx.try_iter().for_each(std::mem::drop); + sync(db, ddm, log, rt) +} + +/// Synchronize MRIB origination to DDM. +/// +/// This compares the current MRIB (`loc_mrib`) with what DDM has advertised, +/// reconciling any differences. +/// +/// # Errors +/// +/// Returns an error if DDM's originated groups cannot be read. +pub(crate) fn sync( + db: &Db, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + let wanted: HashSet = db + .loc_mrib(None) + .values() + .filter_map(|route| match ddm_origin(route) { + Ok(origin) => Some(origin), + Err(e) => { + warn!(log, "skipping MRIB route {}: {e}", route.key); + None + } + }) + .collect(); + + let current: HashSet = rt + .block_on(ddm.get_originated_multicast_groups()) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + add_multicast_routes(ddm, wanted.difference(¤t), rt, log) + .map_err(|e| format!("failed to advertise multicast groups: {e}"))?; + remove_multicast_routes(ddm, current.difference(&wanted), rt, log) + .map_err(|e| format!("failed to withdraw multicast groups: {e}"))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::net::{Ipv4Addr, Ipv6Addr}; + use std::sync::mpsc::TryRecvError; + + use rdb::test::get_test_db; + use rdb::types::{ + MulticastAddr, MulticastRouteKey, MulticastSourceProtocol, + UnderlayMulticastIpv6, Vni, + }; + + use super::*; + use crate::platform::test::TestDdm; + + fn discard_logger() -> Logger { + Logger::root(slog::Discard, slog::o!()) + } + + /// Runtime handle for `sync`'s internal `block_on`. Tests run on a std + /// thread (not a tokio worker). + fn runtime() -> (tokio::runtime::Runtime, Arc) { + let rt = tokio::runtime::Runtime::new().expect("build runtime"); + let handle = Arc::new(rt.handle().clone()); + (rt, handle) + } + + fn vni() -> Vni { + Vni::new(77).expect("valid VNI") + } + + fn test_underlay() -> UnderlayMulticastIpv6 { + UnderlayMulticastIpv6::new(Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1)) + .expect("valid underlay address") + } + + fn origin(route: &MulticastRoute) -> MulticastOrigin { + ddm_origin(route).expect("valid origin") + } + + /// Build an any-source (*,G) static route for the given IPv4 group. + fn asm_route(a: u8, b: u8, c: u8, d: u8) -> MulticastRoute { + let group = MulticastAddr::new(Ipv4Addr::new(a, b, c, d).into()) + .expect("valid group"); + let key = MulticastRouteKey::any_source(group, vni()).expect("key"); + MulticastRoute::new( + key, + test_underlay(), + MulticastSourceProtocol::Static, + ) + } + + /// Build an any-source (*,G) static route for an IPv6 group in + /// `ff0e::/16`, with a distinct underlay (1:1 overlay-to-underlay + /// mapping). + fn asm_route_v6(last: u16) -> MulticastRoute { + let group = MulticastAddr::new( + Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, last).into(), + ) + .expect("valid group"); + let underlay = UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0x1, last, + )) + .expect("valid underlay address"); + let key = MulticastRouteKey::any_source(group, vni()).expect("key"); + MulticastRoute::new(key, underlay, MulticastSourceProtocol::Static) + } + + #[test] + fn sync_pending_reconciles_after_dropping_notifications() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_pending", log.clone()).expect("db"); + let rx = watch(&db); + let removed = asm_route(225, 1, 1, 1); + let added = asm_route_v6(1); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&removed)); + + db.add_static_mcast_routes(std::slice::from_ref(&removed)) + .expect("add initial route"); + db.add_static_mcast_routes(std::slice::from_ref(&added)) + .expect("add replacement route"); + db.remove_static_mcast_routes(&[removed.key]) + .expect("remove initial route"); + rx.try_recv().expect("queued change notification"); + + sync_pending(&db, &ddm, &log, &handle, &rx).expect("sync burst"); + + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + assert_eq!( + *ddm.multicast_originated.lock().unwrap(), + vec![origin(&added)] + ); + + db.remove_static_mcast_routes(&[added.key]) + .expect("remove remaining route"); + rx.try_recv().expect("subsequent change notification"); + + sync_pending(&db, &ddm, &log, &handle, &rx) + .expect("sync subsequent change"); + + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn sync_advertises_groups_missing_from_ddm() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(*originated, vec![origin(&route)]); + } + + #[test] + fn sync_advertises_v4_and_v6_overlay_groups() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_v6", log.clone()).expect("db"); + let v4 = asm_route(225, 1, 1, 1); + let v6 = asm_route_v6(1); + db.add_static_mcast_routes(&[v4.clone(), v6.clone()]) + .expect("add routes"); + + let ddm = TestDdm::default(); + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 2); + assert!(originated.contains(&origin(&v4))); + assert!(originated.contains(&origin(&v6))); + } + + #[test] + fn sync_withdraws_groups_absent_from_mrib() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_remove", log.clone()).expect("db"); + + let stale = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&stale)); + + sync(&db, &ddm, &log, &handle).expect("sync"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn sync_in_sync_makes_no_changes() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&route)); + + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + *originated, + vec![origin(&route)], + "in-sync group must not be re-added" + ); + } + + #[test] + fn sync_adds_and_withdraws_together() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_mixed", log.clone()).expect("db"); + let keep = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&keep)) + .expect("add route"); + + let ddm = TestDdm::default(); + let stale = asm_route(225, 2, 2, 2); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&stale)); + + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(*originated, vec![origin(&keep)]); + } + + #[test] + fn sync_withdraws_removed_route() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_withdraw", log.clone()).expect("db"); + let removed = asm_route(225, 1, 1, 1); + let other = asm_route(225, 2, 2, 2); + db.add_static_mcast_routes(&[removed.clone(), other.clone()]) + .expect("add routes"); + + let ddm = TestDdm::default(); + sync(&db, &ddm, &log, &handle).expect("sync"); + + db.remove_static_mcast_routes(&[removed.key]) + .expect("remove route"); + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + *originated, + vec![origin(&other)], + "unrelated group must remain advertised" + ); + } +} diff --git a/mg-lower/src/platform.rs b/mg-lower/src/platform.rs index 8350f245b..effe380ff 100644 --- a/mg-lower/src/platform.rs +++ b/mg-lower/src/platform.rs @@ -8,7 +8,7 @@ use std::net::IpAddr; use std::time::Duration; use ddm_admin_client::types::Error as DdmError; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use dpd_client::types::{Error as DpdError, *}; // Use the crate-specific re-exports of progenitor_client types to avoid // version conflicts: dpd_client re-exports from progenitor-client 0.11 (via @@ -217,6 +217,47 @@ pub trait Ddm { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, >; + + /// Get multicast group subscriptions originated by this router. + /// + /// Each `MulticastOrigin` pairs an overlay group address with its + /// underlay mapping (ff04::/64) along with an optional source for + /// (S,G) routes. + /// + /// Method names follow the DDM admin API convention + /// (`originated_multicast_groups`, not `originated_multicast_origins`). + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + >; + + /// Advertise multicast group subscriptions to DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; + + /// Withdraw multicast group subscriptions from DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; } /// This trait wraps the methods that have expectations about switch zone @@ -406,6 +447,35 @@ impl Ddm for ProductionDdm { > { self.client.withdraw_tunnel_endpoints(body).await } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + self.client.get_originated_multicast_groups().await + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.advertise_multicast_groups(body).await + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.withdraw_multicast_groups(body).await + } } /// Production switch zone that uses libnet for route lookups (illumos only). @@ -700,6 +770,7 @@ pub(crate) mod test { pub(crate) struct TestDdm { pub(crate) tunnel_originated: Mutex>, pub(crate) originated: Mutex>, + pub(crate) multicast_originated: Mutex>, } impl Default for TestDdm { @@ -707,6 +778,7 @@ pub(crate) mod test { Self { tunnel_originated: Mutex::new(Vec::default()), originated: Mutex::new(Vec::default()), + multicast_originated: Mutex::new(Vec::default()), } } } @@ -767,6 +839,45 @@ pub(crate) mod test { .retain(|x| !body.contains(x)); Ok(ddm_response_ok!(())) } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + Ok(ddm_response_ok!( + self.multicast_originated.lock().unwrap().clone() + )) + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.multicast_originated + .lock() + .unwrap() + .extend(body.clone()); + Ok(ddm_response_ok!(())) + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.multicast_originated + .lock() + .unwrap() + .retain(|x| !body.contains(x)); + Ok(ddm_response_ok!(())) + } } /// A mock switch zone implementation. diff --git a/mgd/src/main.rs b/mgd/src/main.rs index 15a65c2e6..f19bbc63a 100644 --- a/mgd/src/main.rs +++ b/mgd/src/main.rs @@ -179,14 +179,23 @@ async fn run(args: RunArgs) { client: mg_lower::new_ddm_client(&log), }; let sw = mg_lower::ProductionSwitchZone {}; - Builder::new() - .name("mg-lower".to_string()) - .spawn(move || { - mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); - }) + let backends = mg_lower::Backends { dpd, ddm, sw }; + mg_lower::spawn(ctx.tep, db, log, stats, rt, backends) .expect("failed to start mg-lower"); } + #[cfg(all(feature = "mg-lower", target_os = "illumos"))] + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let log = log.clone(); + let db = context.db.clone(); + let ddm = mg_lower::ProductionDdm { + client: mg_lower::new_ddm_client(&log), + }; + mg_lower::mrib::spawn(db, log, rt, ddm) + .expect("failed to start mg-lower mrib"); + } + start_bgp_routers( context.clone(), db.get_bgp_routers() diff --git a/openapi/ddm-admin/ddm-admin-5.0.0-658a80.json b/openapi/ddm-admin/ddm-admin-5.0.0-658a80.json new file mode 100644 index 000000000..1140bc8fb --- /dev/null +++ b/openapi/ddm-admin/ddm-admin-5.0.0-658a80.json @@ -0,0 +1,965 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "DDM Admin", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "5.0.0" + }, + "paths": { + "/disable-stats": { + "post": { + "operationId": "disable_stats", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/enable-stats": { + "post": { + "operationId": "enable_stats", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnableStatsRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/external_peers": { + "get": { + "operationId": "get_external_peers", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExternalPeers" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "set_external_peers", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExternalPeers" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/multicast_group": { + "put": { + "operationId": "advertise_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/multicast_groups": { + "get": { + "operationId": "get_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_ImportedMulticastRoute", + "type": "array", + "items": { + "$ref": "#/components/schemas/ImportedMulticastRoute" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/originated": { + "get": { + "operationId": "get_originated", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_Ipv6Net", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/originated_multicast_groups": { + "get": { + "operationId": "get_originated_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/originated_tunnel_endpoints": { + "get": { + "operationId": "get_originated_tunnel_endpoints", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelOrigin" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/peers": { + "get": { + "operationId": "get_peers", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_PeerInfo", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/PeerInfo" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/peers/{addr}": { + "delete": { + "operationId": "expire_peer", + "parameters": [ + { + "in": "path", + "name": "addr", + "required": true, + "schema": { + "type": "string", + "format": "ipv6" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/prefix": { + "put": { + "operationId": "advertise_prefixes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_Ipv6Net", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_prefixes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_Ipv6Net", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/prefixes": { + "get": { + "operationId": "get_prefixes", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_Set_of_PathVector", + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PathVector" + }, + "uniqueItems": true + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/sync": { + "put": { + "operationId": "sync", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/tunnel_endpoint": { + "put": { + "operationId": "advertise_tunnel_endpoints", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_tunnel_endpoints", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/tunnel_endpoints": { + "get": { + "operationId": "get_tunnel_endpoints", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelRoute", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelRoute" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Duration": { + "type": "object", + "properties": { + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "nanos", + "secs" + ] + }, + "EnableStatsRequest": { + "type": "object", + "properties": { + "rack_id": { + "type": "string", + "format": "uuid" + }, + "sled_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "rack_id", + "sled_id" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "ExternalMulticastIp": { + "description": "A validated IP address for a customer-visible overlay multicast group.\n\nThe overlay multicast group is the address guests can send to and join, e.g. `233.252.0.1` or `ff0e::1`, and spans both address families. Its rack-internal underlay mapping is validated separately via `UnderlayMulticastIpv6`, with a 1:1 mapping between the two.\n\nJSON uses the ordinary IP-address string form. Deserialization rejects addresses that are not multicast, use a reserved, interface-local, or link-local IPv6 scope, fall in the IPv4 local network control block (`224.0.0.0/24`) or the `ff04::/64` underlay subnet, are a reserved base or SSM null address, or carry malformed prefix-based or embedded-RP flags.", + "type": "string", + "format": "ip" + }, + "ExternalPeers": { + "type": "object", + "properties": { + "address_objects": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": [ + "address_objects" + ] + }, + "ImportedMulticastRoute": { + "description": "A multicast route with the interface on which its peer was discovered.", + "type": "object", + "properties": { + "if_name": { + "description": "Local interface on which the advertising peer was discovered.", + "type": "string" + }, + "route": { + "description": "The imported multicast route.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastRoute" + } + ] + } + }, + "required": [ + "if_name", + "route" + ] + }, + "IpNet": { + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::IpNet", + "version": "0.1.0" + }, + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Net" + } + ] + } + ] + }, + "Ipv4Net": { + "example": "192.168.1.0/24", + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and prefix length", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv4Net", + "version": "0.1.0" + }, + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$" + }, + "Ipv6Net": { + "example": "fd12:3456::/64", + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv6Net", + "version": "0.1.0" + }, + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" + }, + "MulticastOrigin": { + "description": "The multicast group origin as exchanged between DDM peers.", + "type": "object", + "properties": { + "metric": { + "description": "The metric for path selection. Among paths of equal length, a lower value is preferred.", + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "overlay_group": { + "description": "The overlay multicast group address (IPv4 or IPv6).", + "allOf": [ + { + "$ref": "#/components/schemas/ExternalMulticastIp" + } + ] + }, + "source": { + "nullable": true, + "description": "Optional source address for Source-Specific Multicast `(S,G)` routes. `None` for Any-Source Multicast `(*,G)` routes.", + "default": null, + "type": "string", + "format": "ip" + }, + "underlay_group": { + "description": "The underlay multicast group address that the overlay group maps to (1:1).", + "allOf": [ + { + "$ref": "#/components/schemas/UnderlayMulticastIpv6" + } + ] + }, + "vni": { + "description": "VNI identifying the VPC/network context for this group.", + "type": "integer", + "format": "uint32", + "minimum": 0, + "maximum": 16777215 + } + }, + "required": [ + "overlay_group", + "underlay_group", + "vni" + ] + }, + "MulticastPathHop": { + "description": "A single hop in the multicast path.\n\n`router_id` identifies the hop for loop detection, and `underlay_addr` is the address replication toward this router targets.", + "type": "object", + "properties": { + "router_id": { + "description": "Router identifier (i.e., the ddmd router ID, which defaults to the hostname).", + "type": "string" + }, + "underlay_addr": { + "description": "The underlay address of this router (for replication targeting).", + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "router_id", + "underlay_addr" + ] + }, + "MulticastRoute": { + "description": "A multicast route learned via DDM.\n\nCarries a `MulticastOrigin` (overlay group + ff04::/64 underlay mapping) and the path vector from the originating subscriber through intermediate transit routers.", + "type": "object", + "properties": { + "nexthop": { + "description": "Underlay nexthop address (DDM peer that advertised this route). Used to associate the route with a peer for expiration.", + "type": "string", + "format": "ipv6" + }, + "origin": { + "description": "The multicast group origin information.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastOrigin" + } + ] + }, + "path": { + "description": "Path vector from the originating subscriber outward. Each hop records the router that redistributed this subscription announcement. Used for loop detection on pull and for future replication optimization in multi-rack topologies.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastPathHop" + } + } + }, + "required": [ + "nexthop", + "origin" + ] + }, + "PathVector": { + "type": "object", + "properties": { + "destination": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "destination", + "path" + ] + }, + "PeerInfo": { + "description": "Peer information with an optional interface name.", + "type": "object", + "properties": { + "addr": { + "type": "string", + "format": "ipv6" + }, + "host": { + "type": "string" + }, + "if_name": { + "nullable": true, + "description": "Interface name the peer was discovered on (e.g., \"tfportrear0_0\").", + "default": null, + "type": "string" + }, + "kind": { + "$ref": "#/components/schemas/RouterKind" + }, + "status": { + "$ref": "#/components/schemas/PeerStatus" + } + }, + "required": [ + "addr", + "host", + "kind", + "status" + ] + }, + "PeerStatus": { + "description": "Status of a DDM peer, including how long the peer has been in its current state.", + "oneOf": [ + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Init" + ] + } + }, + "required": [ + "duration", + "type" + ] + }, + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Solicit" + ] + } + }, + "required": [ + "duration", + "type" + ] + }, + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Exchange" + ] + } + }, + "required": [ + "duration", + "type" + ] + }, + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Expired" + ] + } + }, + "required": [ + "duration", + "type" + ] + } + ] + }, + "RouterKind": { + "type": "integer", + "enum": [ + 0, + 1 + ] + }, + "TunnelOrigin": { + "type": "object", + "properties": { + "boundary_addr": { + "type": "string", + "format": "ipv6" + }, + "metric": { + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "overlay_prefix": { + "$ref": "#/components/schemas/IpNet" + }, + "vni": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "boundary_addr", + "overlay_prefix", + "vni" + ] + }, + "TunnelRoute": { + "type": "object", + "properties": { + "nexthop": { + "type": "string", + "format": "ipv6" + }, + "origin": { + "$ref": "#/components/schemas/TunnelOrigin" + } + }, + "required": [ + "nexthop", + "origin" + ] + }, + "UnderlayMulticastIpv6": { + "description": "A validated underlay multicast IPv6 address.\n\nUnderlay multicast addresses must be within the subnet allocated by Omicron for rack-internal multicast traffic (ff04::/64). This is a subset of the admin-local scope (ff04::/16) defined in RFC 4291.", + "type": "string", + "format": "ipv6" + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index cc34c49e0..d49eb9945 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-4.0.0-27a7bd.json \ No newline at end of file +ddm-admin-5.0.0-658a80.json \ No newline at end of file diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 682efcce7..4f95d7976 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -8,6 +8,8 @@ client-common.workspace = true zone = { git = "https://github.com/oxidecomputer/zone" } ddm-admin-client = { path = "../ddm-admin-client" } ddm-api-types-versions.workspace = true +dpd-client.workspace = true +dpd-types-versions.workspace = true anyhow.workspace = true slog.workspace = true diff --git a/tests/conf/dpd-ports-dual.toml b/tests/conf/dpd-ports-dual.toml new file mode 100644 index 000000000..17777befc --- /dev/null +++ b/tests/conf/dpd-ports-dual.toml @@ -0,0 +1,17 @@ +[int0] +fec = "None" +speed = "Speed10G" +autoneg = true +kr = true + +[rear0] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true + +[rear1] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true diff --git a/tests/conf/dpd-ports-sextet.toml b/tests/conf/dpd-ports-sextet.toml new file mode 100644 index 000000000..b513cbb8a --- /dev/null +++ b/tests/conf/dpd-ports-sextet.toml @@ -0,0 +1,31 @@ +[int0] +fec = "None" +speed = "Speed10G" +autoneg = true +kr = true + +[rear0] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true + +[rear1] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true + +[qsfp0] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true +allow_ddm_traffic = true + +[qsfp1] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true +allow_ddm_traffic = true diff --git a/tests/conf/softnpu-dual-sidecar_a.dual.toml b/tests/conf/softnpu-dual-sidecar_a.dual.toml new file mode 100644 index 000000000..3a277802c --- /dev/null +++ b/tests/conf/softnpu-dual-sidecar_a.dual.toml @@ -0,0 +1,5 @@ +p4_program = "/opt/libsidecar_lite.so" +ports = [ + { sidecar = "sw0", scrimlet = "sr0", mtu = 1524 }, + { sidecar = "sw1", scrimlet = "sr1", mtu = 1524 }, +] diff --git a/tests/conf/softnpu-dual-sidecar_b.dual.toml b/tests/conf/softnpu-dual-sidecar_b.dual.toml new file mode 100644 index 000000000..fd2cce475 --- /dev/null +++ b/tests/conf/softnpu-dual-sidecar_b.dual.toml @@ -0,0 +1,5 @@ +p4_program = "/opt/libsidecar_lite.so" +ports = [ + { sidecar = "sw2", scrimlet = "sr2", mtu = 1524 }, + { sidecar = "sw3", scrimlet = "sr3", mtu = 1524 }, +] diff --git a/tests/src/ddm.rs b/tests/src/ddm.rs index 3d61175f7..642019ea4 100644 --- a/tests/src/ddm.rs +++ b/tests/src/ddm.rs @@ -8,14 +8,21 @@ use ddm_admin_client::Client; use ddm_api_types_versions::latest::db::PeerStatus; use ddm_api_types_versions::latest::external_peers::ExternalPeers; use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin, Vni}; +use dpd_types_versions::latest::link::LinkId; +use dpd_types_versions::latest::mcast::{ + Direction, MulticastGroupCreateUnderlayEntry, MulticastGroupMember, + MulticastGroupUnderlayResponse, +}; use oxnet::Ipv6Net; use slog::{Drain, Logger}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::env; use std::net::Ipv6Addr; use std::ops::{Deref, DerefMut}; use std::thread::sleep; use std::time::{Duration, Instant}; +use tokio::sync::oneshot; +use tokio::sync::oneshot::error::TryRecvError; use zone::Zlogin; use ztest::*; @@ -56,12 +63,88 @@ macro_rules! ip6_net { }; } +type MulticastImports = HashSet<(String, MulticastOrigin, u64)>; + +macro_rules! assert_client_state { + ($client:expr, $read:expr, $expected:expr, $period:expr, $count:expr) => {{ + let expected = $expected; + let mut actual = $read.await; + for _ in 0..$count { + if actual.as_ref().ok() == Some(&expected) { + break; + } + sleep(Duration::from_secs($period)); + actual = $read.await; + } + if actual.as_ref().ok() != Some(&expected) { + eprintln_nopipe!( + "{}: expected {expected:#?}, last response: {actual:#?}", + stringify!($client), + ); + dump_client_state(stringify!($client), $client).await; + } + assert_eq!(actual.ok(), Some(expected)); + }}; + ($client:expr, $read:expr, $expected:expr) => { + assert_client_state!($client, $read, $expected, 1, 10) + }; +} + +macro_rules! assert_multicast_imports { + ($client:expr, [$(($iface:expr, $origin:expr)),* $(,)?] + $(, $period:expr, $count:expr)?) => {{ + let expected: MulticastImports = + [$(($iface.to_string(), ($origin).clone(), ($origin).metric())),*] + .into_iter() + .collect(); + assert_client_state!( + $client, + multicast_imports($client), + expected + $(, $period, $count)? + ); + }}; +} + +macro_rules! assert_multicast_origins { + ($client:expr, [$($origin:expr),* $(,)?]) => {{ + let expected: HashSet<(MulticastOrigin, u64)> = + [$((($origin).clone(), ($origin).metric())),*] + .into_iter() + .collect(); + assert_client_state!($client, multicast_origins($client), expected); + }}; +} + const ZONE_BRAND: &str = "omicron1"; /// How long to wait for a stopped ddmd to exit, and how often to check. const DDMD_STOP_TIMEOUT: Duration = Duration::from_secs(30); const DDMD_STOP_POLL: Duration = Duration::from_millis(100); +const MULTICAST_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30); +const MULTICAST_POLL_INTERVAL: Duration = Duration::from_millis(250); + +/// The VNI shared by every multicast origin these tests advertise. +const MULTICAST_VNI: u32 = 77; + +/// The duration for holding multicast members steady after an import change. +const MEMBER_HOLD: Duration = Duration::from_secs(3); + +/// How often the failover observer samples, less coarse than +/// [`MULTICAST_POLL_INTERVAL`] to narrow the window a flap could slip by. +const FAILOVER_POLL_INTERVAL: Duration = Duration::from_millis(100); +/// How long the failover observer keeps watching after the transit converges. +const FAILOVER_SETTLE: Duration = Duration::from_secs(3); + +/// DPD API version for the raw underlay-group requests. +/// +/// This is the version the pinned `dpd-client` is generated from, matching +/// both what `ddmd` sends out and the `latest` types these requests use. +fn dpd_api_version() -> &'static str { + >::api_version() +} + struct SoftnpuZone<'a> { zfs: &'a Zfs, zone: Zone, @@ -496,10 +579,6 @@ macro_rules! run_topo { } #[tokio::test] -async fn test_trio_v3() -> Result<()> { - test_trio().await -} - async fn test_trio() -> Result<()> { // A trio. Two server routers and one transit router. // @@ -580,10 +659,9 @@ async fn run_trio_tests( let s2 = Client::new("http://10.0.0.2:8000", log.clone()); let t1 = Client::new("http://10.0.0.3:8000", log.clone()); - // If we never get a response from a server, return 99 as a sentinel value. - wait_for_eq!(s1.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(s2.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(t1.get_peers().await.map_or(99, |x| x.len()), 2); + assert_client_state!(&s1, peer_count(&s1), 1); + assert_client_state!(&s2, peer_count(&s2), 1); + assert_client_state!(&t1, peer_count(&t1), 2); println_nopipe!("initial peering test passed"); @@ -767,14 +845,14 @@ async fn run_trio_tests( println_nopipe!("tunnel endpoint withdraw passed"); - run_trio_mcast(zs1, &s1, &s2, &t1, &log).await + run_trio_mcast(zs1, zt1, &s1, &s2, &t1, &log).await } /// Multicast group advertise/withdraw across the trio. /// -/// This emulates how mg-lower in the switch zone publishes +/// This emulates how mg-lower operates in the switch zone when publishing /// overlay→underlay multicast bindings: the transit router -/// originates an advertisement, and the server routers learn it via +/// originates an advertisement and the server routers learn it via /// the DDM exchange. /// /// ```text @@ -785,26 +863,32 @@ async fn run_trio_tests( /// (O4) /// ``` /// -/// - t1 originates one IPv4 and one IPv6 group, and both servers import -/// them. t1 imports nothing back (a re-import would be a loop). -/// - s1 restarts and relearns both groups from t1. -/// - t1 withdraws, and both servers clear. -/// - s1 originates O4. t1 redistributes it to s2, and s1 imports nothing -/// in return. s1 withdraws, and t1 and s2 clear. +/// 1. t1 originates an IPv4 and an IPv6 group, and both servers import +/// them. t1 imports nothing in return (a re-import would cause a loop). +/// 2. s1 restarts and re-learns both groups from t1. +/// 3. t1 withdraws, and both servers clear. +/// 4. s1 originates O4. t1 redistributes it to s2, s1 imports nothing in +/// return, and t1 programs rear0 for O4's underlay group. +/// 5. s2 also subscribes to O4. t1 programs rear0 and rear1, then rear1 once +/// s1 withdraws, and drains the group when s2 withdraws. +/// 6. s1 subscribes to an origin whose underlay group doesn't exist yet. The +/// group is still missing until it is created, after which the +/// periodic sweep programs rear0. s1 withdraws, and the group drains. async fn run_trio_mcast( zs1: &RouterZone<'_>, + zt1: &RouterZone<'_>, s1: &Client, s2: &Client, t1: &Client, log: &Logger, ) -> Result<()> { - wait_for_eq!(multicast_originated_count(t1).await?, 0); + assert_multicast_origins!(t1, []); // Cover advertisement and withdrawal for both IP families. let mcast_origin_v4 = MulticastOrigin::new( "233.252.0.1".parse().unwrap(), "ff04::100".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); @@ -812,32 +896,48 @@ async fn run_trio_mcast( let mcast_origin_v6 = MulticastOrigin::new( "ff0e::1".parse().unwrap(), "ff04::101".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); + let ipv4_origin_underlay = create_multicast_underlay_group( + zt1, + mcast_origin_v4.underlay_group().into(), + )?; + let s1_member = [underlay_member("rear0")]; + let s2_member = [underlay_member("rear1")]; + t1.advertise_multicast_groups(&vec![ mcast_origin_v4.clone(), mcast_origin_v6.clone(), ]) .await?; - wait_for_eq!(multicast_originated_count(t1).await?, 2); - wait_for_eq!(multicast_group_count(s1).await?, 2); - wait_for_eq!(multicast_group_count(s2).await?, 2); + assert_multicast_origins!(t1, [mcast_origin_v4, mcast_origin_v6]); + assert_multicast_imports!( + s1, + [("sl0", mcast_origin_v4), ("sl0", mcast_origin_v6)] + ); + assert_multicast_imports!( + s2, + [("sl1", mcast_origin_v4), ("sl1", mcast_origin_v6)] + ); // A looped announcement bouncing back from a server would show up here. - assert_eq!(multicast_group_count(t1).await?, 0); + assert_eq!(multicast_imports(t1).await?, MulticastImports::new()); println_nopipe!("multicast group advertise passed"); - // Server router restart: s1's view of the multicast group must - // converge again after ddmd restarts. `wait_for_eq` tolerates the - // restart window via the `unwrap_or` sentinel. + // Server router restart: s1's view of the multicast group must converge + // again after ddmd restarts. `assert_multicast_imports!` retries through + // the restart window. zs1.stop_router()?; zs1.start_router(false)?; let s1 = Client::new("http://10.0.0.1:8000", log.clone()); - wait_for_eq!(multicast_group_count(&s1).await.unwrap_or(99), 2); + assert_multicast_imports!( + &s1, + [("sl0", mcast_origin_v4), ("sl0", mcast_origin_v6)] + ); println_nopipe!("multicast router restart passed"); @@ -847,31 +947,395 @@ async fn run_trio_mcast( ]) .await?; - wait_for_eq!(multicast_originated_count(t1).await?, 0); + assert_multicast_origins!(t1, []); // A periodic pull can also clear the routes within this deadline. - wait_for_eq!(multicast_group_count(&s1).await?, 0, 1, 5); - wait_for_eq!(multicast_group_count(s2).await?, 0, 1, 5); - assert_eq!(multicast_group_count(t1).await?, 0); + assert_multicast_imports!(&s1, [], 1, 5); + assert_multicast_imports!(s2, [], 1, 5); + assert_eq!(multicast_imports(t1).await?, MulticastImports::new()); println_nopipe!("multicast group withdraw passed"); s1.advertise_multicast_groups(&vec![mcast_origin_v4.clone()]) .await?; - wait_for_eq!(multicast_originated_count(&s1).await?, 1); - wait_for_eq!(multicast_group_count(t1).await?, 1); - wait_for_eq!(multicast_group_count(s2).await?, 1); - assert_eq!(multicast_group_count(&s1).await?, 0); + assert_multicast_origins!(&s1, [mcast_origin_v4]); + assert_multicast_imports!(t1, [("tfportrear0_0", mcast_origin_v4)]); + assert_multicast_imports!(s2, [("sl1", mcast_origin_v4)]); + assert_eq!(multicast_imports(&s1).await?, MulticastImports::new()); + + wait_for_multicast_members(zt1, &ipv4_origin_underlay, &s1_member)?; println_nopipe!("multicast server-originated redistribution passed"); - s1.withdraw_multicast_groups(&vec![mcast_origin_v4]).await?; + let origin = mcast_origin_v4; + let group = &ipv4_origin_underlay; + s2.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + wait_for_multicast_members( + zt1, + group, + &[s1_member[0].clone(), s2_member[0].clone()], + )?; - wait_for_eq!(multicast_originated_count(&s1).await?, 0); - wait_for_eq!(multicast_group_count(t1).await?, 0, 1, 5); - wait_for_eq!(multicast_group_count(s2).await?, 0, 1, 5); + s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_origins!(&s1, []); + assert_multicast_imports!(t1, [("tfportrear1_0", origin)], 1, 5); + wait_for_multicast_members(zt1, group, &s2_member)?; + + s2.withdraw_multicast_groups(&vec![origin]).await?; + assert_multicast_imports!(t1, [], 1, 5); + assert_multicast_imports!(&s1, [], 1, 5); + assert_multicast_imports!(s2, [], 1, 5); + wait_for_multicast_members(zt1, group, &[])?; + + println_nopipe!("multicast underlay replication members passed"); + + let delayed_origin = MulticastOrigin::new( + "233.252.0.2".parse().unwrap(), + "ff04::102".parse().unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), + None, + ) + .unwrap(); + let delayed_group_ip: Ipv6Addr = delayed_origin.underlay_group().into(); + + s1.advertise_multicast_groups(&vec![delayed_origin.clone()]) + .await?; + assert_multicast_imports!(t1, [("tfportrear0_0", delayed_origin)]); + assert_multicast_imports!(s2, [("sl1", delayed_origin)]); + assert_multicast_underlay_group_missing(zt1, delayed_group_ip)?; - println_nopipe!("multicast server-originated withdraw passed"); + let delayed_group = create_multicast_underlay_group(zt1, delayed_group_ip)?; + wait_for_multicast_members(zt1, &delayed_group, &s1_member)?; + + s1.withdraw_multicast_groups(&vec![delayed_origin]).await?; + assert_multicast_imports!(t1, []); + wait_for_multicast_members(zt1, &delayed_group, &[])?; + + println_nopipe!("multicast delayed underlay group creation passed"); + + Ok(()) +} + +fn assert_multicast_underlay_group_missing( + zone: &RouterZone<'_>, + group_ip: Ipv6Addr, +) -> Result<()> { + let version = dpd_api_version(); + let status = zone.zexec(&format!( + "curl --noproxy '*' --globoff --silent --show-error \ + --max-time 5 --output /dev/null --write-out '%{{http_code}}' \ + --header 'api-version: {version}' \ + 'http://[::1]:12224/multicast/underlay-groups/{group_ip}'" + ))?; + if status.trim() != "404" { + return Err(anyhow!( + "underlay group {group_ip}: expected HTTP 404, got {status:?}" + )); + } + Ok(()) +} + +fn create_multicast_underlay_group( + zone: &RouterZone<'_>, + group_ip: Ipv6Addr, +) -> Result { + create_multicast_underlay_group_with(zone, group_ip, &[]) +} + +fn create_multicast_underlay_group_with( + zone: &RouterZone<'_>, + group_ip: Ipv6Addr, + members: &[MulticastGroupMember], +) -> Result { + let body = serde_json::to_string(&MulticastGroupCreateUnderlayEntry { + group_ip: group_ip.try_into()?, + tag: Some(format!("{}:{group_ip}", zone.zone.name)), + members: members.to_vec(), + })? + .replace('\'', "'\\''"); + let version = dpd_api_version(); + let response = zone.zexec(&format!( + "curl --noproxy '*' --globoff --fail --silent --show-error \ + --max-time 5 --request POST \ + --header 'api-version: {version}' \ + --header 'Content-Type: application/json' --data '{body}' \ + 'http://[::1]:12224/multicast/underlay-groups'" + ))?; + Ok(serde_json::from_str(&response)?) +} + +fn get_multicast_underlay_group( + zone: &RouterZone<'_>, + group: &MulticastGroupUnderlayResponse, +) -> Result { + let version = dpd_api_version(); + let body = zone.zexec(&format!( + "curl --noproxy '*' --globoff --fail --silent --show-error \ + --max-time 5 --header 'api-version: {version}' \ + 'http://[::1]:12224/multicast/underlay-groups/{}'", + group.group_ip, + ))?; + Ok(serde_json::from_str(&body)?) +} + +fn same_group( + actual: &MulticastGroupUnderlayResponse, + group: &MulticastGroupUnderlayResponse, +) -> bool { + actual.group_ip == group.group_ip + && actual.tag == group.tag + && actual.external_group_id == group.external_group_id + && actual.underlay_group_id == group.underlay_group_id +} + +fn same_membership( + actual: &[MulticastGroupMember], + members: &[MulticastGroupMember], +) -> bool { + actual.len() == members.len() + && members.iter().all(|member| actual.contains(member)) +} + +fn wait_for_multicast_members( + zone: &RouterZone<'_>, + group: &MulticastGroupUnderlayResponse, + members: &[MulticastGroupMember], +) -> Result<()> { + let start = Instant::now(); + loop { + let response = get_multicast_underlay_group(zone, group); + if let Ok(actual) = &response + && same_group(actual, group) + && same_membership(&actual.members, members) + { + return Ok(()); + } + if start.elapsed() >= MULTICAST_CONVERGENCE_TIMEOUT { + return Err(anyhow!( + "underlay group {} did not converge to {members:?}; \ + last response: {response:?}", + group.group_ip, + )); + } + sleep(MULTICAST_POLL_INTERVAL); + } +} + +fn assert_multicast_members_unchanged( + zone: &RouterZone<'_>, + group: &MulticastGroupUnderlayResponse, + members: &[MulticastGroupMember], + window: Duration, +) -> Result<()> { + let start = Instant::now(); + while start.elapsed() < window { + let actual = get_multicast_underlay_group(zone, group)?; + if !same_group(&actual, group) + || !same_membership(&actual.members, members) + { + return Err(anyhow!( + "underlay group {} left {members:?} within {window:?}: \ + {actual:?}", + group.group_ip, + )); + } + sleep(MULTICAST_POLL_INTERVAL); + } + Ok(()) +} + +#[tokio::test] +async fn test_dual_sidecar() -> Result<()> { + // sidecar A sidecar B + // t1 rear0 --- sr0 | sw0 --- sl0 s1 sl1 --- sw2 | sr2 --- rear0 t2 + // t1 rear1 --- sr1 | sw1 --- sl2 s2 sl3 --- sw3 | sr3 --- rear1 t2 + let tra_sr_0 = SimnetLink::new("tra0", "sr0")?; + let tra_sr_1 = SimnetLink::new("tra1", "sr1")?; + let trb_sr_2 = SimnetLink::new("trb0", "sr2")?; + let trb_sr_3 = SimnetLink::new("trb1", "sr3")?; + + let sl0_sw0 = SimnetLink::new("sl0", "sw0")?; + let sl2_sw1 = SimnetLink::new("sl2", "sw1")?; + let sl1_sw2 = SimnetLink::new("sl1", "sw2")?; + let sl3_sw3 = SimnetLink::new("sl3", "sw3")?; + + let mgmt0 = Etherstub::new("mgmt0")?; + let mg0 = Vnic::new("mg0", &mgmt0.name)?; + let mgd1 = Vnic::new("mgd1", &mgmt0.name)?; + let mgd2 = Vnic::new("mgd2", &mgmt0.name)?; + let mgd3 = Vnic::new("mgd3", &mgmt0.name)?; + let mgd4 = Vnic::new("mgd4", &mgmt0.name)?; + + let _mgip = Ip::new("10.0.0.254/24", &mg0.name, "test")?; + + let zfs = Zfs::new("mgdual")?; + + let sidecar_a = SoftnpuZone::new( + "sidecar_a.dual", + &zfs, + &[ + &tra_sr_0.end_b, + &tra_sr_1.end_b, + &sl0_sw0.end_b, + &sl2_sw1.end_b, + ], + "dual", + )?; + let sidecar_b = SoftnpuZone::new( + "sidecar_b.dual", + &zfs, + &[ + &trb_sr_2.end_b, + &trb_sr_3.end_b, + &sl1_sw2.end_b, + &sl3_sw3.end_b, + ], + "dual", + )?; + + println_nopipe!("start zone s1"); + let s1 = RouterZone::server( + "s1.dual", + &zfs, + &mgd1.name, + &[&sl0_sw0.end_a, &sl1_sw2.end_a], + )?; + println_nopipe!("start zone s2"); + let s2 = RouterZone::server( + "s2.dual", + &zfs, + &mgd2.name, + &[&sl2_sw1.end_a, &sl3_sw3.end_a], + )?; + + println_nopipe!("start zone t1"); + let mut t1 = RouterZone::transit( + "t1.dual", + &zfs, + &mgd3.name, + &[&tra_sr_0.end_a, &tra_sr_1.end_a], + "dual", + "sidecar_a.dual", + )?; + t1.set_port_map(BTreeMap::from([ + ("tra0".into(), "tfportrear0_0".into()), + ("tra1".into(), "tfportrear1_0".into()), + ])); + + println_nopipe!("start zone t2"); + let mut t2 = RouterZone::transit( + "t2.dual", + &zfs, + &mgd4.name, + &[&trb_sr_2.end_a, &trb_sr_3.end_a], + "dual", + "sidecar_b.dual", + )?; + t2.set_port_map(BTreeMap::from([ + ("trb0".into(), "tfportrear0_0".into()), + ("trb1".into(), "tfportrear1_0".into()), + ])); + + println_nopipe!("waiting for zones to come up"); + sleep(Duration::from_secs(10)); + + sidecar_a.setup()?; + sidecar_b.setup()?; + s1.setup(1)?; + s2.setup(2)?; + t1.setup(3)?; + t2.setup(4)?; + + run_topo!(run_dual_sidecar_mcast(&s1, &t1, &t2).await) +} + +async fn run_dual_sidecar_mcast( + zs1: &RouterZone<'_>, + zt1: &RouterZone<'_>, + zt2: &RouterZone<'_>, +) -> Result<()> { + let log = init_logger(); + let s1 = Client::new("http://10.0.0.1:8000", log.clone()); + let s2 = Client::new("http://10.0.0.2:8000", log.clone()); + let t1 = Client::new("http://10.0.0.3:8000", log.clone()); + let t2 = Client::new("http://10.0.0.4:8000", log.clone()); + + assert_client_state!(&s1, peer_count(&s1), 2); + assert_client_state!(&s2, peer_count(&s2), 2); + assert_client_state!(&t1, peer_count(&t1), 2); + assert_client_state!(&t2, peer_count(&t2), 2); + + println_nopipe!("dual sidecar peering passed"); + + let origin = MulticastOrigin::new( + "233.252.0.20".parse().unwrap(), + "ff04::120".parse().unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), + None, + ) + .unwrap(); + let group_ip: Ipv6Addr = origin.underlay_group().into(); + let group_a = create_multicast_underlay_group(zt1, group_ip)?; + let group_b = create_multicast_underlay_group(zt2, group_ip)?; + let s1_member = [underlay_member("rear0")]; + let s2_member = [underlay_member("rear1")]; + let both = [s1_member[0].clone(), s2_member[0].clone()]; + + s1.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(&t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(&t2, [("tfportrear0_0", origin)]); + wait_for_multicast_members(zt1, &group_a, &s1_member)?; + wait_for_multicast_members(zt2, &group_b, &s1_member)?; + + s2.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!( + &t1, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + assert_multicast_imports!( + &t2, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + wait_for_multicast_members(zt1, &group_a, &both)?; + wait_for_multicast_members(zt2, &group_b, &both)?; + assert_multicast_imports!(&s1, [("sl0", origin), ("sl1", origin)]); + + println_nopipe!("dual sidecar subscription passed"); + + zt2.stop_router()?; + zt2.start_router(false)?; + let t2 = Client::new("http://10.0.0.4:8000", log.clone()); + assert_multicast_imports!( + &t2, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + assert_multicast_members_unchanged(zt2, &group_b, &both, MEMBER_HOLD)?; + + println_nopipe!("dual sidecar transit restart passed"); + + zs1.zexec("ipadm delete-addr sl1/v6")?; + assert_client_state!(&s1, exchange_peer_count(&s1), 1, 1, 30); + assert_multicast_imports!(&t2, [("tfportrear1_0", origin)], 1, 30); + assert_multicast_imports!(&s1, [("sl0", origin)], 1, 30); + wait_for_multicast_members(zt2, &group_b, &s2_member)?; + assert_multicast_members_unchanged(zt1, &group_a, &both, MEMBER_HOLD)?; + + zs1.zexec("ipadm create-addr -t -T addrconf sl1/v6")?; + assert_client_state!(&s1, exchange_peer_count(&s1), 2, 1, 30); + assert_multicast_imports!( + &t2, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)], + 1, + 30 + ); + wait_for_multicast_members(zt2, &group_b, &both)?; + assert_multicast_imports!(&s1, [("sl0", origin), ("sl1", origin)], 1, 30); + + println_nopipe!("dual sidecar link failover passed"); Ok(()) } @@ -980,11 +1444,10 @@ async fn run_quartet_tests( let s3 = Client::new("http://10.0.0.3:8000", log.clone()); let t1 = Client::new("http://10.0.0.4:8000", log.clone()); - // If we never get a response from a server, return 99 as a sentinel value. - wait_for_eq!(s1.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(s2.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(s3.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(t1.get_peers().await.map_or(99, |x| x.len()), 3); + assert_client_state!(&s1, peer_count(&s1), 1); + assert_client_state!(&s2, peer_count(&s2), 1); + assert_client_state!(&s3, peer_count(&s3), 1); + assert_client_state!(&t1, peer_count(&t1), 3); println_nopipe!("initial peering test passed"); @@ -1038,8 +1501,8 @@ async fn run_quartet_tests( /// (origin O) /// ``` /// -/// - s1 withdraws origin O. t1 still reaches O via s2. s3 holds its route -/// through a 3 s window, shorter than the 10 s periodic pull. +/// - s1 withdraws origin O. t1 still reaches O via s2. The observer checks +/// s3's route through t1's convergence and for [`FAILOVER_SETTLE`] afterward. /// - s1 still holds s2's copy of O via t1, and s2 drops the copy it /// learned from s1. /// - s2 withdraws O, and t1, s1, and s3 clear. @@ -1052,7 +1515,7 @@ async fn run_quartet_mcast_failover( let origin = MulticastOrigin::new( "233.252.0.10".parse().unwrap(), "ff04::110".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); @@ -1060,34 +1523,56 @@ async fn run_quartet_mcast_failover( s1.advertise_multicast_groups(&vec![origin.clone()]).await?; s2.advertise_multicast_groups(&vec![origin.clone()]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 2); - wait_for_eq!(multicast_group_count(s1).await?, 1); - wait_for_eq!(multicast_group_count(s2).await?, 1); - wait_for_eq!(multicast_group_count(s3).await?, 1); + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + assert_multicast_imports!(s1, [("sl0", origin)]); + assert_multicast_imports!(s2, [("sl1", origin)]); + assert_multicast_imports!(s3, [("sl2", origin)]); println_nopipe!("multicast dual origination passed"); - s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 1); - - let deadline = Instant::now() + Duration::from_secs(3); - while Instant::now() < deadline { - let count = multicast_group_count(s3).await?; - if count != 1 { - return Err(anyhow!( - "s3 lost the group after s1 withdrew (count {count})" - )); - } - sleep(Duration::from_millis(100)); - } + let expected = MulticastImports::from([( + "sl2".to_string(), + origin.clone(), + origin.metric(), + )]); + let expected_transit = MulticastImports::from([( + "tfportrear1_0".to_string(), + origin.clone(), + origin.metric(), + )]); + let (ready_tx, ready_rx) = oneshot::channel(); + let (done_tx, done_rx) = oneshot::channel(); + + let failover = async { + ready_rx.await?; + s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; + wait_for_imports( + "t1", + t1, + &expected_transit, + MULTICAST_CONVERGENCE_TIMEOUT, + ) + .await?; + tokio::time::sleep(FAILOVER_SETTLE).await; + done_tx + .send(()) + .map_err(|_| anyhow!("s3 observation stopped before completion")) + }; + tokio::try_join!( + observe_imports_unchanged("s3", s3, &expected, ready_tx, done_rx), + failover, + )?; - wait_for_eq!(multicast_group_count(s1).await?, 1); - wait_for_eq!(multicast_group_count(s2).await?, 0); + assert_multicast_imports!(s1, [("sl0", origin)]); + assert_multicast_imports!(s2, []); println_nopipe!("multicast failover passed"); s2.withdraw_multicast_groups(&vec![origin]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 0); - wait_for_eq!(multicast_group_count(s1).await?, 0); - wait_for_eq!(multicast_group_count(s3).await?, 0); + assert_multicast_imports!(t1, []); + assert_multicast_imports!(s1, []); + assert_multicast_imports!(s3, []); println_nopipe!("multicast final withdrawal passed"); Ok(()) @@ -1115,14 +1600,14 @@ async fn run_quartet_mcast_resync( let origin = MulticastOrigin::new( "233.252.0.11".parse().unwrap(), "ff04::111".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); s1.advertise_multicast_groups(&vec![origin.clone()]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 1); - wait_for_eq!(multicast_group_count(s3).await?, 1); + assert_multicast_imports!(t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(s3, [("sl2", origin)]); let peers = t1.get_peers().await?; let peer = peers @@ -1137,6 +1622,7 @@ async fn run_quartet_mcast_resync( let groups = s3.get_multicast_groups().await?; let route = groups .iter() + .map(|imported| &imported.route) .find(|route| route.origin == origin) .ok_or_else(|| anyhow!("s3 has not imported the test origin"))?; let vector = serde_json::json!({ @@ -1174,27 +1660,33 @@ async fn run_quartet_mcast_resync( }; let started = Instant::now(); - wait_for_eq!( - { + assert_client_state!( + s3, + async { push(false)?; - multicast_group_count(s3).await? + multicast_imports(s3).await }, - 0 + MulticastImports::new() ); - wait_for_eq!(multicast_group_count(s3).await?, 1, 1, 30); + assert_multicast_imports!(s3, [("sl2", origin)], 1, 30); println_nopipe!("multicast resync import passed"); - s1.withdraw_multicast_groups(&vec![origin]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 0); - wait_for_eq!(multicast_group_count(s3).await?, 0); - wait_for_eq!( - { + s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(t1, []); + assert_multicast_imports!(s3, []); + assert_client_state!( + s3, + async { push(true)?; - multicast_group_count(s3).await? + multicast_imports(s3).await }, - 1 + MulticastImports::from([( + "sl2".to_string(), + origin.clone(), + origin.metric(), + )]) ); - wait_for_eq!(multicast_group_count(s3).await?, 0, 1, 30); + assert_multicast_imports!(s3, [], 1, 30); println_nopipe!("multicast resync withdrawal passed"); let min_uptime = *uptime + started.elapsed(); @@ -1402,8 +1894,8 @@ async fn run_sextet_tests( _zs2: &RouterZone<'_>, _zs3: &RouterZone<'_>, _zs4: &RouterZone<'_>, - _zt1: &RouterZone<'_>, - _zt2: &RouterZone<'_>, + zt1: &RouterZone<'_>, + zt2: &RouterZone<'_>, ) -> Result<()> { let log = init_logger(); @@ -1434,13 +1926,7 @@ async fn run_sextet_tests( let name = self.name.clone(); tokio::task::block_in_place(|| { rt.block_on(async move { - println_nopipe!("{name}:"); - if let Ok(peers) = c.get_peers().await { - println_nopipe!("peers: {peers:#?}"); - } - if let Ok(prefixes) = c.get_prefixes().await { - println_nopipe!("prefixes: {prefixes:#?}"); - } + dump_client_state(&name, &c).await; }); }); } @@ -1506,10 +1992,7 @@ async fn run_sextet_tests( stringify!($client), $count ); - wait_for_eq!( - $client.get_peers().await.map(|x| x.len()).ok(), - Some($count) - ); + assert_client_state!(&$client, peer_count(&$client), $count); }}; } @@ -1531,21 +2014,17 @@ async fn run_sextet_tests( stringify!($client), $reach ); - wait_for_eq!( - $client - .get_prefixes() - .await - .map(|x| x - .values() - .cloned() - .into_iter() - .flat_map(|x| x - .clone() - .into_iter() - .map(|y| y.destination)) - .collect::>()) - .ok(), - Some($reach) + assert_client_state!( + &$client, + async { + $client.get_prefixes().await.map(|x| { + x.values() + .flatten() + .map(|route| route.destination) + .collect::>() + }) + }, + $reach ); }}; } @@ -1637,7 +2116,7 @@ async fn run_sextet_tests( assert_peer_counts!(PeerCounts::default().server(1).transit(4)); // - // Going down to the first peer should result in three peering sessions + // Going down to the 1st peer should result in three peering sessions // per transit router. Note in the model above that qsfp0/qsfp1 are cross // connected between the two transit routers. Here we are connecting // swA/qsfp0 <-> swB/qsfp1 @@ -1791,6 +2270,219 @@ async fn run_sextet_tests( assert_all_nexthops_are_peers!(); } + run_sextet_mcast(zt1, zt2, &s1, &s3, &t1, &t2).await +} + +fn underlay_member(port: &str) -> MulticastGroupMember { + MulticastGroupMember { + port_id: port.parse().unwrap(), + link_id: LinkId(0), + direction: Direction::Underlay, + } +} + +fn external_member(port: &str) -> MulticastGroupMember { + MulticastGroupMember { + port_id: port.parse().unwrap(), + link_id: LinkId(0), + direction: Direction::External, + } +} + +/// Multicast replication members across the two racks of the sextet. +/// +/// Each transit router's `ddmd` programs the underlay members of its own +/// switch from the subscriptions it imports. Subscriptions still cross racks +/// over the qsfp external peers, but only rear ports become members, since +/// members on both parallel qsfp links would circulate replicas between the +/// racks. The two qsfp links are cross-connected, akin to the diagram on +/// `test_external_peer_sextet`. +/// +/// ```text +/// rack A rack B +/// s1 ---- rear0 [t1] qsfp0 ------------------ qsfp1 [t2] rear0 ---- s3 +/// qsfp1 ------------------ qsfp0 +/// ``` +/// +/// This test checks cross-rack subscriptions and local replication membership. +/// Cross-rack underlay replication remains unsupported until inter-rack +/// replication prevents cycles. +/// +/// 1. Peer t1/qsfp0 <-> t2/qsfp1 only, and s1 subscribes. t1 programs rear0; +/// t2 imports the subscription over qsfp1 but doesn't program anything. +/// 2. s3 subscribes. Each switch imports the other rack's subscription over +/// qsfp and programs only its own rear0. +/// 3. Move the peering to t1/qsfp1 <-> t2/qsfp0. The imports follow the link; +/// both switches stay at rear0. +/// 4. Peer over both links. Each switch imports the other rack over two next +/// hops, a case that would cause looping with qsfp members, and stays at +/// rear0. +/// 5. Drop back to t1/qsfp0 <-> t2/qsfp1. Each switch loses only the dropped +/// link's import, remaining at rear0. +/// 6. s3 withdraws. t1 keeps rear0 and t2 drains to empty membership. +/// 7. s1 withdraws. t2 drops its import, and t1 keeps only its `External` +/// member. +async fn run_sextet_mcast( + zt1: &RouterZone<'_>, + zt2: &RouterZone<'_>, + s1: &Client, + s3: &Client, + t1: &Client, + t2: &Client, +) -> Result<()> { + const QSFP0: &str = "tfportqsfp0_0/v6"; + const QSFP1: &str = "tfportqsfp1_0/v6"; + let external_peers = |aobjs: &[&str]| ExternalPeers { + address_objects: aobjs.iter().map(|x| x.to_string()).collect(), + }; + + // + // Nexus creates each underlay group before subscriptions arrive. + // Each switch gets its own copy of the group. + // + + let origin = MulticastOrigin::new( + "233.252.0.2".parse().unwrap(), + "ff04::200".parse().unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), + None, + ) + .unwrap(); + let rear0 = [underlay_member("rear0")]; + let t1_external = [external_member("qsfp0")]; + let t1_members = [rear0[0].clone(), t1_external[0].clone()]; + + // Nexus-owned `External` member on t1, which ddmd must never remove. + let group_t1 = create_multicast_underlay_group_with( + zt1, + origin.underlay_group().into(), + &t1_external, + )?; + let group_t2 = + create_multicast_underlay_group(zt2, origin.underlay_group().into())?; + + // + // Peer swA/qsfp0 <-> swB/qsfp1 only, then subscribe s1. t1 learns the + // subscription from s1 on rear0 and redistributes it to t2, which imports + // it over qsfp1. t2 has no local subscriber and leaves its group empty. + // + + t1.set_external_peers(&external_peers(&[QSFP0])).await?; + t2.set_external_peers(&external_peers(&[QSFP1])).await?; + + s1.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(t2, [("tfportqsfp1_0", origin)]); + wait_for_multicast_members(zt1, &group_t1, &t1_members)?; + wait_for_multicast_members(zt2, &group_t2, &[])?; + + println_nopipe!("sextet multicast remote import passed"); + + // + // A 2nd subscriber in rack B provides each switch with a local rear member. + // Each switch also imports the other rack's subscription over qsfp, and the + // exact wait on t2 fails if that import resolves to a qsfp member. + // + + s3.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportqsfp0_0", origin)] + ); + assert_multicast_imports!( + t2, + [("tfportrear0_0", origin), ("tfportqsfp1_0", origin)] + ); + wait_for_multicast_members(zt2, &group_t2, &rear0)?; + + println_nopipe!("sextet multicast subscribers in both racks passed"); + + // + // Switch to swA/qsfp1 <-> swB/qsfp0. Shutting down the old external peer + // expires its routes, and the imports move to the new link while ddmd + // programs only rear0 on both switches. + // + + t1.set_external_peers(&external_peers(&[QSFP1])).await?; + t2.set_external_peers(&external_peers(&[QSFP0])).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportqsfp1_0", origin)] + ); + assert_multicast_imports!( + t2, + [("tfportrear0_0", origin), ("tfportqsfp0_0", origin)] + ); + + println_nopipe!("sextet multicast external link move passed"); + + // + // Peer over both links. Each switch now learns the other rack's + // subscriber over two nexthops. With qsfp members, replicas would cycle + // between the racks here, so ddmd still programs only rear0. + // + + let both = external_peers(&[QSFP0, QSFP1]); + t1.set_external_peers(&both).await?; + t2.set_external_peers(&both).await?; + for client in [t1, t2] { + assert_multicast_imports!( + client, + [ + ("tfportrear0_0", origin), + ("tfportqsfp0_0", origin), + ("tfportqsfp1_0", origin), + ] + ); + } + wait_for_multicast_members(zt1, &group_t1, &t1_members)?; + wait_for_multicast_members(zt2, &group_t2, &rear0)?; + + println_nopipe!("sextet multicast both external links passed"); + + t1.set_external_peers(&external_peers(&[QSFP0])).await?; + t2.set_external_peers(&external_peers(&[QSFP1])).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportqsfp0_0", origin)] + ); + assert_multicast_imports!( + t2, + [("tfportrear0_0", origin), ("tfportqsfp1_0", origin)] + ); + wait_for_multicast_members(zt1, &group_t1, &t1_members)?; + wait_for_multicast_members(zt2, &group_t2, &rear0)?; + + println_nopipe!("sextet multicast partial link loss passed"); + + // + // With only s1 subscribed, t1 holds rear0 and t2 drains its rear0. + // + + s3.withdraw_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(t2, [("tfportqsfp1_0", origin)]); + wait_for_multicast_members(zt2, &group_t2, &[])?; + + println_nopipe!("sextet multicast remote withdraw passed"); + + // + // Withdrawing the last subscriber drains both groups. The groups remain + // in DPD for Nexus to delete, and the external peers are cleared for any + // test that runs after this one. + // + + s1.withdraw_multicast_groups(&vec![origin]).await?; + assert_multicast_imports!(t1, []); + assert_multicast_imports!(t2, []); + wait_for_multicast_members(zt1, &group_t1, &t1_external)?; + wait_for_multicast_members(zt2, &group_t2, &[])?; + + t1.set_external_peers(&external_peers(&[])).await?; + t2.set_external_peers(&external_peers(&[])).await?; + + println_nopipe!("sextet multicast withdraw passed"); + Ok(()) } @@ -1802,6 +2494,18 @@ async fn prefix_count(c: &Client) -> Result { .sum::()) } +async fn peer_count(c: &Client) -> Result { + Ok(c.get_peers().await?.len()) +} + +async fn exchange_peer_count(c: &Client) -> Result { + Ok(c.get_peers() + .await? + .values() + .filter(|peer| matches!(peer.status, PeerStatus::Exchange(_))) + .count()) +} + async fn tunnel_endpoint_count(c: &Client) -> Result { Ok(c.get_tunnel_endpoints().await?.len()) } @@ -1810,12 +2514,104 @@ async fn tunnel_originated_endpoint_count(c: &Client) -> Result { Ok(c.get_originated_tunnel_endpoints().await?.len()) } -async fn multicast_group_count(c: &Client) -> Result { - Ok(c.get_multicast_groups().await?.len()) +async fn multicast_imports(c: &Client) -> Result { + Ok(c.get_multicast_groups() + .await? + .iter() + .map(|imported| { + ( + imported.if_name.clone(), + imported.route.origin.clone(), + imported.route.origin.metric(), + ) + }) + .collect()) } -async fn multicast_originated_count(c: &Client) -> Result { - Ok(c.get_originated_multicast_groups().await?.len()) +async fn multicast_origins( + c: &Client, +) -> Result> { + Ok(c.get_originated_multicast_groups() + .await? + .iter() + .map(|origin| (origin.clone(), origin.metric())) + .collect()) +} + +/// Wait for `client`'s multicast imports to equal what's `expected`. +/// +/// Unlike [`assert_multicast_imports!`], this yields between reads, which means +/// it can run alongside another future in the same task. +async fn wait_for_imports( + name: &str, + client: &Client, + expected: &MulticastImports, + timeout: Duration, +) -> Result<()> { + tokio::time::timeout(timeout, async { + while multicast_imports(client).await? != *expected { + tokio::time::sleep(FAILOVER_POLL_INTERVAL).await; + } + Ok(()) + }) + .await + .map_err(|_| anyhow!("{name} imports did not converge to {expected:#?}"))? +} + +/// Fail if any read of `client`'s multicast imports differs from `expected`. +/// +/// This signals `ready` after the 1st matching read, and then keeps sampling +/// until `done` arrives. +async fn observe_imports_unchanged( + name: &str, + client: &Client, + expected: &MulticastImports, + ready: oneshot::Sender<()>, + mut done: oneshot::Receiver<()>, +) -> Result<()> { + let mut ready = Some(ready); + loop { + let imports = multicast_imports(client).await?; + if imports != *expected { + dump_client_state(name, client).await; + return Err(anyhow!("{name} imports changed: {imports:#?}")); + } + if let Some(ready) = ready.take() { + ready + .send(()) + .map_err(|_| anyhow!("stopped before {name} was observed"))?; + } + match done.try_recv() { + Ok(()) => return Ok(()), + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Closed) => { + return Err(anyhow!("stopped before {name} observation ended")); + } + } + tokio::time::sleep(FAILOVER_POLL_INTERVAL).await; + } +} + +async fn dump_client_state(name: &str, c: &Client) { + let dump = async { + println_nopipe!("{name}:"); + println_nopipe!("peers: {:#?}", c.get_peers().await); + println_nopipe!("prefixes: {:#?}", c.get_prefixes().await); + println_nopipe!( + "originated multicast: {:#?}", + c.get_originated_multicast_groups().await + ); + println_nopipe!( + "imported multicast: {:#?}", + c.get_multicast_groups().await + ); + }; + if tokio::time::timeout(Duration::from_secs(10), dump) + .await + .is_err() + { + eprintln_nopipe!("{name}: diagnostic dump timed out"); + } } fn init_logger() -> Logger { From 7d6d1bd3708e4e6494d1d8a7cc6a6bf0b5f28fd2 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Mon, 9 Nov 2026 05:04:36 +0000 Subject: [PATCH 2/2] [deps] deps -> dednrite, sidecar-lite, softnpu --- .github/buildomat/test-ddm-common.sh | 4 ++-- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 4 ++-- falcon-lab/src/main.rs | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/buildomat/test-ddm-common.sh b/.github/buildomat/test-ddm-common.sh index a630893ee..6f264cb86 100755 --- a/.github/buildomat/test-ddm-common.sh +++ b/.github/buildomat/test-ddm-common.sh @@ -3,8 +3,8 @@ source .github/buildomat/common.sh export MAGHEMITE_VERSION=`git rev-parse HEAD` -export SOFTNPU_VERSION=a6a878533f9eae542552ef66223a37443f931149 -export SIDECAR_LITE_VERSION=be3c09fe9a7e9c6bc5f5c6d184ad203b62b1eb32 +export SOFTNPU_VERSION=63687602f0d26add71d4dae4ac62d46563ba23f5 +export SIDECAR_LITE_VERSION=5f1d8eb2c7b3cd550c37e6a2118b91c95b49b6de export DENDRITE_VERSION=`grep -A3 'dpd-client' Cargo.toml | sed -n 's/^rev = "\(.*\)"/\1/p'` function cleanup { diff --git a/Cargo.lock b/Cargo.lock index 009d2e60b..d8a37dd47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -947,7 +947,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -963,7 +963,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "anyhow", "chrono", @@ -1732,7 +1732,7 @@ dependencies = [ [[package]] name = "dpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "async-trait", "chrono", @@ -1757,7 +1757,7 @@ dependencies = [ [[package]] name = "dpd-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "dpd-types-versions", ] @@ -1765,7 +1765,7 @@ dependencies = [ [[package]] name = "dpd-types-versions" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=1d5c2961981d149d85ea08c9ea06f7510a72adbf#1d5c2961981d149d85ea08c9ea06f7510a72adbf" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "chrono", "oxnet", @@ -2038,7 +2038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3408,7 +3408,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6016,7 +6016,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6479,7 +6479,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6546,7 +6546,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7768,7 +7768,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7777,7 +7777,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7787,7 +7787,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9156,7 +9156,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 251edf4b8..d7395bf96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,8 +158,8 @@ rev = "44052786cb63ad9ff32ec47bd8d65edb6a184e21" [workspace.dependencies.dpd-client] git = "https://github.com/oxidecomputer/dendrite" -rev = "1d5c2961981d149d85ea08c9ea06f7510a72adbf" +rev = "a1b7c0652650a86a05d9c0f23d5773c79aab2613" [workspace.dependencies.dpd-types-versions] git = "https://github.com/oxidecomputer/dendrite" -rev = "1d5c2961981d149d85ea08c9ea06f7510a72adbf" +rev = "a1b7c0652650a86a05d9c0f23d5773c79aab2613" diff --git a/falcon-lab/src/main.rs b/falcon-lab/src/main.rs index a95a5b5fe..7c54b1bfb 100644 --- a/falcon-lab/src/main.rs +++ b/falcon-lab/src/main.rs @@ -22,7 +22,7 @@ mod scenario; mod topo; mod util; -const DEFAULT_NPUVM_COMMIT: &str = "a6a878533f9eae542552ef66223a37443f931149"; +const DEFAULT_NPUVM_COMMIT: &str = "63687602f0d26add71d4dae4ac62d46563ba23f5"; #[derive(Debug, Parser)] struct Cli {