diff --git a/.github/buildomat/jobs/test-ddm-dual.sh b/.github/buildomat/jobs/test-ddm-dual.sh new file mode 100755 index 000000000..023ed9e97 --- /dev/null +++ b/.github/buildomat/jobs/test-ddm-dual.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#: +#: name = "test-ddm-dual" +#: variety = "basic" +#: target = "helios-3.0" +#: rust_toolchain = "stable" +#: output_rules = [ +#: "/work/*.log", +#: ] + +source .github/buildomat/test-ddm-common.sh + +# +# dual sidecar tests +# + +banner "dual" +pfexec cargo test --release -p mg-tests test_dual_sidecar -- --nocapture diff --git a/.github/buildomat/jobs/test-ddm-trio.sh b/.github/buildomat/jobs/test-ddm-trio.sh index ab668eca2..49d40e4bc 100755 --- a/.github/buildomat/jobs/test-ddm-trio.sh +++ b/.github/buildomat/jobs/test-ddm-trio.sh @@ -15,4 +15,4 @@ source .github/buildomat/test-ddm-common.sh # banner "trio" -pfexec cargo test --release -p mg-tests test_trio_v3 -- --nocapture +pfexec cargo test --release -p mg-tests test_trio -- --nocapture diff --git a/.github/buildomat/jobs/test-ddm.sh b/.github/buildomat/jobs/test-ddm.sh index f42ff2dc5..547e49038 100755 --- a/.github/buildomat/jobs/test-ddm.sh +++ b/.github/buildomat/jobs/test-ddm.sh @@ -16,4 +16,10 @@ set -o pipefail source .github/buildomat/test-common.sh banner ddm -cargo nextest run -p ddm -p ddm-api-types -p ddm-api-types-versions --nocapture +cargo nextest run \ + -p ddm \ + -p ddm-api-types \ + -p ddm-api-types-versions \ + -p ddm-protocol \ + -p mg-common \ + --nocapture diff --git a/.github/buildomat/test-ddm-common.sh b/.github/buildomat/test-ddm-common.sh index a630893ee..6f264cb86 100755 --- a/.github/buildomat/test-ddm-common.sh +++ b/.github/buildomat/test-ddm-common.sh @@ -3,8 +3,8 @@ source .github/buildomat/common.sh export MAGHEMITE_VERSION=`git rev-parse HEAD` -export SOFTNPU_VERSION=a6a878533f9eae542552ef66223a37443f931149 -export SIDECAR_LITE_VERSION=be3c09fe9a7e9c6bc5f5c6d184ad203b62b1eb32 +export SOFTNPU_VERSION=63687602f0d26add71d4dae4ac62d46563ba23f5 +export SIDECAR_LITE_VERSION=5f1d8eb2c7b3cd550c37e6a2118b91c95b49b6de export DENDRITE_VERSION=`grep -A3 'dpd-client' Cargo.toml | sed -n 's/^rev = "\(.*\)"/\1/p'` function cleanup { diff --git a/Cargo.lock b/Cargo.lock index 432359360..d8a37dd47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -963,7 +963,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "anyhow", "chrono", @@ -1431,6 +1431,7 @@ dependencies = [ "ddm-api-types", "ddm-protocol", "dpd-client", + "dpd-types-versions", "dropshot", "hostname 0.4.2", "http-body-util", @@ -1731,7 +1732,7 @@ dependencies = [ [[package]] name = "dpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "async-trait", "chrono", @@ -1756,7 +1757,7 @@ dependencies = [ [[package]] name = "dpd-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "dpd-types-versions", ] @@ -1764,7 +1765,7 @@ dependencies = [ [[package]] name = "dpd-types-versions" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=6a5425b2e6ae84d6c4d18bd3a984a16731bf4734#6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +source = "git+https://github.com/oxidecomputer/dendrite?rev=a1b7c0652650a86a05d9c0f23d5773c79aab2613#a1b7c0652650a86a05d9c0f23d5773c79aab2613" dependencies = [ "chrono", "oxnet", @@ -3998,11 +3999,13 @@ dependencies = [ "anyhow", "backoff", "clap", + "dpd-client", "libc", "libnet", "oximeter", "oximeter-producer", "oxnet", + "proptest", "slog", "slog-async", "slog-bunyan", @@ -4061,6 +4064,8 @@ dependencies = [ "client-common", "ddm-admin-client", "ddm-api-types-versions", + "dpd-client", + "dpd-types-versions", "oxnet", "rand 0.10.3", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index a75c9f589..d7395bf96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,8 +158,8 @@ rev = "44052786cb63ad9ff32ec47bd8d65edb6a184e21" [workspace.dependencies.dpd-client] git = "https://github.com/oxidecomputer/dendrite" -rev = "6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +rev = "a1b7c0652650a86a05d9c0f23d5773c79aab2613" [workspace.dependencies.dpd-types-versions] git = "https://github.com/oxidecomputer/dendrite" -rev = "6a5425b2e6ae84d6c4d18bd3a984a16731bf4734" +rev = "a1b7c0652650a86a05d9c0f23d5773c79aab2613" diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index fc60883db..658340769 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -15,6 +15,7 @@ progenitor::generate_api!( ExternalMulticastIp = ddm_api_types_versions::latest::net::OverlayMulticast, UnderlayMulticastIpv6 = ddm_api_types_versions::latest::net::UnderlayMulticastIpv6, MulticastRoute = ddm_api_types_versions::latest::db::MulticastRoute, + ImportedMulticastRoute = ddm_api_types_versions::latest::db::ImportedMulticastRoute, MulticastPathHop = ddm_api_types_versions::latest::exchange::MulticastPathHop, PeerInfo = ddm_api_types_versions::latest::db::PeerInfo, PeerStatus = ddm_api_types_versions::latest::db::PeerStatus, diff --git a/ddm-api-types/versions/src/impls/mcast.rs b/ddm-api-types/versions/src/impls/mcast.rs index ebcc36cbc..bc89a5b5d 100644 --- a/ddm-api-types/versions/src/impls/mcast.rs +++ b/ddm-api-types/versions/src/impls/mcast.rs @@ -36,14 +36,16 @@ impl From for MulticastOrigin { #[cfg(test)] mod tests { + use std::collections::HashSet; use std::hash::{DefaultHasher, Hash, Hasher}; use std::net::Ipv6Addr; - use crate::latest::db::MulticastRoute; - use crate::latest::exchange::MulticastPathHop; - use crate::latest::net::{ + use crate::v4::db::MulticastRoute; + use crate::v4::exchange::MulticastPathHop; + use crate::v4::net::{ MulticastOrigin, OverlayMulticast, UnderlayMulticastIpv6, Vni, }; + use crate::v5::db::ImportedMulticastRoute; fn origin(overlay: &str) -> MulticastOrigin { MulticastOrigin::new( @@ -55,6 +57,33 @@ mod tests { .unwrap() } + #[test] + fn v4_conversion_drops_interface_ident() { + let route = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: "fe80::1".parse().unwrap(), + path: vec![MulticastPathHop::new( + "peer".into(), + "fe80::1".parse().unwrap(), + )], + }; + let imported = HashSet::from([ + ImportedMulticastRoute { + route: route.clone(), + if_name: "tfportrear0_0".into(), + }, + ImportedMulticastRoute { + route: route.clone(), + if_name: "tfportrear1_0".into(), + }, + ]); + assert_eq!(imported.len(), 2); + + let prev: HashSet = + imported.into_iter().map(Into::into).collect(); + assert_eq!(prev, HashSet::from([route])); + } + /// Use the same initial hasher state for each route comparison. fn hash_of(route: &MulticastRoute) -> u64 { let mut hasher = DefaultHasher::new(); diff --git a/ddm-api-types/versions/src/latest.rs b/ddm-api-types/versions/src/latest.rs index a187e983b..798da112b 100644 --- a/ddm-api-types/versions/src/latest.rs +++ b/ddm-api-types/versions/src/latest.rs @@ -18,6 +18,8 @@ pub mod db { pub use crate::v4::db::MulticastRoute; pub use crate::v4::db::PeerInfo; + + pub use crate::v5::db::ImportedMulticastRoute; } pub mod exchange { diff --git a/ddm-api-types/versions/src/lib.rs b/ddm-api-types/versions/src/lib.rs index 4af2b9be6..c6eb560df 100644 --- a/ddm-api-types/versions/src/lib.rs +++ b/ddm-api-types/versions/src/lib.rs @@ -39,3 +39,5 @@ pub mod v2; pub mod v3; #[path = "multicast_support/mod.rs"] pub mod v4; +#[path = "multicast_interfaces/mod.rs"] +pub mod v5; diff --git a/ddm-api-types/versions/src/multicast_interfaces/db.rs b/ddm-api-types/versions/src/multicast_interfaces/db.rs new file mode 100644 index 000000000..87d019523 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_interfaces/db.rs @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v4; + +/// A multicast route with the interface on which its peer was discovered. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, +)] +pub struct ImportedMulticastRoute { + /// The imported multicast route. + pub route: v4::db::MulticastRoute, + /// Local interface on which the advertising peer was discovered. + pub if_name: String, +} + +impl From for v4::db::MulticastRoute { + fn from(imported: ImportedMulticastRoute) -> Self { + imported.route + } +} diff --git a/ddm-api-types/versions/src/multicast_interfaces/mod.rs b/ddm-api-types/versions/src/multicast_interfaces/mod.rs new file mode 100644 index 000000000..8e8e7e533 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_interfaces/mod.rs @@ -0,0 +1,10 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `MULTICAST_INTERFACES` of the DDM Admin API. +//! +//! Imported multicast routes convey the interface that their peer was +//! discovered on. + +pub mod db; diff --git a/ddm-api/src/lib.rs b/ddm-api/src/lib.rs index 0ae6cf6d7..e0730016f 100644 --- a/ddm-api/src/lib.rs +++ b/ddm-api/src/lib.rs @@ -2,9 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use ddm_api_types_versions::latest; -use ddm_api_types_versions::v1; -use ddm_api_types_versions::v2; +use ddm_api_types_versions::{latest, v1, v2, v4}; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -27,6 +25,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (5, MULTICAST_INTERFACES), (4, MULTICAST_SUPPORT), (3, EXTERNAL_PEERS), (2, PEER_DURATIONS), @@ -177,11 +176,33 @@ pub trait DdmAdminApi { #[endpoint { method = GET, path = "/multicast_groups", - versions = VERSION_MULTICAST_SUPPORT.. + versions = VERSION_MULTICAST_INTERFACES.. }] async fn get_multicast_groups( ctx: RequestContext, - ) -> Result>, HttpError>; + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + #[endpoint { + operation_id = "get_multicast_groups", + method = GET, + path = "/multicast_groups", + versions = VERSION_MULTICAST_SUPPORT..VERSION_MULTICAST_INTERFACES + }] + async fn get_multicast_groups_v4( + ctx: RequestContext, + ) -> Result>, HttpError> + { + let resp = Self::get_multicast_groups(ctx).await?; + Ok(HttpResponseOk( + resp.0 + .into_iter() + .map(v4::db::MulticastRoute::from) + .collect(), + )) + } #[endpoint { method = PUT, diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index 0acd6d17b..d2aa6bec9 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" pretty_assertions.workspace = true camino-tempfile.workspace = true serde_urlencoded.workspace = true +tokio = { workspace = true, features = ["test-util"] } [dependencies] slog.workspace = true @@ -38,6 +39,7 @@ oxnet.workspace = true uuid.workspace = true ddm-api.workspace = true dpd-client.workspace = true +dpd-types-versions.workspace = true # illumos-only deps used by the routing state machine and platform sys layer. # Gated by the `backend` feature so stub builds (e.g. Linux test fixtures diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 5dbf60c3f..22b0494f0 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -15,7 +15,9 @@ use camino::Utf8PathBuf; use ddm_api::DdmAdminApi; use ddm_api::ddm_admin_api_mod; use ddm_api_types::admin::{EnableStatsRequest, ExpirePathParams, PrefixMap}; -use ddm_api_types::db::{MulticastRoute, PeerInfo, RouterKind, TunnelRoute}; +use ddm_api_types::db::{ + ImportedMulticastRoute, PeerInfo, RouterKind, TunnelRoute, +}; use ddm_api_types::exchange::{PathVector, RouterId}; use ddm_api_types::external_peers::ExternalPeers; use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; @@ -436,9 +438,15 @@ impl DdmAdminApi for DdmAdminApiImpl { async fn get_multicast_groups( ctx: RequestContext, - ) -> Result>, HttpError> { + ) -> Result>, HttpError> + { let ctx = lock!(ctx.context()); - let imported = ctx.db.imported_mcast(); + let imported = ctx + .db + .imported_mcast_links() + .into_iter() + .map(|(route, if_name)| ImportedMulticastRoute { route, if_name }) + .collect(); Ok(HttpResponseOk(imported)) } diff --git a/ddm/src/db/mcast.rs b/ddm/src/db/mcast.rs index da954e79a..91196dda8 100644 --- a/ddm/src/db/mcast.rs +++ b/ddm/src/db/mcast.rs @@ -16,12 +16,29 @@ use slog::error; use super::{Db, Error, RemovedNexthopRoutes}; +mod imported; + +pub(in crate::db) use imported::ImportedMcast; +use imported::Inserted; + +/// The effect of [`Db::splice_imported_mcast`] on the imported set. +pub struct SplicedMulticast { + /// Imports that added an entry or changed an entry's path or metric. + pub imported: HashSet, + /// Routes that were removed. + pub removed: HashSet, +} + /// Persistent tree for locally originated multicast groups. const MCAST_ORIGINATE: &str = "mcast_originate"; impl Db { - pub fn imported_mcast(&self) -> HashSet { - lock!(self.data).imported_mcast.routes().cloned().collect() + pub fn imported_mcast_links(&self) -> Vec<(MulticastRoute, String)> { + lock!(self.data) + .imported_mcast + .links() + .map(|(route, ifname)| (route.clone(), ifname.to_string())) + .collect() } pub fn shortest_path_mcast( @@ -35,34 +52,44 @@ impl Db { .collect() } - /// Sequence cutoff for routes imported before a pull occurs. + /// Pull sequence number used to identify earlier imports. pub fn next_mcast_seq(&self) -> u64 { lock!(self.data).mcast_seq + 1 } /// Apply multicast imports and removals under a held data lock, returning - /// the routes that were actually removed. - pub fn update_imported_mcast( + /// the imports that affected the RIB and the routes that were actually + /// removed. + pub fn splice_imported_mcast( &self, update: MulticastRibChange<'_>, - ) -> HashSet { + ) -> SplicedMulticast { let mut guard = lock!(self.data); let data = &mut *guard; + let mut edit = data.imported_mcast.edit(&self.mcast_wake); // Path and metric live outside of route identity and may change on // re-import. - for x in update.import { - data.mcast_seq += 1; - data.imported_mcast.insert(x, data.mcast_seq); - } + let imported = update + .import + .iter() + .filter(|x| { + data.mcast_seq += 1; + edit.insert(x, update.ifname, data.mcast_seq) + != Inserted::Unchanged + }) + .cloned() + .collect(); - update + let removed = update .remove .iter() .filter_map(|x| { - data.imported_mcast.remove(x, update.imported_before) + edit.remove(x, update.ifname, update.imported_before) }) - .collect() + .collect(); + + SplicedMulticast { imported, removed } } /// Persist multicast origins to advertise to peers. @@ -144,14 +171,19 @@ impl Db { self.path_snapshot(&origins, HashMap::new()) } - /// Remove multicast routes via `nexthop` on a downgrade below V4. + /// Remove multicast routes via `nexthop` on `ifname` on a downgrade below + /// V4. /// /// Underlay and tunnel routes remain intact. pub fn remove_nexthop_mcast_routes( &self, nexthop: Ipv6Addr, + ifname: &str, ) -> RemovedNexthopRoutes { - let removed = lock!(self.data).imported_mcast.extract_nexthop(nexthop); + let removed = lock!(self.data) + .imported_mcast + .edit(&self.mcast_wake) + .extract_link(nexthop, ifname); RemovedNexthopRoutes { underlay: HashSet::new(), @@ -160,17 +192,18 @@ impl Db { } } - /// Origins imported via `nexthop` before `seq` that are not in + /// Origins imported via `nexthop` on `ifname` before `seq` that are not in /// `announced`. pub fn stale_mcast_by_nexthop( &self, nexthop: Ipv6Addr, + ifname: &str, seq: u64, announced: &HashSet, ) -> Vec { lock!(self.data) .imported_mcast - .stale(nexthop, seq, announced) + .stale(nexthop, ifname, seq, announced) } /// Read local origins to complete the snapshot of `imported`. @@ -213,148 +246,6 @@ impl Db { } } -#[derive(Debug, Clone)] -struct ImportedEntry { - route: MulticastRoute, - seq: u64, -} - -/// A type wrapper over the set of imported multicast routes, keyed by origin -/// and then by nexthop. -/// -/// A path snapshot for an update's origins is a lookup per origin. Each -/// origin's routes are sorted by nexthop, with the lowest-nexthop route -/// coming first. Per-peer work (e.g., stale-route scans, expiry) visits each -/// origin once. -// Note: We use a sorted `Vec` here over a `BTreeMap` because an origin rarely -// has more than a few nexthops. -#[derive(Debug, Default, Clone)] -pub struct ImportedMcast(HashMap>); - -fn find_nexthop( - nexthops: &[ImportedEntry], - nexthop: Ipv6Addr, -) -> Result { - nexthops.binary_search_by_key(&nexthop, |entry| entry.route.nexthop) -} - -impl ImportedMcast { - fn routes(&self) -> impl Iterator { - self.0.values().flatten().map(|entry| &entry.route) - } - - fn shortest_path_routes<'a>( - &'a self, - peer_router_id: &'a RouterId, - ) -> impl Iterator { - self.0.keys().filter_map(move |origin| { - self.shortest_path(origin, peer_router_id) - }) - } - - fn insert(&mut self, route: &MulticastRoute, seq: u64) { - let nexthops = self.0.entry(route.origin.clone()).or_default(); - match find_nexthop(nexthops, route.nexthop) { - Ok(idx) => { - let entry = &mut nexthops[idx]; - if entry.route.path != route.path - || entry.route.origin.metric() != route.origin.metric() - { - entry.route = route.clone(); - } - entry.seq = seq; - } - Err(idx) => { - // Grow one slot at a time. - nexthops.reserve_exact(1); - nexthops.insert( - idx, - ImportedEntry { - route: route.clone(), - seq, - }, - ); - } - } - } - - fn remove( - &mut self, - route: &MulticastRoute, - imported_before: Option, - ) -> Option { - let nexthops = self.0.get_mut(&route.origin)?; - let idx = find_nexthop(nexthops, route.nexthop).ok()?; - if imported_before.is_some_and(|seq| nexthops[idx].seq >= seq) { - return None; - } - let entry = nexthops.remove(idx); - if nexthops.is_empty() { - self.0.remove(&route.origin); - } - Some(entry.route) - } - - pub(super) fn extract_nexthop( - &mut self, - nexthop: Ipv6Addr, - ) -> HashSet { - let mut extracted = HashSet::new(); - self.0.retain(|_, nexthops| { - if let Ok(idx) = find_nexthop(nexthops, nexthop) { - extracted.insert(nexthops.remove(idx).route); - } - !nexthops.is_empty() - }); - extracted - } - - /// Pick the route with the shortest path, breaking ties by the lowest - /// origin metric (first) and then by the lowest nexthop address (e.g., - /// `fe80::1` < `fe80::2`). - /// - /// Routes whose path contains `peer_router_id` (the peer an update gets - /// sent to) are omitted. That peer would discard such a path as a loop - /// and drop its route through us. - fn shortest_path( - &self, - origin: &MulticastOrigin, - peer_router_id: &RouterId, - ) -> Option<&MulticastRoute> { - self.0 - .get(origin)? - .iter() - .map(|entry| &entry.route) - .filter(|route| { - !route - .path - .iter() - .any(|hop| &hop.router_id == peer_router_id) - }) - .min_by_key(|route| { - (route.path.len(), route.origin.metric(), route.nexthop) - }) - } - - fn stale( - &self, - nexthop: Ipv6Addr, - seq: u64, - announced: &HashSet, - ) -> Vec { - self.0 - .values() - .filter_map(|nexthops| { - find_nexthop(nexthops, nexthop).ok().map(|i| &nexthops[i]) - }) - .filter(|entry| { - entry.seq < seq && !announced.contains(&entry.route.origin) - }) - .map(|entry| entry.route.origin.clone()) - .collect() - } -} - /// Persistent origin key, excluding metric as in `PartialEq` and `Hash`. fn mcast_origin_key(origin: &MulticastOrigin) -> Result { Ok(serde_json::to_string(&( @@ -368,20 +259,23 @@ fn mcast_origin_key(origin: &MulticastOrigin) -> Result { /// A batch of multicast imports and removals. #[derive(Debug, Clone, Copy)] pub struct MulticastRibChange<'a> { + pub ifname: &'a str, pub import: &'a HashSet, pub remove: &'a HashSet, /// Remove only routes imported before this sequence number. - /// Pull reconciliation uses this cutoff to preserve concurrent pushes. + /// Pull reconciliation uses this sequence to preserve concurrent pushes. pub imported_before: Option, } impl<'a> MulticastRibChange<'a> { - /// A change without a sequence cutoff, as used for push withdrawals. + /// A change without a pull sequence, as used for push withdrawals. pub fn new( + ifname: &'a str, import: &'a HashSet, remove: &'a HashSet, ) -> Self { Self { + ifname, import, remove, imported_before: None, @@ -422,6 +316,7 @@ pub(crate) mod tests { use ddm_protocol::v4::MulticastPathHop; use pretty_assertions::assert_eq; use slog::Logger; + use tokio::sync::watch; use super::*; @@ -456,7 +351,8 @@ pub(crate) mod tests { } fn import(db: &Db, routes: &HashSet) { - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", routes, &HashSet::new(), )); @@ -488,7 +384,8 @@ pub(crate) mod tests { path: Vec::new(), }; db.originate_mcast(&HashSet::from([good.clone()])).unwrap(); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([route.clone()]), &HashSet::new(), )); @@ -523,7 +420,8 @@ pub(crate) mod tests { db.originate_mcast(&HashSet::from([local_origin.clone()])) .unwrap(); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([route.clone()]), &HashSet::new(), )); @@ -604,7 +502,7 @@ pub(crate) mod tests { db.import_tunnel(&HashSet::from([tunnel])); let removed = - db.remove_nexthop_mcast_routes("fe80::a".parse().unwrap()); + db.remove_nexthop_mcast_routes("fe80::a".parse().unwrap(), "test0"); // A snapshot taken after removal no longer views the removed routes. let paths = db @@ -615,7 +513,7 @@ pub(crate) mod tests { assert_eq!(removed.multicast, HashSet::from([a, a2])); assert!(removed.underlay.is_empty()); assert!(removed.tunnel.is_empty()); - assert_eq!(db.imported_mcast(), HashSet::from([b.clone()])); + assert_eq!(db.imported_mcast_links(), vec![(b, "test0".to_string())]); assert_eq!(db.imported_count(), 1); assert_eq!(db.imported_tunnel(), HashSet::from([tunnel])); } @@ -656,16 +554,11 @@ pub(crate) mod tests { let tunnel_b = tunnel("fe80::b", "fd00:1701::b"); db.import_tunnel(&HashSet::from([tunnel_a, tunnel_b])); - let removed = db.remove_nexthop_routes("fe80::a".parse().unwrap()); + let removed = + db.remove_nexthop_routes("fe80::a".parse().unwrap(), "test0"); assert_eq!(removed.underlay, HashSet::from([underlay_a])); assert_eq!(removed.tunnel, HashSet::from([tunnel_a])); - let paths = db.snapshot_mcast_paths( - [&multicast_a.origin, &multicast_b.origin], - &NO_PEER, - ); - assert!(paths.imported(&multicast_a.origin).is_none()); - assert_eq!(paths.imported(&multicast_b.origin), Some(&multicast_b)); assert_eq!(removed.multicast, HashSet::from([multicast_a])); assert_eq!(db.imported(), HashSet::from([underlay_b])); assert_eq!(db.imported_tunnel(), HashSet::from([tunnel_b])); @@ -692,6 +585,13 @@ pub(crate) mod tests { other_peer, ]), ); + let other_link = + route(mcast_origin("233.252.0.5", "ff04::5"), "fe80::a"); + db.splice_imported_mcast(MulticastRibChange::new( + "test1", + &HashSet::from([other_link.clone()]), + &HashSet::new(), + )); let seq = db.next_mcast_seq(); let late = route(mcast_origin("233.252.0.4", "ff04::4"), "fe80::a"); @@ -699,16 +599,25 @@ pub(crate) mod tests { let stale = db.stale_mcast_by_nexthop( peer, + "test0", seq, - &HashSet::from([announced.origin]), + &HashSet::from([announced.origin.clone()]), ); assert_eq!(stale, vec![unannounced.origin]); + + let stale = db.stale_mcast_by_nexthop( + peer, + "test1", + seq, + &HashSet::from([announced.origin]), + ); + assert_eq!(stale, vec![other_link.origin]); } - /// The cutoff removes an untouched route and preserves imports made - /// after it. + /// Remove an untouched route and preserve imports made at or after the + /// pull sequence. #[test] - fn stale_removal_skips_routes_imported_after_the_cutoff() { + fn stale_removal_skips_routes_imported_at_or_after_the_pull_seq() { let (_dir, db) = test_db(); let prev = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); @@ -716,7 +625,7 @@ pub(crate) mod tests { route(mcast_origin("233.252.0.2", "ff04::2"), "fe80::a"); import(&db, &HashSet::from([prev.clone(), refreshed.clone()])); - let started_at = db.next_mcast_seq(); + let pull_seq = db.next_mcast_seq(); // A concurrent push re-announces `refreshed` and adds a `new` route // while the pull is reading a snapshot that contains neither of these. @@ -736,14 +645,25 @@ pub(crate) mod tests { ..new.clone() }, ]); - let removed = db.update_imported_mcast(MulticastRibChange { - import: &HashSet::new(), - remove: &candidates, - imported_before: Some(started_at), - }); + let removed = db + .splice_imported_mcast(MulticastRibChange { + ifname: "test0", + import: &HashSet::new(), + remove: &candidates, + imported_before: Some(pull_seq), + }) + .removed; assert_eq!(removed, HashSet::from([prev])); - assert_eq!(db.imported_mcast(), HashSet::from([refreshed, new])); + assert_eq!( + db.imported_mcast_links() + .into_iter() + .collect::>(), + HashSet::from([ + (refreshed, "test0".to_string()), + (new, "test0".to_string()), + ]), + ); } #[test] @@ -766,7 +686,8 @@ pub(crate) mod tests { Some((low.nexthop, low.path.clone())) ); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::new(), &HashSet::from([low.clone()]), )); @@ -789,7 +710,7 @@ pub(crate) mod tests { ); import(&db, &HashSet::from([low.clone()])); - let removed = db.remove_nexthop_mcast_routes(low.nexthop); + let removed = db.remove_nexthop_mcast_routes(low.nexthop, "test0"); assert_eq!(removed.multicast.len(), 2); assert_eq!( shortest(&db, &origin), @@ -797,9 +718,9 @@ pub(crate) mod tests { ); assert_eq!(shortest(&db, &other), None); - db.remove_nexthop_mcast_routes(high.nexthop); + db.remove_nexthop_mcast_routes(high.nexthop, "test0"); assert_eq!(shortest(&db, &origin), None); - assert!(db.imported_mcast().is_empty()); + assert!(db.imported_mcast_links().is_empty()); } #[test] @@ -818,9 +739,12 @@ pub(crate) mod tests { let peer_in_path = via("fe80::1", &["peer", "x"]); let long = via("fe80::2", &["a", "b", "c"]); let short = via("fe80::3", &["d", "e"]); - imported.insert(&peer_in_path, 1); - imported.insert(&long, 2); - imported.insert(&short, 3); + let wake = watch::Sender::new(0); + let mut edit = imported.edit(&wake); + edit.insert(&peer_in_path, "test0", 1); + edit.insert(&long, "test0", 2); + edit.insert(&short, "test0", 3); + drop(edit); let picked = imported.shortest_path(&origin, &"peer".into()).unwrap(); assert_eq!(picked.nexthop, short.nexthop); @@ -845,13 +769,16 @@ pub(crate) mod tests { let low_nexthop = via("fe80::1", 20, &["a", "b"]); let low_metric = via("fe80::2", 10, &["c", "d"]); let shorter = via("fe80::3", 30, &["e"]); - imported.insert(&low_nexthop, 1); - imported.insert(&low_metric, 2); + let wake = watch::Sender::new(0); + let mut edit = imported.edit(&wake); + edit.insert(&low_nexthop, "test0", 1); + edit.insert(&low_metric, "test0", 2); + drop(edit); let picked = imported.shortest_path(&origin, &"peer".into()).unwrap(); assert_eq!(picked.nexthop, low_metric.nexthop); - imported.insert(&shorter, 3); + imported.edit(&wake).insert(&shorter, "test0", 3); let picked = imported.shortest_path(&origin, &"peer".into()).unwrap(); assert_eq!(picked.nexthop, shorter.nexthop); } @@ -859,14 +786,234 @@ pub(crate) mod tests { #[test] fn imported_mcast_drops_emptied_origins() { let mut imported = ImportedMcast::default(); + let wake = watch::Sender::new(0); let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::1"); - imported.insert(&a, 1); - assert_eq!(imported.remove(&a, None), Some(a.clone())); - assert!(imported.0.is_empty()); + let mut edit = imported.edit(&wake); + edit.insert(&a, "test0", 1); + assert_eq!(edit.remove(&a, "test0", None), Some(a.clone())); + drop(edit); + assert!(imported.links().next().is_none()); + } + + #[test] + fn imports_are_keyed_per_link() { + let mut imported = ImportedMcast::default(); + let wake = watch::Sender::new(0); + let ifnames = |imported: &ImportedMcast| -> Vec { + imported + .links() + .map(|(_, ifname)| ifname.to_string()) + .collect() + }; + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::1"); + + let mut edit = imported.edit(&wake); + assert_eq!(edit.insert(&a, "tfportrear0_0", 1), Inserted::New); + assert_eq!(edit.insert(&a, "tfportrear1_0", 2), Inserted::New); + drop(edit); + assert_eq!(*wake.borrow(), 1); + + assert_eq!( + imported.edit(&wake).insert(&a, "tfportrear0_0", 3), + Inserted::Unchanged + ); + assert_eq!(*wake.borrow(), 1); + assert_eq!(imported.links().count(), 2); + + assert_eq!( + imported.edit(&wake).remove(&a, "tfportrear0_0", None), + Some(a.clone()) + ); + assert_eq!(ifnames(&imported), vec!["tfportrear1_0"]); + + let mut edit = imported.edit(&wake); + edit.insert(&a, "tfportrear0_0", 4); + assert_eq!( + edit.extract_link(a.nexthop, "tfportrear0_0"), + HashSet::from([a.clone()]), + ); + drop(edit); + assert_eq!(ifnames(&imported), vec!["tfportrear1_0"]); + assert_eq!( + imported + .edit(&wake) + .extract_link(a.nexthop, "tfportrear1_0"), + HashSet::from([a]), + ); + assert!(imported.links().next().is_none()); + assert_eq!(*wake.borrow(), 4); + } + + #[test] + fn splice_reports_only_changed_imports() { + let (_dir, db) = test_db(); + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); + let splice = |routes: &HashSet| { + db.splice_imported_mcast(MulticastRibChange::new( + "test0", + routes, + &HashSet::new(), + )) + .imported + }; + + let routes = HashSet::from([a.clone()]); + assert_eq!(splice(&routes), routes); + assert!(splice(&routes).is_empty()); + + let mut wake = db.subscribe_mcast_sweep(); + wake.borrow_and_update(); + + let remetric = MulticastRoute { + origin: a.origin.clone().with_metric(5), + ..a + }; + let changed = splice(&HashSet::from([remetric.clone()])); + assert_eq!( + changed + .iter() + .map(|r| r.origin.metric()) + .collect::>(), + vec![5], + ); + assert!(splice(&HashSet::from([remetric])).is_empty()); + assert!(!wake.has_changed().unwrap()); + } + + fn check_multicast_removal_by_interface( + remove_routes: impl Fn(&Db, Ipv6Addr, &str) -> RemovedNexthopRoutes, + ) { + let (_dir, db) = test_db(); + let shared = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::1"); + let unique = route(mcast_origin("233.252.0.2", "ff04::2"), "fe80::1"); + let other = route(mcast_origin("233.252.0.3", "ff04::3"), "fe80::2"); + db.splice_imported_mcast(MulticastRibChange::new( + "tfportrear0_0", + &HashSet::from([shared.clone(), unique.clone(), other.clone()]), + &HashSet::new(), + )); + db.splice_imported_mcast(MulticastRibChange::new( + "tfportrear1_0", + &HashSet::from([shared.clone()]), + &HashSet::new(), + )); + + let mut rx = db.subscribe_mcast_sweep(); + let generation = *rx.borrow_and_update(); + let removed = remove_routes(&db, shared.nexthop, "tfportrear2_0"); + assert!(removed.multicast.is_empty()); + assert_eq!(db.imported_mcast_links().len(), 4); + assert_eq!(*rx.borrow_and_update(), generation); + + let removed = remove_routes(&db, shared.nexthop, "tfportrear0_0"); + assert_eq!( + removed.multicast, + HashSet::from([shared.clone(), unique.clone()]), + ); + assert_eq!( + db.imported_mcast_links() + .into_iter() + .collect::>(), + HashSet::from([ + (shared.clone(), "tfportrear1_0".to_string()), + (other.clone(), "tfportrear0_0".to_string()), + ]), + ); + let paths = + db.snapshot_mcast_paths([&shared.origin, &unique.origin], &NO_PEER); + assert_eq!(paths.imported(&shared.origin), Some(&shared)); + assert!(paths.imported(&unique.origin).is_none()); + assert_eq!(*rx.borrow_and_update(), generation + 1); + + let removed = remove_routes(&db, shared.nexthop, "tfportrear0_0"); + assert!(removed.multicast.is_empty()); + assert_eq!(*rx.borrow_and_update(), generation + 1); + + let removed = remove_routes(&db, shared.nexthop, "tfportrear1_0"); + assert_eq!(removed.multicast, HashSet::from([shared.clone()])); + assert_eq!( + db.imported_mcast_links(), + vec![(other, "tfportrear0_0".to_string())], + ); + assert!( + db.snapshot_mcast_paths([&shared.origin], &NO_PEER) + .imported(&shared.origin) + .is_none(), + ); + assert_eq!(*rx.borrow_and_update(), generation + 2); + } + + #[test] + fn expiry_preserves_multicast_on_other_interfaces() { + check_multicast_removal_by_interface(Db::remove_nexthop_routes); + } + + #[test] + fn downgrade_preserves_multicast_on_other_interfaces() { + check_multicast_removal_by_interface(Db::remove_nexthop_mcast_routes); + } + + #[test] + fn membership_changes_wake_subscribers() { + let (_dir, db) = test_db(); + let mut rx = db.subscribe_mcast_sweep(); + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); + let b = route(mcast_origin("233.252.0.2", "ff04::2"), "fe80::a"); + + import(&db, &HashSet::from([a.clone()])); + assert!(rx.has_changed().unwrap()); + rx.borrow_and_update(); + + import(&db, &HashSet::from([b.clone()])); + assert!(rx.has_changed().unwrap()); + rx.borrow_and_update(); + + db.splice_imported_mcast(MulticastRibChange::new( + "test0", + &HashSet::new(), + &HashSet::from([b.clone()]), + )); + assert!(rx.has_changed().unwrap()); + } + + #[test] + fn non_membership_changes_do_not_wake_subscribers() { + let (_dir, db) = test_db(); + let mut rx = db.subscribe_mcast_sweep(); + let a = route(mcast_origin("233.252.0.1", "ff04::1"), "fe80::a"); + let absent = route(mcast_origin("233.252.0.9", "ff04::9"), "fe80::b"); + + let pull_seq = db.next_mcast_seq(); + import(&db, &HashSet::from([a.clone()])); + rx.borrow_and_update(); + + import(&db, &HashSet::from([a.clone()])); + let mut rerouted = a.clone(); + rerouted.path.push(MulticastPathHop::new( + "other".into(), + "fe80::3".parse().unwrap(), + )); + import(&db, &HashSet::from([rerouted])); + + import(&db, &HashSet::new()); + db.splice_imported_mcast(MulticastRibChange::new( + "test0", + &HashSet::new(), + &HashSet::from([absent.clone()]), + )); + let removed = db + .splice_imported_mcast(MulticastRibChange { + ifname: "test0", + import: &HashSet::new(), + remove: &HashSet::from([a.clone()]), + imported_before: Some(pull_seq), + }) + .removed; + assert!(removed.is_empty()); + db.remove_nexthop_mcast_routes(absent.nexthop, "test0"); + db.remove_nexthop_routes(absent.nexthop, "test0"); - imported.insert(&a, 2); - assert_eq!(imported.extract_nexthop(a.nexthop), HashSet::from([a])); - assert!(imported.0.is_empty()); + assert!(!rx.has_changed().unwrap()); } } diff --git a/ddm/src/db/mcast/imported.rs b/ddm/src/db/mcast/imported.rs new file mode 100644 index 000000000..183373bc5 --- /dev/null +++ b/ddm/src/db/mcast/imported.rs @@ -0,0 +1,252 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The imported multicast set, made mutable through an [`ImportedMcastEdit`] +//! guard that wakes the multicast sweep when membership gets updated. + +use std::collections::{HashMap, HashSet}; +use std::net::Ipv6Addr; + +use ddm_api_types::db::MulticastRoute; +use ddm_api_types::net::MulticastOrigin; +use ddm_protocol::RouterId; +use tokio::sync::watch; + +#[derive(Debug, Clone)] +struct ImportedEntry { + route: MulticastRoute, + ifname: String, + seq: u64, +} + +/// A type wrapper over the set of imported multicast routes, keyed by origin +/// and then by nexthop and the interface that its peer was discovered on. +/// +/// A lookup per origin produces the path snapshot for an update. Each +/// origin's routes are sorted by nexthop and then by interface, with the +/// lowest-nexthop route first. +/// +/// Per-peer work (e.g., stale-route scans, expiry) visits each origin just +/// once. +// Note: We use a sorted `Vec` here over a `BTreeMap` because an origin rarely +// has more than a few nexthops. +#[derive(Debug, Default, Clone)] +pub struct ImportedMcast(HashMap>); + +fn find_link( + nexthops: &[ImportedEntry], + nexthop: Ipv6Addr, + ifname: &str, +) -> Result { + nexthops.binary_search_by(|entry| { + (entry.route.nexthop, entry.ifname.as_str()).cmp(&(nexthop, ifname)) + }) +} + +impl ImportedMcast { + /// Begin an edit operation. + /// + /// Dropping the returned guard wakes `wake`'s watched subscribers if the + /// edit added or removed an `(origin, nexthop, ifname)` entry. + pub(in crate::db) fn edit<'a>( + &'a mut self, + wake: &'a watch::Sender, + ) -> ImportedMcastEdit<'a> { + ImportedMcastEdit { + imported: self, + wake, + changed: false, + } + } + + /// Each imported route paired with the interface its peer was discovered + /// on. + pub(in crate::db) fn links( + &self, + ) -> impl Iterator { + self.0 + .values() + .flatten() + .map(|entry| (&entry.route, entry.ifname.as_str())) + } + + pub(in crate::db) fn shortest_path_routes<'a>( + &'a self, + peer_router_id: &'a RouterId, + ) -> impl Iterator { + self.0.keys().filter_map(move |origin| { + self.shortest_path(origin, peer_router_id) + }) + } + + /// Pick the route with the shortest path, breaking ties by the lowest + /// origin metric (first), then by the lowest nexthop address (e.g., + /// `fe80::1` < `fe80::2`), and finally by the interface name. + /// + /// Routes whose path contains `peer_router_id` (the peer an update gets + /// sent to) are omitted. That peer would discard such a path as a loop + /// and drop its route through us. + pub(in crate::db) fn shortest_path( + &self, + origin: &MulticastOrigin, + peer_router_id: &RouterId, + ) -> Option<&MulticastRoute> { + let key = |entry: &ImportedEntry| { + ( + entry.route.path.len(), + entry.route.origin.metric(), + entry.route.nexthop, + ) + }; + self.0 + .get(origin)? + .iter() + .filter(|entry| { + !entry + .route + .path + .iter() + .any(|hop| &hop.router_id == peer_router_id) + }) + .min_by(|a, b| { + key(a).cmp(&key(b)).then_with(|| a.ifname.cmp(&b.ifname)) + }) + .map(|entry| &entry.route) + } + + /// Return origins imported via `nexthop` on `ifname` before a `seq` + /// that are not contained in `announced`. + pub(in crate::db) fn stale( + &self, + nexthop: Ipv6Addr, + ifname: &str, + seq: u64, + announced: &HashSet, + ) -> Vec { + self.0 + .values() + .filter_map(|nexthops| { + find_link(nexthops, nexthop, ifname) + .ok() + .map(|i| &nexthops[i]) + }) + .filter(|entry| { + entry.seq < seq && !announced.contains(&entry.route.origin) + }) + .map(|entry| entry.route.origin.clone()) + .collect() + } +} + +/// The effect of [`ImportedMcastEdit::insert`] on the imported set. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(in crate::db) enum Inserted { + /// A new `(origin, nexthop, ifname)` entry. + New, + /// An existing entry whose path or metric changed. + Changed, + /// An existing entry refreshed without change. + Unchanged, +} + +/// An edit guard for [`ImportedMcast`]. +/// +/// Mutations record whether or not they added or removed an entry. If an +/// update occurred, then dropping the guard wakes the multicast sweep process. +pub(in crate::db) struct ImportedMcastEdit<'a> { + imported: &'a mut ImportedMcast, + wake: &'a watch::Sender, + changed: bool, +} + +impl ImportedMcastEdit<'_> { + /// Insert or refresh a `route` at `seq` number, reporting whether it added + /// a new `(origin, nexthop, ifname)` entry, changed an existing entry's + /// path or metric, or left it unchanged. + /// + /// Only a new entry wakes the sweep, because multicast membership depends + /// only on which `(nexthop, ifname)` links import a group. + pub(in crate::db) fn insert( + &mut self, + route: &MulticastRoute, + ifname: &str, + seq: u64, + ) -> Inserted { + let nexthops = self.imported.0.entry(route.origin.clone()).or_default(); + let inserted = match find_link(nexthops, route.nexthop, ifname) { + Ok(idx) => { + let entry = &mut nexthops[idx]; + entry.seq = seq; + if entry.route.path != route.path + || entry.route.origin.metric() != route.origin.metric() + { + entry.route = route.clone(); + Inserted::Changed + } else { + Inserted::Unchanged + } + } + Err(idx) => { + // Grow one slot at a time. + nexthops.reserve_exact(1); + nexthops.insert( + idx, + ImportedEntry { + route: route.clone(), + ifname: ifname.to_string(), + seq, + }, + ); + Inserted::New + } + }; + self.changed |= inserted == Inserted::New; + inserted + } + + pub(in crate::db) fn remove( + &mut self, + route: &MulticastRoute, + ifname: &str, + imported_before: Option, + ) -> Option { + let nexthops = self.imported.0.get_mut(&route.origin)?; + let idx = find_link(nexthops, route.nexthop, ifname).ok()?; + if imported_before.is_some_and(|seq| nexthops[idx].seq >= seq) { + return None; + } + let entry = nexthops.remove(idx); + if nexthops.is_empty() { + self.imported.0.remove(&route.origin); + } + self.changed = true; + Some(entry.route) + } + + pub(in crate::db) fn extract_link( + &mut self, + nexthop: Ipv6Addr, + ifname: &str, + ) -> HashSet { + let mut extracted = HashSet::new(); + self.imported.0.retain(|_, nexthops| { + if let Ok(idx) = find_link(nexthops, nexthop, ifname) { + extracted.insert(nexthops.remove(idx).route); + } + !nexthops.is_empty() + }); + self.changed |= !extracted.is_empty(); + extracted + } +} + +impl Drop for ImportedMcastEdit<'_> { + fn drop(&mut self) { + if self.changed { + self.wake.send_modify(|generation| { + *generation = generation.wrapping_add(1); + }); + } + } +} diff --git a/ddm/src/db/mod.rs b/ddm/src/db/mod.rs index 8b7fa85c0..f47c76087 100644 --- a/ddm/src/db/mod.rs +++ b/ddm/src/db/mod.rs @@ -13,10 +13,12 @@ use slog::{Logger, error}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::net::Ipv6Addr; use std::sync::{Arc, Mutex}; +use tokio::sync::watch; pub(crate) mod mcast; -pub use mcast::{ImportedMcast, McastPathSnapshot, MulticastRibChange}; +use mcast::ImportedMcast; +pub use mcast::{McastPathSnapshot, MulticastRibChange, SplicedMulticast}; /// The handle used to open a persistent key-value tree for originated /// prefixes. @@ -45,6 +47,10 @@ pub enum Error { pub struct Db { data: Arc>, persistent_data: sled::Db, + /// A generation counter incremented once per edit of the imported + /// multicast set that adds or removes an `(origin, nexthop, ifname)` + /// entry, waking [`crate::mcast::run`]. + mcast_wake: Arc>, log: Logger, } @@ -54,7 +60,7 @@ pub struct DbData { pub imported_tunnel: HashSet, /// Imported multicast routes, each paired with the value of /// `mcast_seq` at its most recent import. - pub imported_mcast: ImportedMcast, + pub(in crate::db) imported_mcast: ImportedMcast, pub mcast_seq: u64, pub external_peers: BTreeSet, } @@ -69,6 +75,7 @@ impl Db { Ok(Self { data: Arc::new(Mutex::new(DbData::default())), persistent_data: sled::open(db_path)?, + mcast_wake: Arc::new(watch::Sender::new(0)), log, }) } @@ -76,6 +83,11 @@ impl Db { lock!(self.data).clone() } + /// Subscribe to wakes on modifications made to the imported multicast set. + pub fn subscribe_mcast_sweep(&self) -> watch::Receiver { + self.mcast_wake.subscribe() + } + pub fn imported(&self) -> HashSet { lock!(self.data).imported.clone() } @@ -229,18 +241,28 @@ impl Db { } /// Remove all routes via `nexthop`. + /// + /// Underlay and tunnel routes are removed on every interface. Multicast + /// routes are removed only for the specific link `(nexthop, ifname)` entry. pub fn remove_nexthop_routes( &self, nexthop: Ipv6Addr, + ifname: &str, ) -> RemovedNexthopRoutes { - let mut data = lock!(self.data); - let underlay = - data.imported.extract_if(|x| x.nexthop == nexthop).collect(); - let tunnel = data - .imported_tunnel - .extract_if(|x| x.nexthop == nexthop) - .collect(); - let multicast = data.imported_mcast.extract_nexthop(nexthop); + let (underlay, tunnel, multicast) = { + let mut data = lock!(self.data); + let underlay = + data.imported.extract_if(|x| x.nexthop == nexthop).collect(); + let tunnel = data + .imported_tunnel + .extract_if(|x| x.nexthop == nexthop) + .collect(); + let multicast = data + .imported_mcast + .edit(&self.mcast_wake) + .extract_link(nexthop, ifname); + (underlay, tunnel, multicast) + }; RemovedNexthopRoutes { underlay, diff --git a/ddm/src/exchange/mcast.rs b/ddm/src/exchange/mcast.rs index 0cb9b6a7b..93de390fc 100644 --- a/ddm/src/exchange/mcast.rs +++ b/ddm/src/exchange/mcast.rs @@ -151,7 +151,8 @@ mod tests { if !originated.is_empty() { db.originate_mcast(&originated).unwrap(); } - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &imported, &HashSet::new(), )); @@ -438,7 +439,8 @@ mod tests { let still_imported = origin("233.252.0.1", "ff04::1", 10); let local_origin = origin("233.252.0.2", "ff04::2", 10); let remaining_hop = hop("remaining", 2); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([MulticastRoute { origin: still_imported.clone(), nexthop: "fe80::2".parse().unwrap(), @@ -495,7 +497,8 @@ mod tests { let other_hop = hop("other", 3); let local_hop = hop("local", 2); let withdrawn_hop = hop("withdrawn", 4); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &HashSet::from([ MulticastRoute { origin: alternate.clone(), diff --git a/ddm/src/exchange/runtime/mcast.rs b/ddm/src/exchange/runtime/mcast.rs index f34210363..1538e25a8 100644 --- a/ddm/src/exchange/runtime/mcast.rs +++ b/ddm/src/exchange/runtime/mcast.rs @@ -144,15 +144,16 @@ pub(super) fn apply_multicast_update( imported_before: Option, ) -> AppliedMulticastUpdate { let batch = MulticastRibChange { + ifname: &ctx.config.if_name, import: &import, remove, imported_before, }; - let removed = ctx.db.update_imported_mcast(batch); + let spliced = ctx.db.splice_imported_mcast(batch); AppliedMulticastUpdate { - imported: import, - removed, + imported: spliced.imported, + removed: spliced.removed, hop: MulticastPathHop::new(ctx.router_id.clone(), ctx.config.addr), } } @@ -229,8 +230,8 @@ pub(super) fn partition_multicast_update( path: Vec::new(), }); } - // A simultaneous import would receive a sequence number above the pull - // cutoff. + // A simultaneous import would receive a sequence number at or above + // the pull sequence. import.retain(|route| !remove.contains(route)); (import, remove) @@ -359,17 +360,21 @@ mod tests { let log = Logger::root(Discard, o!()); let db = Db::new(dir.path().as_str(), log).unwrap(); let existing = HashSet::from([route(1)]); - db.update_imported_mcast(MulticastRibChange::new( + db.splice_imported_mcast(MulticastRibChange::new( + "test0", &existing, &HashSet::new(), )); let (import, remove) = partition(&update); - let removed = - db.update_imported_mcast(MulticastRibChange::new(&import, &remove)); + let removed = db + .splice_imported_mcast(MulticastRibChange::new( + "test0", &import, &remove, + )) + .removed; assert_eq!(removed, existing); - assert!(db.imported_mcast().is_empty()); + assert!(db.imported_mcast_links().is_empty()); assert!(import.is_empty()); } diff --git a/ddm/src/exchange/runtime/mod.rs b/ddm/src/exchange/runtime/mod.rs index af0264259..d85f40d59 100644 --- a/ddm/src/exchange/runtime/mod.rs +++ b/ddm/src/exchange/runtime/mod.rs @@ -346,8 +346,9 @@ fn fetch_pull_body( /// Withdrawals are synthesized only after the complete snapshot has been /// imported. A failed pull imports nothing and synthesizes no withdrawals. /// -/// `started_at` is the cutoff for this pull. A route updated during the read -/// stays out of stale-route removal, including a refresh after the scan. +/// `pull_seq` is recorded before reading routes from the peer. A route +/// updated during the read stays out of stale-route removal, including a +/// refresh after the scan. /// /// Underlay and tunnel withdrawals remain push-only at every version. pub(crate) fn pull( @@ -359,7 +360,7 @@ pub(crate) fn pull( mode: UpdateMode, ) -> Result<(), ExchangeError> { let version = session.version; - let started_at = ctx.db.next_mcast_seq(); + let pull_seq = ctx.db.next_mcast_seq(); if stop.load(Ordering::Relaxed) { return Ok(()); @@ -390,7 +391,7 @@ pub(crate) fn pull( let withdraw: HashSet = ctx .db - .stale_mcast_by_nexthop(addr, started_at, &announced) + .stale_mcast_by_nexthop(addr, &ctx.config.if_name, pull_seq, &announced) .into_iter() .map(|origin| MulticastPathVector { origin, @@ -410,7 +411,7 @@ pub(crate) fn pull( ctx, addr, mode, - Some(started_at), + Some(pull_seq), session, ); } @@ -877,9 +878,9 @@ fn handle_update( if session.is_canceled() { redistribute = redistribute.withdrawals_only(); - if redistribute.is_empty() { - return; - } + } + if redistribute.is_empty() { + return; } if let Err(e) = ctx.tx.send(Event::Peer(PeerEvent::Redistribute { @@ -1121,8 +1122,8 @@ mod tests { use crate::defaults::EXCHANGE_TIMEOUT; use crate::discovery::Version; use crate::sm::{ - Config, Event, ExchangeSession, InterfaceState, PeerIdentity, - SessionStats, SmContext, + Config, Event, ExchangeSession, InterfaceState, PeerEvent, + PeerIdentity, SessionStats, SmContext, }; use ddm_api_types::db::{MulticastRoute, RouterKind}; use ddm_api_types::net::{OverlayMulticast, UnderlayMulticastIpv6, Vni}; @@ -1274,7 +1275,7 @@ mod tests { struct ExchangeFixture { ctx: SmContext, - _events: Receiver, + events: Receiver, _dir: camino_tempfile::Utf8TempDir, } @@ -1319,7 +1320,7 @@ mod tests { discovery_stop: None, first_run: true, }, - _events: events, + events, _dir: dir, } } @@ -1416,6 +1417,68 @@ mod tests { }); } + fn redistributed_multicast(events: &Receiver) -> Vec { + events + .try_iter() + .filter_map(|event| match event { + Event::Peer(PeerEvent::Redistribute { update, .. }) => Some( + update.multicast.as_ref().map_or(0, |m| m.announce.len()), + ), + _ => None, + }) + .collect() + } + + #[test] + fn transit_redistributes_only_changed_multicast() { + let rt = Runtime::new().unwrap(); + let _guard = rt.enter(); + let mut fixture = ExchangeFixture::new(&rt); + fixture.ctx.config.kind = RouterKind::Transit; + let session = publish(&fixture.ctx, Version::V4); + let server = fixture.server(session); + let (update, route) = multicast_announcement(); + + push_v4(&rt, &server, &update); + push_v4(&rt, &server, &update); + assert_eq!(redistributed_multicast(&fixture.events), vec![1]); + + let rerouted: v4::Update = + v4::MulticastUpdate::announce(HashSet::from([ + v4::MulticastPathVector { + origin: route.origin, + path: vec![ + v4::MulticastPathHop::new("peer".into(), PEER), + v4::MulticastPathHop::new("far".into(), PEER), + ], + }, + ])) + .into(); + push_v4(&rt, &server, &rerouted); + push_v4(&rt, &server, &rerouted); + assert_eq!(redistributed_multicast(&fixture.events), vec![1]); + + let missing = v4::MulticastOrigin::new( + OverlayMulticast::new("233.252.0.9".parse().unwrap()).unwrap(), + UnderlayMulticastIpv6::new("ff04::109".parse().unwrap()).unwrap(), + Vni::new(77).unwrap(), + None, + ) + .unwrap(); + let withdraw: v4::Update = + v4::MulticastUpdate::withdraw(HashSet::from([ + v4::MulticastPathVector { + origin: missing, + path: Vec::new(), + }, + ])) + .into(); + push_v4(&rt, &server, &withdraw); + assert!(redistributed_multicast(&fixture.events).is_empty()); + + rt.block_on(server.teardown()); + } + #[test] fn stale_push_cannot_restore_multicast_after_session_replacement() { let rt = Runtime::new().unwrap(); @@ -1426,26 +1489,29 @@ mod tests { let old_server = fixture.server(old_session); let (update, route) = multicast_announcement(); + let imported = vec![(route.clone(), ctx.config.if_name.clone())]; push_v4(&rt, &old_server, &update); - assert_eq!(ctx.db.imported_mcast(), HashSet::from([route.clone()])); + assert_eq!(ctx.db.imported_mcast_links(), imported); { let mut session = ctx.iface.session(); *session = Some(Arc::new(ExchangeSession::new(Version::V3))); - let removed = ctx.db.remove_nexthop_mcast_routes(PEER); + let removed = ctx + .db + .remove_nexthop_mcast_routes(PEER, &ctx.config.if_name); assert_eq!(removed.multicast, HashSet::from([route.clone()])); } - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); push_v4(&rt, &old_server, &update); - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); let fresh_session = publish(ctx, Version::V4); push_v4(&rt, &old_server, &update); - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); let fresh_server = fixture.server(fresh_session); push_v4(&rt, &fresh_server, &update); - assert_eq!(ctx.db.imported_mcast(), HashSet::from([route])); + assert_eq!(ctx.db.imported_mcast_links(), imported); rt.block_on(fresh_server.teardown()); rt.block_on(old_server.teardown()); @@ -1474,7 +1540,7 @@ mod tests { .unwrap() }); assert_eq!(error.message, "multicast exchange requires a V4 peer"); - assert!(fixture.ctx.db.imported_mcast().is_empty()); + assert!(fixture.ctx.db.imported_mcast_links().is_empty()); rt.block_on(server.teardown()); } @@ -1489,7 +1555,7 @@ mod tests { let server = fixture.server(session); push_v4(&rt, &server, &v4::Update::default()); - assert!(fixture.ctx.db.imported_mcast().is_empty()); + assert!(fixture.ctx.db.imported_mcast_links().is_empty()); rt.block_on(server.teardown()); } @@ -1506,7 +1572,7 @@ mod tests { session.cancel(); push_v4(&rt, &server, &update); - assert!(ctx.db.imported_mcast().is_empty()); + assert!(ctx.db.imported_mcast_links().is_empty()); rt.block_on(server.teardown()); } diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index 19326ec0a..a1421a583 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -7,6 +7,7 @@ pub mod db; pub mod defaults; pub mod discovery; pub mod exchange; +pub mod mcast; pub mod oxstats; pub mod sm; #[cfg(all(feature = "backend", target_os = "illumos"))] @@ -15,6 +16,7 @@ pub mod sys; pub const COMPONENT_DDM: &str = "ddm"; pub const MOD_ADMIN: &str = "admin"; pub const MOD_EXCHANGE: &str = "exchange"; +pub(crate) const DDM_TAG: &str = "ddmd"; #[macro_export] macro_rules! err { diff --git a/ddm/src/mcast.rs b/ddm/src/mcast.rs new file mode 100644 index 000000000..c35c6a822 --- /dev/null +++ b/ddm/src/mcast.rs @@ -0,0 +1,1545 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Underlay multicast membership programming. +//! +//! Programs multicast routes DDM has imported from peers into the local +//! switch's underlay multicast replication member set operated by the +//! Dendrite (DPD) dataplane. This is the multicast analog of +//! unicast's import-to-DPD setup which `add_underlay_routes` performs +//! in-process. With `ddmd` owning its input locally, i.e., the imported +//! multicast set along with the interface each route's peer was discovered +//! on ([`imported_mcast_links`]), it programs DPD directly. +//! +//! ## Membership Reconciliation +//! +//! Membership reconciliation runs by way of a periodic sweep over all active +//! underlay groups, similar to how the unicast lower-half's resync works. When +//! updates are made to the imported multicast set, whether from the exchange +//! update handler, peer expiry, or protocol downgrade, they all operate through +//! [`Db`], which wakes the sweep mechanism early (see +//! [`Db::subscribe_mcast_sweep`]). Otherwise, the sweep waits out +//! `RECONCILE_INTERVAL` after each pass before starting the next one. +//! +//! On each pass, we recompute the wanted member set for every group containing +//! imports or programmed members (withdrawn groups included), then repair +//! drift, drain withdrawn groups, and program members we could not +//! previously resolve (e.g., the group hadn't existed yet or an interface had +//! not resolved to a switch link). +//! +//! The sweep removes only members recorded as programmed since this `ddmd` +//! process started up, akin to ddmd's handling of unicast and tunnel routes. +//! A matching read or a successful write records the members derived from +//! imports, including those already in DPD. A failed write retains the prior +//! record and adds the wanted members because DPD may have already applied it. +//! Programmed members are removed when no import can derive them, and a group +//! stays in the sweep until it has no imports or programmed members. +//! +//! A member no import has derived since this process started, such as one left +//! by an earlier `ddmd` process, remains in place. +//! +//! DPD replaces the full member set on each write. Unicast and tunnel updates +//! can add or remove entries without reading back the full set. However, inline +//! multicast updates would need serialization per group along with a retry +//! mechanism. The sweep provides both: a single task reconciling groups one at +//! a time, and subsequent passes retrying failures. Each pass consolidates +//! route updates into at most one read-modify-write per group, avoiding +//! unnecessary writes when the computed member set matches DPD. +//! +//! ## DPD Groups and Membership +//! +//! Each imported [`MulticastRoute`] identifies an advertising peer (`nexthop`), +//! and is kept with the interface that peer was discovered on, akin to how +//! unicast routes work. This interface supplies the switch `(PortId, LinkId)` +//! tuple entry, while resolved members are deduplicated per underlay group +//! address (which is DPD's group key); the overlay group is not part of this +//! key. +//! +//! Nexus creates each underlay group and deletes it by tag. Within an existing +//! group, `ddmd` writes the members whose packet copies keep their Geneve +//! encapsulation (`Underlay` in DPD) and preserves the members whose copies +//! are decapsulated before delivery (`External`). +//! Rack-originated traffic already carries Geneve encapsulation addressed to +//! the underlay group and bypasses ingress NAT ([DPD packet paths]). +//! +//! Writes pass back the tag from the same pass's read, which the +//! [DPD update API] checks against the group's tag. So `ddmd` only ever +//! replaces the member list (never the tag, never the group itself). +//! +//! See [RFD 488] for the multicast architecture. +//! +//! [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +//! [`imported_mcast_links`]: crate::db::Db::imported_mcast_links +//! [DPD packet paths]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/dpd/src/mcast/mod.rs#L22-L25 +//! [DPD update API]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/dpd-api/src/lib.rs#L2496-L2512 + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::time::Duration; + +use ddm_api_types::db::MulticastRoute; +use ddm_api_types::net::UnderlayMulticastIpv6; +use dpd_client::types::{ + Direction, MulticastGroupMember, MulticastGroupUnderlayResponse, + MulticastGroupUpdateUnderlayEntry, PortId, +}; +use dpd_client::{Client, ClientState}; +use dpd_types_versions::latest::mcast::MulticastTag; +use hyper::StatusCode; +use slog::{Logger, debug, error, info, warn}; +use tokio::sync::watch; + +use crate::DDM_TAG; +use crate::db::Db; +use crate::sm::DpdConfig; + +/// The interval between the sweep's periodic membership reconciliation passes. +const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); + +/// Run the multicast membership reconciliation sweep until it's explicitly +/// canceled. +/// +/// This reconciles imported and draining groups on wakes from +/// [`Db::subscribe_mcast_sweep`] or via periodic ticks. Switch links are +/// resolved on each pass from the interface each route's peer was discovered +/// on, including dynamically added external peers. +/// +/// Each new `ddmd` process starts without a record of programmed membership, +/// which means that existing members it never derives again could remain +/// in DPD indefinitely. +/// +/// TODO: Recover forwarding state after restart across multicast, unicast, +/// and tunnel routes. We retain existing behavior here: [unicast and +/// tunnel cleanup] also uses only the current imported set. +/// +/// TODO: Count sweep outcomes (passes, writes, failures, missing groups) in +/// [`RouterStats`] once the `ddm-router.toml` schema can export them. +/// +/// [`RouterStats`]: crate::admin::RouterStats +/// [unicast and tunnel cleanup]: https://github.com/oxidecomputer/maghemite/blob/04a64f5b6b80dadbfa6ece123a30304b2f82414e/ddm/src/sm/state.rs#L357-L378 +pub async fn run(db: Db, dpd: DpdConfig, log: Logger) { + let mcast = db.subscribe_mcast_sweep(); + let client = Client::new( + &format!("http://{}:{}", dpd.host, dpd.port), + ClientState { + tag: DDM_TAG.into(), + log: log.clone(), + }, + ); + run_with_client(db, &client, mcast, log).await; +} + +async fn run_with_client( + db: Db, + client: &C, + mut mcast: watch::Receiver, + log: Logger, +) { + let mut programmed: ProgrammedMembers = HashMap::new(); + let mut unresolved = HashSet::new(); + + loop { + // Before reading the imported set, we mark the current generation + // number as seen. If an update advances the generation during this + // pass, another pass starts without waiting for the timer. + mcast.borrow_and_update(); + + // For each group within a pass, the imported set is the same snapshot, + // which means we can read it once here, avoiding a per-group lookup. + let imported = db.imported_mcast_links(); + + reconcile_pass( + &mut programmed, + &mut unresolved, + &imported, + client, + &log, + ) + .await; + + // Wait for a wake cycle or one idle reconcile interval. + // + // Both waits (below) are cancel-safe. + // + // If an update races with selection of the sleep branch, the next + // pass will mark it seen and read the set that includes it. + tokio::select! { + Ok(()) = mcast.changed() => {} + _ = tokio::time::sleep(RECONCILE_INTERVAL) => {} + } + } +} + +/// The per-group `Underlay` members programmed since this sweep started. +type ProgrammedMembers = HashMap; +type Members = HashSet; + +/// Reconcile active groups with imports or programmed members. +/// +/// An empty result drops the group's entry, recorded nothing for it. +/// A group that still contains imports (e.g., a group Nexus has not yet +/// created) is picked back up on the next pass via those imports. +async fn reconcile_pass( + programmed: &mut ProgrammedMembers, + unresolved: &mut HashSet, + imported: &[(MulticastRoute, String)], + client: &C, + log: &Logger, +) { + let mut by_group: HashMap> = + HashMap::new(); + for (route, ifname) in imported { + by_group + .entry(route.origin.underlay_group()) + .or_default() + .push(ifname); + } + let groups: BTreeSet = + by_group.keys().chain(programmed.keys()).copied().collect(); + let links = resolve_links( + imported.iter().map(|(_, ifname)| ifname.as_str()), + unresolved, + log, + ); + + for group_ip in groups { + let ifnames = by_group.get(&group_ip).map_or(&[][..], Vec::as_slice); + let prev = programmed.remove(&group_ip).unwrap_or_default(); + let members = reconcile_group( + group_ip, + ifnames.iter().copied(), + &links, + prev, + client, + log, + ) + .await; + + if !members.is_empty() { + programmed.insert(group_ip, members); + } + } +} + +/// The outcome of writing a group's member list to DPD. +#[derive(Clone, Copy)] +enum WriteOutcome { + /// Members were written. + Updated, + /// The write failed and should be retried on the next pass. + Retry, +} + +/// The outcome of reading a group's state from DPD. +#[derive(Clone)] +enum FetchOutcome { + /// The fetched group exists, and its authorization tag and current members + /// were read without issue. + Found(MulticastTag, Vec), + /// The fetched group does not exist in DPD, maybe because Nexus has not + /// created it yet or because it was deleted before this operation. + Missing, + /// The read operation failed, making the group's state unknown on this + /// pass, for which a retry should ensue. + Retry, +} + +/// Behavior encompassing DPD group operations that the reconciliation relies +/// on. +/// +/// NOTE: tests implement this trait in order to inject failures and pause +/// requests. +trait GroupClient { + /// Read an underlay group's current members along with its authorization + /// tag. + async fn fetch_group( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + ) -> FetchOutcome; + + /// Write `members` to an underlay group, authorized by its current `tag`. + async fn write_members( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + tag: &MulticastTag, + members: Vec, + ) -> WriteOutcome; +} + +impl GroupClient for Client { + /// This distinguishes a group that is genuinely missing + /// (`FetchOutcome::Missing`) from one with state that could not be read + /// (`FetchOutcome::Retry`) at the time of execution. + /// + /// Therefore, a withdrawn group is not dropped from the sweep when + /// encountering a transient read failure before its members are confirmed + /// drained. + async fn fetch_group( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + ) -> FetchOutcome { + match self.multicast_group_get_underlay(&group_ip).await { + Ok(resp) => { + let MulticastGroupUnderlayResponse { tag, members, .. } = + resp.into_inner(); + MulticastTag::try_from(tag) + .map(|tag| FetchOutcome::Found(tag, members)) + .unwrap_or_else(|e| { + error!( + log, + "tag for underlay group {group_ip} is invalid, \ + retrying next pass: {e}" + ); + FetchOutcome::Retry + }) + } + // The creation and deletion of an underlay group is owned by Nexus, + // which creates the group before traffic flows toward it. + // + // We bypass membership programming until the group exists. + Err(e) if e.status() == Some(StatusCode::NOT_FOUND) => { + debug!( + log, + "underlay group {group_ip} does not exist yet, bypassing \ + until Nexus creates it" + ); + FetchOutcome::Missing + } + Err(e) => { + warn!(log, "failed to get underlay group {group_ip}: {e}"); + FetchOutcome::Retry + } + } + } + + /// Any failure, including a 404 or a 400 for a retag since the read, is + /// [`WriteOutcome::Retry`]. DPD also returns 404 for a missing ASIC + /// resource on a group that still exists, so the next pass's read decides + /// whether the group is gone. + async fn write_members( + &self, + log: &Logger, + group_ip: UnderlayMulticastIpv6, + tag: &MulticastTag, + members: Vec, + ) -> WriteOutcome { + let body = MulticastGroupUpdateUnderlayEntry { members }; + match self + .multicast_group_update_underlay(&group_ip, tag, &body) + .await + { + Ok(_) => WriteOutcome::Updated, + Err(dpd_client::Error::ErrorResponse(resp)) + if resp.error_code.as_deref() + == Some(dpd_client::MCAST_TAG_MISMATCH_ERROR_CODE) => + { + debug!( + log, + "underlay group {group_ip} tag changed since the read, \ + retrying next pass" + ); + WriteOutcome::Retry + } + Err(e) => { + warn!( + log, + "failed to update underlay group {group_ip} members: {e}" + ); + WriteOutcome::Retry + } + } + } +} + +/// Aggregate one underlay group's wanted replication members. +/// +/// `ifnames` are the interfaces on which the peers behind an underlay group's +/// imports were discovered. +/// Interfaces that don't resolve to a switch link during this pass do not +/// contribute toward membership. +/// +/// Members are derived from each import's interface. All distinct downstream +/// links convey distinct interfaces, making each of these its own member. +/// Subscribers reached through a duplicate downstream link collapse into one +/// member because a single egress port toward that peer is sufficient enough +/// and the next set of hop handlers all fan-out further. +fn group_members<'a>( + ifnames: impl IntoIterator, + links: &HashMap<&str, MulticastGroupMember>, +) -> Members { + ifnames + .into_iter() + .filter_map(|ifname| links.get(ifname).cloned()) + .collect() +} + +/// Resolve each distinct interface in `imported` to an `Underlay` member once +/// per pass. +/// +/// Interfaces that cannot resolve to a switch link, or that resolve to a +/// non-rear port, are not retained. +fn resolve_links<'a>( + imported: impl IntoIterator, + unresolved: &mut HashSet, + log: &Logger, +) -> HashMap<&'a str, MulticastGroupMember> { + let ifnames: BTreeSet<&str> = imported.into_iter().collect(); + unresolved.retain(|ifname| ifnames.contains(ifname.as_str())); + + ifnames + .into_iter() + .filter_map(|ifname| { + let (port_id, link_id) = + mg_common::tfport::port_link_from_untagged_ifname(ifname) + .inspect_err(|e| { + if !unresolved.contains(ifname) { + unresolved.insert(ifname.to_string()); + warn!( + log, + "cannot resolve interface {ifname} to a \ + switch link, omitting as multicast \ + member: {e}" + ) + } + }) + .ok()?; + + // TODO: Allow QSFP members once inter-rack replication has loop + // prevention (e.g., RPF or a designated forwarder). + if !matches!(port_id, PortId::Rear(_)) { + if !unresolved.contains(ifname) { + unresolved.insert(ifname.to_string()); + debug!( + log, + "skipping multicast member on {ifname} ({port_id}), \ + cross-rack replication is not supported" + ); + } + return None; + } + unresolved.remove(ifname); + Some(( + ifname, + MulticastGroupMember { + port_id, + link_id, + direction: Direction::Underlay, + }, + )) + }) + .collect() +} + +/// Reconcile an underlay group's members in DPD against the imported multicast +/// routes DDM contains, returning the members to record as programmed. +/// +/// With wanted members `D`, existing DPD members `E`, and previously +/// programmed members `P`, the write consists of `D ∪ (E \ P)`. +/// +/// Existing members outside of `P` are preserved, including those left over +/// from an earlier run. When no longer wanted, members in `P` are removed. +/// +/// # Notes +/// +/// - A matching read or a successful write records `D`, including any existing +/// member it must now account for. +/// - A failed read preserves `P`, as the next pass may need to remove +/// members no longer wanted. +/// - A failed write records `P ∪ D`, as DPD may have already applied the +/// operation. +/// - A missing group records nothing whatsoever. +async fn reconcile_group<'a, C: GroupClient>( + group_ip: UnderlayMulticastIpv6, + ifnames: impl IntoIterator, + links: &HashMap<&str, MulticastGroupMember>, + programmed: Members, + client: &C, + log: &Logger, +) -> Members { + let wanted = group_members(ifnames, links); + + let (tag, existing) = match client.fetch_group(log, group_ip).await { + FetchOutcome::Found(tag, existing) => { + (tag, existing.into_iter().collect::()) + } + // The group is missing from DPD -> Nexus has not created it or it has + // been deleted, holding 0 members. + // + // We keep the group while it still has imports, giving way for a later + // pass to program it once it exists. + FetchOutcome::Missing => return Members::new(), + // The read state is unknown at this point. + // + // We keep the programmed members around for the next pass to remove + // what was programmed. + FetchOutcome::Retry => return programmed, + }; + + let mut to_write = &existing - &programmed; + to_write.extend(wanted.iter().cloned()); + if to_write == existing { + return wanted; + } + + let members = to_write.into_iter().collect(); + match client.write_members(log, group_ip, &tag, members).await { + WriteOutcome::Updated => { + info!(log, "updated underlay group {group_ip} members"); + wanted + } + WriteOutcome::Retry => &programmed | &wanted, + } +} + +#[cfg(test)] +mod tests { + use std::convert::Infallible; + use std::net::{IpAddr, Ipv6Addr}; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use ddm_api_types::net::{MulticastOrigin, OverlayMulticast, Vni}; + use dpd_client::types::LinkId; + use http_body_util::{BodyExt, Full}; + use hyper::body::{Bytes, Incoming}; + use hyper::server::conn::http1; + use hyper::service::service_fn; + use hyper::{Method, Request, Response}; + use hyper_util::rt::TokioIo; + use slog::Drain; + use tokio::net::TcpListener; + use tokio::sync::{Notify, oneshot}; + + use super::*; + + fn underlay(last: u16) -> UnderlayMulticastIpv6 { + UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0, last, + )) + .unwrap() + } + + fn route( + nexthop: Ipv6Addr, + group: UnderlayMulticastIpv6, + ) -> MulticastRoute { + MulticastRoute { + origin: MulticastOrigin::new( + OverlayMulticast::new(IpAddr::V6(Ipv6Addr::new( + 0xff0e, 0, 0, 0, 0, 0, 0, 1, + ))) + .unwrap(), + group, + Vni::new(77).unwrap(), + None, + ) + .unwrap(), + nexthop, + path: Vec::new(), + } + } + + fn learned( + imported: &HashSet, + links: &HashMap, + ) -> Vec<(MulticastRoute, String)> { + imported + .iter() + .map(|route| { + let ifname = links.get(&route.nexthop).map_or_else( + || "vioif0".to_string(), + |(port_id, link_id)| { + format!("tfport{port_id}_{}", link_id.0) + }, + ); + (route.clone(), ifname) + }) + .collect() + } + + fn group_ifnames( + group: UnderlayMulticastIpv6, + learned: &[(MulticastRoute, String)], + ) -> impl Iterator { + learned + .iter() + .filter(move |(route, _)| route.origin.underlay_group() == group) + .map(|(_, ifname)| ifname.as_str()) + } + + fn resolved_members( + group: UnderlayMulticastIpv6, + imported: &HashSet, + links: &HashMap, + ) -> Members { + let learned = learned(imported, links); + let log = Logger::root(slog::Discard, slog::o!()); + group_members( + group_ifnames(group, &learned), + &resolve_links( + learned.iter().map(|(_, ifname)| ifname.as_str()), + &mut HashSet::new(), + &log, + ), + ) + } + + fn rear(port: &str, link: u8) -> (PortId, LinkId) { + ( + PortId::Rear(dpd_client::types::Rear::try_from(port).unwrap()), + LinkId(link), + ) + } + + fn member(port: &str, link: u8) -> MulticastGroupMember { + let (port_id, link_id) = rear(port, link); + MulticastGroupMember { + port_id, + link_id, + direction: Direction::Underlay, + } + } + + #[test] + fn unresolved_interfaces_warn_once_until_removed() { + let warnings = Arc::new(AtomicUsize::new(0)); + let skips = Arc::new(AtomicUsize::new(0)); + let (warned, skipped) = (warnings.clone(), skips.clone()); + let drain = slog::Filter::new(slog::Discard, move |record| { + match record.level() { + slog::Level::Warning => warned.fetch_add(1, Ordering::Relaxed), + slog::Level::Debug => skipped.fetch_add(1, Ordering::Relaxed), + _ => 0, + }; + true + }); + let log = Logger::root(drain.ignore_res(), slog::o!()); + let mut unresolved = HashSet::new(); + + let links = resolve_links( + ["vioif0", "vioif0", "bad0", "tfportqsfp0_0", "tfportrear0_0"], + &mut unresolved, + &log, + ); + assert_eq!( + links, + HashMap::from([("tfportrear0_0", member("rear0", 0))]), + ); + assert_eq!(warnings.load(Ordering::Relaxed), 2); + assert_eq!(skips.load(Ordering::Relaxed), 1); + assert_eq!( + unresolved, + HashSet::from([ + "vioif0".to_string(), + "bad0".to_string(), + "tfportqsfp0_0".to_string(), + ]), + ); + + resolve_links( + ["vioif0", "bad0", "tfportqsfp0_0", "tfportrear0_0"], + &mut unresolved, + &log, + ); + assert_eq!(warnings.load(Ordering::Relaxed), 2); + assert_eq!(skips.load(Ordering::Relaxed), 1); + + resolve_links(["bad0"], &mut unresolved, &log); + assert_eq!(unresolved, HashSet::from(["bad0".to_string()])); + assert_eq!(warnings.load(Ordering::Relaxed), 2); + + resolve_links( + ["vioif0", "bad0", "tfportqsfp0_0"], + &mut unresolved, + &log, + ); + assert_eq!(warnings.load(Ordering::Relaxed), 3); + assert_eq!(skips.load(Ordering::Relaxed), 2); + + resolve_links([], &mut unresolved, &log); + assert!(unresolved.is_empty()); + } + + #[test] + fn membership_follows_links() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let both = vec![ + (route(peer, group), "tfportrear0_0".to_string()), + (route(peer, group), "tfportrear1_0".to_string()), + ]; + let one = vec![(route(peer, group), "tfportrear1_0".to_string())]; + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + let mut programmed = HashMap::new(); + let mut unresolved = HashSet::new(); + rt.block_on(reconcile_pass( + &mut programmed, + &mut unresolved, + &both, + &mock, + &log, + )); + assert_eq!( + programmed, + HashMap::from([( + group, + set(&[member("rear0", 0), member("rear1", 0)]) + )]) + ); + + rt.block_on(reconcile_pass( + &mut programmed, + &mut unresolved, + &one, + &mock, + &log, + )); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear1", 0)]))]) + ); + let writes = mock.writes(); + assert_eq!(writes.len(), 2); + assert_eq!( + set(&writes[0]), + set(&[member("rear0", 0), member("rear1", 0)]) + ); + assert_eq!(writes[1], vec![member("rear1", 0)]); + } + + #[test] + fn same_link_peers_share_membership() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + // Two distinct peers that resolve to the same switch link. Replicating + // twice out one (PortId, LinkId) would duplicate delivery on that link, + // so the members collapse to one. The dedup keys on the resolved link, + // not on the next hop. + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear0", 0)), + ]); + + let members = resolved_members(group, &imported, &peer_links); + assert_eq!(members, set(&[member("rear0", 0)])); + } + + #[test] + fn unresolved_interfaces_are_skipped() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(3); + + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + let members = resolved_members(group, &imported, &peer_links); + assert_eq!(members, set(&[member("rear0", 0)])); + } + + /// The sender signals that a fetch started, and the receiver waits for the + /// test to resume it. + type FetchChannels = (oneshot::Sender<()>, oneshot::Receiver<()>); + + /// Records write operations and makes successful updates visible to + /// fetches. + struct MockDpd { + fetch: std::sync::Mutex, + fetch_by_group: + std::sync::Mutex>, + write_outcome: WriteOutcome, + writes: std::sync::Mutex< + Vec<(UnderlayMulticastIpv6, Vec)>, + >, + fetch_channels: std::sync::Mutex>, + fetch_result_tx: + std::sync::Mutex>>, + write_attempted: Notify, + } + + impl MockDpd { + fn new(fetch: FetchOutcome, write_outcome: WriteOutcome) -> Self { + Self { + fetch: std::sync::Mutex::new(fetch), + fetch_by_group: std::sync::Mutex::new(HashMap::new()), + write_outcome, + writes: std::sync::Mutex::new(Vec::new()), + fetch_channels: std::sync::Mutex::new(None), + fetch_result_tx: std::sync::Mutex::new(None), + write_attempted: Notify::new(), + } + } + + fn writes(&self) -> Vec> { + self.group_writes() + .into_iter() + .map(|(_, members)| members) + .collect() + } + + fn group_writes( + &self, + ) -> Vec<(UnderlayMulticastIpv6, Vec)> { + self.writes.lock().unwrap().clone() + } + } + + impl GroupClient for MockDpd { + async fn fetch_group( + &self, + _log: &Logger, + group_ip: UnderlayMulticastIpv6, + ) -> FetchOutcome { + let channels = self.fetch_channels.lock().unwrap().take(); + if let Some((started, resume)) = channels { + started.send(()).unwrap(); + resume.await.unwrap(); + } + let outcome = + self.fetch_by_group.lock().unwrap().get(&group_ip).cloned(); + let outcome = + outcome.unwrap_or_else(|| self.fetch.lock().unwrap().clone()); + if let Some(tx) = self.fetch_result_tx.lock().unwrap().take() { + assert!(tx.send(outcome.clone()).is_ok()); + } + outcome + } + + async fn write_members( + &self, + _log: &Logger, + group_ip: UnderlayMulticastIpv6, + tag: &MulticastTag, + members: Vec, + ) -> WriteOutcome { + self.writes + .lock() + .unwrap() + .push((group_ip, members.clone())); + + if matches!(self.write_outcome, WriteOutcome::Updated) { + let updated = FetchOutcome::Found(tag.clone(), members); + let overridden = { + let mut by_group = self.fetch_by_group.lock().unwrap(); + match by_group.get_mut(&group_ip) { + Some(outcome) => { + *outcome = updated.clone(); + true + } + None => false, + } + }; + if !overridden { + *self.fetch.lock().unwrap() = updated; + } + } + self.write_attempted.notify_one(); + self.write_outcome + } + } + + fn found(members: Vec) -> FetchOutcome { + FetchOutcome::Found(MulticastTag::try_from("tag").unwrap(), members) + } + + fn reconcile( + group: UnderlayMulticastIpv6, + imported: &HashSet, + peer_links: &HashMap, + programmed: Members, + mock: &MockDpd, + ) -> Members { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let learned = learned(imported, peer_links); + + rt.block_on(reconcile_group( + group, + group_ifnames(group, &learned), + &resolve_links( + learned.iter().map(|(_, ifname)| ifname.as_str()), + &mut HashSet::new(), + &log, + ), + programmed, + mock, + &log, + )) + } + + fn run_sweep( + mut programmed: ProgrammedMembers, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> ProgrammedMembers { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + rt.block_on(reconcile_pass( + &mut programmed, + &mut HashSet::new(), + &learned(imported, peer_links), + mock, + &log, + )); + programmed + } + + fn set(members: &[MulticastGroupMember]) -> Members { + members.iter().cloned().collect() + } + + #[tokio::test(start_paused = true)] + async fn mid_sweep_change_wakes_next_sweep() { + let dir = camino_tempfile::Utf8TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().as_str(), log.clone()).unwrap(); + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + db.splice_imported_mcast(crate::db::MulticastRibChange::new( + "tfportrear0_0", + &imported, + &HashSet::new(), + )); + + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + let (started_tx, started_rx) = oneshot::channel(); + let (resume_tx, resume_rx) = oneshot::channel(); + *mock.fetch_channels.lock().unwrap() = Some((started_tx, resume_rx)); + + let sweep = + run_with_client(db.clone(), &mock, db.subscribe_mcast_sweep(), log); + let check = async { + tokio::time::timeout(RECONCILE_INTERVAL / 2, started_rx) + .await + .expect("first sweep did not reach the DPD fetch") + .unwrap(); + db.remove_nexthop_mcast_routes(peer, "tfportrear0_0"); + resume_tx.send(()).unwrap(); + + tokio::time::timeout( + RECONCILE_INTERVAL / 2, + mock.write_attempted.notified(), + ) + .await + .expect("withdrawal during sweep did not wake the next sweep"); + assert_eq!(mock.writes(), vec![Vec::::new()]); + }; + tokio::select! { + _ = sweep => panic!("sweep exited"), + _ = check => {} + } + } + + #[test] + fn group_drains_and_returns_across_sweeps() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let active = HashSet::from([route(peer, group)]); + let withdrawn = HashSet::new(); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // Sweep 1: imported and already in sync, the sweep counts the + // existing member as having been programmed (without an explicit + // write). + let programmed = run_sweep(HashMap::new(), &active, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert!(mock.writes().is_empty()); + + // Sweep 2: the withdrawn group carries over from sweep 1, its + // programmed member is removed, and then it drops out of the sweep. + let programmed = run_sweep(programmed, &withdrawn, &peer_links, &mock); + assert!(programmed.is_empty()); + assert_eq!(mock.writes(), vec![Vec::::new()]); + + // Sweep 3: the dropped group is re-imported and reconfigured (the group + // contains no membership in DPD). + let programmed = run_sweep(programmed, &active, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert_eq!(mock.writes(), vec![Vec::new(), vec![member("rear0", 0)]]); + } + + /// Case: after a restart, the sweep acts as a no-op for members it has not + /// yet programmed. + /// + /// - A leftover member by the previous `ddmd` process is not removed, akin + /// to stale unicast routes. + /// - A leftover member that an import re-derives is counted as programmed + /// and is removed once that import is finally withdrawn. + #[test] + fn a_restart_keeps_leftover_members() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear1", 0))]); + let mock = MockDpd::new( + found(vec![member("rear0", 0), member("rear1", 0)]), + WriteOutcome::Updated, + ); + + let programmed = + run_sweep(HashMap::new(), &imported, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear1", 0)]))]) + ); + assert!(mock.writes().is_empty()); + + let programmed = + run_sweep(programmed, &HashSet::new(), &peer_links, &mock); + assert!(programmed.is_empty()); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn existing_members_are_kept_on_addition() { + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![member("rear1", 0), external("qsfp0", 0)]), + WriteOutcome::Updated, + ); + + let programmed = + reconcile(group, &imported, &peer_links, Members::new(), &mock); + assert_eq!(programmed, set(&[member("rear0", 0)])); + let writes = mock.writes(); + assert_eq!(writes.len(), 1); + assert_eq!( + set(&writes[0]), + set(&[ + member("rear0", 0), + member("rear1", 0), + external("qsfp0", 0) + ]) + ); + + let programmed = + reconcile(group, &imported, &peer_links, programmed.clone(), &mock); + assert_eq!(programmed, set(&[member("rear0", 0)])); + assert_eq!(mock.writes().len(), 1); + } + + #[tokio::test(start_paused = true)] + async fn missing_group_is_retried_on_periodic_sweep() { + let dir = camino_tempfile::Utf8TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().as_str(), log.clone()).unwrap(); + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + db.splice_imported_mcast(crate::db::MulticastRibChange::new( + "tfportrear0_0", + &imported, + &HashSet::new(), + )); + let imports = db.imported_mcast_links(); + let wakes = db.subscribe_mcast_sweep(); + + let mock = MockDpd::new(FetchOutcome::Missing, WriteOutcome::Updated); + let (fetched_tx, fetched_rx) = oneshot::channel(); + *mock.fetch_result_tx.lock().unwrap() = Some(fetched_tx); + let sweep = + run_with_client(db.clone(), &mock, db.subscribe_mcast_sweep(), log); + let check = async { + let result = + tokio::time::timeout(RECONCILE_INTERVAL / 2, fetched_rx) + .await + .expect("first sweep did not fetch the missing group") + .unwrap(); + assert!(matches!(result, FetchOutcome::Missing)); + assert!(mock.writes().is_empty()); + + let missing_at = tokio::time::Instant::now(); + *mock.fetch.lock().unwrap() = found(Vec::new()); + tokio::time::timeout( + RECONCILE_INTERVAL * 2, + mock.write_attempted.notified(), + ) + .await + .expect("periodic sweep did not program the newly created group"); + + assert!(missing_at.elapsed() >= RECONCILE_INTERVAL); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + assert_eq!(db.imported_mcast_links(), imports); + assert!(!wakes.has_changed().unwrap()); + }; + tokio::select! { + _ = sweep => panic!("sweep exited"), + _ = check => {} + } + } + + #[test] + fn deleted_group_drops() { + let group = underlay(1); + let mock = MockDpd::new(FetchOutcome::Missing, WriteOutcome::Updated); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert!(mock.writes().is_empty()); + } + + #[test] + fn read_retry_keeps_withdrawn_group() { + let group = underlay(1); + let mock = MockDpd::new(FetchOutcome::Retry, WriteOutcome::Updated); + + // A withdrawn group must not drop out on a transient read failure, or + // its programmed members would stay with nothing to remove them. + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + set(&[member("rear0", 0)]) + ); + assert!(mock.writes().is_empty()); + } + + #[tokio::test] + async fn transport_failure_retries_withdrawal() { + read_failure_retries_withdrawal(None).await; + } + + #[tokio::test] + async fn http_503_retries_withdrawal() { + read_failure_retries_withdrawal(Some(StatusCode::SERVICE_UNAVAILABLE)) + .await; + } + + async fn read_failure_retries_withdrawal( + failure_status: Option, + ) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let client = Client::new( + &format!("http://{}", listener.local_addr().unwrap()), + ClientState { + tag: DDM_TAG.into(), + log: log.clone(), + }, + ); + let group = underlay(1); + let rear0 = member("rear0", 0); + let preserved = external("qsfp0", 0); + let members = + std::sync::Mutex::new(vec![rear0.clone(), preserved.clone()]); + let unavailable = AtomicBool::new(false); + let failures = AtomicUsize::new(0); + let writes = AtomicUsize::new(0); + + let server = async { + loop { + let (socket, _) = listener.accept().await.unwrap(); + if failure_status.is_none() + && unavailable.load(Ordering::Relaxed) + { + failures.fetch_add(1, Ordering::Relaxed); + drop(socket); + continue; + } + + let service = service_fn(|request: Request| { + let members = &members; + let writes = &writes; + let unavailable = &unavailable; + let failures = &failures; + async move { + assert_eq!( + request.uri().path(), + format!("/multicast/underlay-groups/{group}") + ); + if let Some(status) = failure_status + .filter(|_| unavailable.load(Ordering::Relaxed)) + { + assert_eq!(request.method(), Method::GET); + failures.fetch_add(1, Ordering::Relaxed); + let body = serde_json::json!({ + "message": "multicast group repair failed", + "request_id": "test", + }); + return Ok::<_, Infallible>( + Response::builder() + .status(status) + .header("content-type", "application/json") + .body(Full::new(Bytes::from( + body.to_string(), + ))) + .unwrap(), + ); + } + if request.method() == Method::PUT { + assert_eq!(request.uri().query(), Some("tag=tag")); + let body = request + .into_body() + .collect() + .await + .unwrap() + .to_bytes(); + let update: MulticastGroupUpdateUnderlayEntry = + serde_json::from_slice(&body).unwrap(); + *members.lock().unwrap() = update.members; + writes.fetch_add(1, Ordering::Relaxed); + } else { + assert_eq!(request.method(), Method::GET); + } + let body = serde_json::to_vec( + &MulticastGroupUnderlayResponse { + group_ip: group, + tag: "tag".into(), + external_group_id: 100, + underlay_group_id: 101, + members: members.lock().unwrap().clone(), + }, + ) + .unwrap(); + Ok::<_, Infallible>( + Response::builder() + .header("content-type", "application/json") + .body(Full::new(Bytes::from(body))) + .unwrap(), + ) + } + }); + http1::Builder::new() + .keep_alive(false) + .serve_connection(TokioIo::new(socket), service) + .await + .unwrap(); + } + }; + let check = async { + let peer = "fe80::1".parse().unwrap(); + let imported = + vec![(route(peer, group), "tfportrear0_0".to_string())]; + let mut programmed = HashMap::new(); + let mut unresolved = HashSet::new(); + + reconcile_pass( + &mut programmed, + &mut unresolved, + &imported, + &client, + &log, + ) + .await; + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert_eq!(writes.load(Ordering::Relaxed), 0); + + unavailable.store(true, Ordering::Relaxed); + reconcile_pass( + &mut programmed, + &mut unresolved, + &[], + &client, + &log, + ) + .await; + assert!(failures.load(Ordering::Relaxed) > 0); + assert_eq!( + programmed, + HashMap::from([(group, set(&[member("rear0", 0)]))]) + ); + assert_eq!( + set(&members.lock().unwrap()), + set(&[rear0, preserved.clone()]) + ); + assert_eq!(writes.load(Ordering::Relaxed), 0); + + unavailable.store(false, Ordering::Relaxed); + reconcile_pass( + &mut programmed, + &mut unresolved, + &[], + &client, + &log, + ) + .await; + assert!(programmed.is_empty()); + assert_eq!(set(&members.lock().unwrap()), set(&[preserved])); + assert_eq!(writes.load(Ordering::Relaxed), 1); + }; + tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + _ = server => panic!("test DPD server exited"), + _ = check => {} + } + }) + .await + .expect("withdrawal did not recover after the read failure"); + } + + #[test] + fn withdrawn_group_with_no_members_drops_without_writing() { + let group = underlay(1); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert!(mock.writes().is_empty()); + } + + #[test] + fn empty_write_retry_keeps_withdrawn_group() { + let group = underlay(1); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Retry); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + set(&[member("rear0", 0)]) + ); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn write_retry_retains_member_for_later_withdrawal() { + let peer = "fe80::1".parse().unwrap(); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mut mock = MockDpd::new(found(Vec::new()), WriteOutcome::Retry); + + let programmed = + reconcile(group, &imported, &peer_links, Members::new(), &mock); + + *mock.fetch.lock().unwrap() = found(vec![member("rear0", 0)]); + mock.write_outcome = WriteOutcome::Updated; + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &peer_links, + programmed.clone(), + &mock + ), + Members::new() + ); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)], Vec::new()]); + } + + #[test] + fn active_group_with_drifted_members_writes_wanted_set() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert_eq!( + reconcile( + group, + &imported, + &peer_links, + set(&[member("rear0", 0)]), + &mock + ), + set(&[member("rear0", 0)]) + ); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + /// A programmed member whose interface no longer resolves to a switch link + /// is removed, which would occur on peer expiration. + #[test] + fn unresolved_interface_removes_programmed_member() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert_eq!( + reconcile( + group, + &imported, + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn only_programmed_members_are_removed() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![ + member("rear0", 0), + member("rear1", 0), + member("rear2", 0), + external("qsfp0", 0), + ]), + WriteOutcome::Updated, + ); + + assert_eq!( + reconcile( + group, + &imported, + &peer_links, + set(&[member("rear0", 0), member("rear1", 0)]), + &mock, + ), + set(&[member("rear0", 0)]) + ); + let writes = mock.writes(); + assert_eq!(writes.len(), 1); + assert_eq!( + set(&writes[0]), + set(&[ + member("rear0", 0), + member("rear2", 0), + external("qsfp0", 0) + ]) + ); + } + + fn external(port: &str, link: u8) -> MulticastGroupMember { + MulticastGroupMember { + port_id: PortId::Qsfp( + dpd_client::types::Qsfp::try_from(port).unwrap(), + ), + link_id: LinkId(link), + direction: Direction::External, + } + } + + #[test] + fn drain_keeps_external_members() { + let group = underlay(1); + let mock = MockDpd::new( + found(vec![member("rear0", 0), external("qsfp0", 0)]), + WriteOutcome::Updated, + ); + + assert_eq!( + reconcile( + group, + &HashSet::new(), + &HashMap::new(), + set(&[member("rear0", 0)]), + &mock, + ), + Members::new() + ); + assert_eq!(mock.writes(), vec![vec![external("qsfp0", 0)]]); + } + + #[test] + fn group_outcomes_are_independent() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let readable_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let retried = underlay(1); + let missing = underlay(2); + let readable = underlay(3); + let imported = HashSet::from([ + route(peer, retried), + route(peer, missing), + route(readable_peer, readable), + ]); + let peer_links = HashMap::from([ + (peer, rear("rear0", 0)), + (readable_peer, rear("rear2", 0)), + ]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + mock.fetch_by_group.lock().unwrap().extend([ + (retried, FetchOutcome::Retry), + (missing, FetchOutcome::Missing), + (readable, found(vec![member("rear1", 0)])), + ]); + + let prev = HashMap::from([ + (retried, set(&[member("rear1", 0)])), + (missing, set(&[member("rear1", 0)])), + ]); + let programmed = run_sweep(prev, &imported, &peer_links, &mock); + assert_eq!( + programmed, + HashMap::from([ + (retried, set(&[member("rear1", 0)])), + (readable, set(&[member("rear2", 0)])), + ]) + ); + let writes = mock.group_writes(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].0, readable); + assert_eq!( + set(&writes[0].1), + set(&[member("rear1", 0), member("rear2", 0)]) + ); + } +} diff --git a/ddm/src/sm/state.rs b/ddm/src/sm/state.rs index 9b6f99e6f..493920d1d 100644 --- a/ddm/src/sm/state.rs +++ b/ddm/src/sm/state.rs @@ -811,7 +811,9 @@ impl Exchange { self.retired.clear(); *session = None; self.ctx.iface.clear_peer(); - self.ctx.db.remove_nexthop_routes(self.peer) + self.ctx + .db + .remove_nexthop_routes(self.peer, &self.ctx.config.if_name) }; let routes = removed .underlay @@ -1104,7 +1106,10 @@ impl Exchange { self.session = Arc::new(ExchangeSession::new(version)); *session = Some(self.session.clone()); if prior_version >= Version::V4 && version < Version::V4 { - Some(self.ctx.db.remove_nexthop_mcast_routes(self.peer)) + Some(self.ctx.db.remove_nexthop_mcast_routes( + self.peer, + &self.ctx.config.if_name, + )) } else { None } diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index 14a4ee266..14e40d8b5 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::sm::{Config, DpdConfig}; -use crate::{dbg, err, inf, wrn}; +use crate::{DDM_TAG, dbg, err, inf, wrn}; use ddm_api_types::db::TunnelRoute; use dpd_client::Client; use dpd_client::ClientState; @@ -24,8 +24,6 @@ use ::{ std::collections::HashMap, }; -const DDM_DPD_TAG: &str = "ddmd"; - #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct Route { pub dest: IpAddr, @@ -145,7 +143,7 @@ pub fn add_routes_dendrite( dbg!(log, ifname, "sending to dpd host={} port={}", host, port); let client_state = ClientState { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), log: log.clone(), }; let client = Client::new(&format!("http://{host}:{port}"), client_state); @@ -183,43 +181,14 @@ pub fn add_routes_dendrite( } }; - // TODO this is gross, use link type properties rather than futzing - // around with strings. - - let port_id = { - if let Some(egress_port_num) = ifname - .strip_prefix("tfportrear") - .and_then(|x| x.strip_suffix("_0")) - .map(|x| x.trim()) - .and_then(|x| x.parse::().ok()) - { - let port_name = format!("rear{}", egress_port_num); - match types::Rear::try_from(port_name.as_str()) { - Ok(rear) => PortId::Rear(rear), - Err(e) => { - err!(log, ifname, "bad port name ({port_name}): {e}"); - continue; - } - } - } else if let Some(egress_port_num) = ifname - .strip_prefix("tfportqsfp") - .and_then(|x| x.strip_suffix("_0")) - .map(|x| x.trim()) - .and_then(|x| x.parse::().ok()) - { - let port_name = format!("qsfp{}", egress_port_num); - match types::Qsfp::try_from(port_name.as_str()) { - Ok(qsfp) => PortId::Qsfp(qsfp), - Err(e) => { - err!(log, ifname, "bad port name ({port_name}): {e}"); - continue; - } + let (port_id, link_id) = + match mg_common::tfport::port_link_from_untagged_ifname(ifname) { + Ok(port_link) => port_link, + Err(e) => { + err!(log, ifname, "{e}"); + continue; } - } else { - err!(log, ifname, "expected tfportrear or tfportqsfp"); - continue; - } - }; + }; inf!( log, @@ -228,14 +197,11 @@ pub fn add_routes_dendrite( r.dest, r.gw, port_id, - 0, + link_id.0, ); - // TODO breakout considerations - let link_id = types::LinkId(0); - let target = types::Ipv6Route { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), port_id, link_id, tgt_ip: gw, @@ -428,7 +394,7 @@ pub fn remove_routes_dendrite( log: &Logger, ) { let client_state = ClientState { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), log: log.clone(), }; let client = Client::new(&format!("http://{host}:{port}"), client_state); @@ -477,7 +443,7 @@ pub fn get_routes_dendrite( log: Logger, ) -> Result, String> { let client_state = ClientState { - tag: DDM_DPD_TAG.into(), + tag: DDM_TAG.into(), log: log.clone(), }; let client = Client::new(&format!("http://{host}:{port}"), client_state); diff --git a/ddmadm/src/main.rs b/ddmadm/src/main.rs index c13046271..79e075ddc 100644 --- a/ddmadm/src/main.rs +++ b/ddmadm/src/main.rs @@ -332,22 +332,28 @@ async fn run() -> Result<()> { SubCommand::MulticastImported => { let msg = client.get_multicast_groups().await?; let mut routes: Vec<_> = msg.into_inner().into_iter().collect(); - routes.sort_by_key(|route| { - (origin_sort_key(&route.origin), route.nexthop) + routes.sort_by_key(|imported| { + ( + origin_sort_key(&imported.route.origin), + imported.route.nexthop, + imported.if_name.clone(), + ) }); let mut tw = TabWriter::new(stdout()); writeln!( &mut tw, - "{}\t{}\t{}\t{}\t{}\t{}\t{}", + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", "Overlay Group".dimmed(), "Underlay Group".dimmed(), "VNI".dimmed(), "Metric".dimmed(), "Source".dimmed(), "Nexthop".dimmed(), + "Interface".dimmed(), "Path".dimmed(), )?; - for route in &routes { + for imported in &routes { + let route = &imported.route; let path: Vec<_> = route .path .iter() @@ -356,13 +362,14 @@ async fn run() -> Result<()> { .collect(); writeln!( &mut tw, - "{}\t{}\t{}\t{}\t{}\t{}\t{}", + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", route.origin.overlay_group(), route.origin.underlay_group(), route.origin.vni(), route.origin.metric(), source_column(route.origin.source()), route.nexthop, + imported.if_name, path.join(" "), )?; } diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index 4c9710943..cf7402b5a 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -184,7 +184,7 @@ async fn run() { let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); let context = Arc::new(Mutex::new(HandlerContext { - db, + db: db.clone(), stats: router_stats, peers, stats_handler: Arc::new(Mutex::new(None)), @@ -208,6 +208,8 @@ async fn run() { router_id: router_id.clone(), })); + start_mcast_sweep(arg.api_only, dpd, db, log.clone()); + if arg.with_stats && let (Some(rack_uuid), Some(sled_uuid)) = (arg.rack_uuid, arg.sled_uuid) @@ -344,6 +346,47 @@ fn start_state_machines( Vec::new() } +/// Spawn the multicast membership sweep. +/// +/// This bypasses the sweep under `--api-only` or without `--dendrite`. In +/// API-only mode, there's no routing, and, therefore, no imports to derive +/// membership from. +/// +/// Builds use `panic = "abort"` (see `.cargo/config.toml`), which means that a +/// panic during a sweep terminates `ddmd` rather than only the task, after +/// which SMF restarts the service. +/// +/// The restarted process bootstraps with no record of programmed members. +/// Existing membership remains in place, and only those an import re-derives +/// come back under management, akin to the unicast and tunnel routes left +/// behind by a `ddmd` crash. +#[cfg(all(feature = "backend", target_os = "illumos"))] +fn start_mcast_sweep( + api_only: bool, + dpd: Option, + db: Db, + log: Logger, +) { + let Some(dpd) = dpd else { + return; + }; + if api_only { + return; + } + tokio::spawn(ddm::mcast::run(db, dpd, log)); +} + +/// Non-illumos variant: underlay multicast replicates only on a switch, meaning +/// that a sweep never starts. +#[cfg(not(all(feature = "backend", target_os = "illumos")))] +fn start_mcast_sweep( + _api_only: bool, + _dpd: Option, + _db: Db, + _log: Logger, +) { +} + /// Install a Ctrl-C handler that withdraws ddmd's imported routes from the /// kernel before exiting. On non-illumos builds there are no kernel routes /// to withdraw, so the handler just exits cleanly. diff --git a/falcon-lab/src/main.rs b/falcon-lab/src/main.rs index a95a5b5fe..7c54b1bfb 100644 --- a/falcon-lab/src/main.rs +++ b/falcon-lab/src/main.rs @@ -22,7 +22,7 @@ mod scenario; mod topo; mod util; -const DEFAULT_NPUVM_COMMIT: &str = "a6a878533f9eae542552ef66223a37443f931149"; +const DEFAULT_NPUVM_COMMIT: &str = "63687602f0d26add71d4dae4ac62d46563ba23f5"; #[derive(Debug, Parser)] struct Cli { diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index 2fc27988b..aaee6c5bb 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -17,6 +17,10 @@ backoff.workspace = true smf.workspace = true uuid.workspace = true libc.workspace = true +dpd-client.workspace = true + +[dev-dependencies] +proptest.workspace = true # We need this on illumos, but must omit it on other platforms [target.'cfg(target_os = "illumos")'.dependencies.libnet] diff --git a/mg-common/src/lib.rs b/mg-common/src/lib.rs index b2af8cd22..abb47e2fb 100644 --- a/mg-common/src/lib.rs +++ b/mg-common/src/lib.rs @@ -8,6 +8,7 @@ pub mod nexus; pub mod smf; pub mod stats; pub mod test; +pub mod tfport; pub mod thread; use std::time::Duration; diff --git a/mg-common/src/tfport.rs b/mg-common/src/tfport.rs new file mode 100644 index 000000000..c826aec57 --- /dev/null +++ b/mg-common/src/tfport.rs @@ -0,0 +1,411 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Parsing of Tofino port (`tfport`) datalink names. +//! +//! Tofino switch ports are surfaced to the host as illumos datalinks named +//! `tfport_`; e.g., `tfportqsfp10_0` (front panel) or +//! `tfportrear0_0` (backplane). This module parses this datalink naming +//! convention into its specific components, allowing callers to easily map a +//! datalink name back to a switch port identifier. +//! +//! Note: VLAN links over an uplink append the VLAN ID after a `.`, e.g., +//! `tfportqsfp0_0.100`, as surfaced by Dendrite's `uplinkd` ([VLAN naming]). +//! +//! [VLAN naming]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/uplinkd/src/main.rs#L251-L256 + +use dpd_client::types; + +/// Prefix shared by every `tfport` datalink name. +const TFPORT_DEVICE_PREFIX: &str = "tfport"; + +/// The switch-port device kind encoded into a `tfport` datalink name. +/// +/// Front-panel links typically appear as `qsfp`. Backplane links to sleds +/// appear as `rear`, which are the only ports ddmd currently programs as +/// underlay multicast members for ingress multicast traffic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TfportKind { + Qsfp, + Rear, +} + +impl TfportKind { + /// The device token as it appears both in a tfport name (`tfportrear0_0`) + /// and in a DPD port name (`rear0`). + fn token(self) -> &'static str { + match self { + TfportKind::Qsfp => "qsfp", + TfportKind::Rear => "rear", + } + } + + /// Parse a device kind from its datalink token (`qsfp`, `rear`). + fn from_token(s: &str) -> Option { + match s { + "qsfp" => Some(TfportKind::Qsfp), + "rear" => Some(TfportKind::Rear), + _ => None, + } + } +} + +/// Components parsed from a `tfport` datalink name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TfportName { + /// Device kind (front-panel vs backplane). + kind: TfportKind, + /// Switch port number within the kind. + port: u8, + /// Link (lane) number within the port. + link: u8, + /// Optional VLAN tag appended after a `.`. + vlan: Option, +} + +/// Parse a `tfport` datalink name into its components. +/// +/// # Errors +/// +/// Returns an error if `name` lacks the specified `tfport` prefix, points to +/// an unrecognized device kind, or has malformed port/link/vlan fields. +fn parse_tfport_name(name: &str) -> Result { + let body = name.strip_prefix(TFPORT_DEVICE_PREFIX).ok_or_else(|| { + format!("{name} missing expected prefix {TFPORT_DEVICE_PREFIX}") + })?; + + // The device kind is the leading alphabetic run (`qsfp`, `rear`), the + // remainder carries the port/link/vlan numbers. + let split = body + .find(|c: char| !c.is_ascii_alphabetic()) + .ok_or_else(|| format!("{name} has no port id"))?; + let (kind_str, rest) = body.split_at(split); + let kind = TfportKind::from_token(kind_str).ok_or_else(|| { + format!("{name} has unsupported device kind {kind_str}") + })?; + + let (port_link, vlan) = rest + .split_once('.') + .map_or((rest, None), |(port_link, vlan)| (port_link, Some(vlan))); + + let (port, link) = port_link + .split_once('_') + .ok_or_else(|| format!("{name} has no link id"))?; + + let port = parse_number::(port) + .ok_or_else(|| format!("{name} has invalid port {port}"))?; + + let link = parse_number::(link) + .ok_or_else(|| format!("{name} has invalid link id {link}"))?; + + // A second `.` (e.g. `tfportqsfp10_0.100.200`) leaves a non-numeric + // remainder, rejected by the parse op below. + let vlan = vlan + .map(|vlan| { + parse_number::(vlan) + .ok_or_else(|| format!("{name} has invalid vlan {vlan}")) + }) + .transpose()?; + + Ok(TfportName { + kind, + port, + link, + vlan, + }) +} + +/// Parse a port, link, or VLAN number made only of ASCII digits. +/// +/// `str::parse` accepts a leading `+`, which no tfport name conveys, +/// so we have to rule out a sign or any other non-digit before parsing. +fn parse_number(s: &str) -> Option { + if !s.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + s.parse().ok() +} + +/// Build a DPD [`PortId`] from a parsed tfport kind and port number. +/// +/// # Errors +/// +/// Returns an error if the synthesized port name is not a valid DPD `qsfp` or +/// `rear` port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +fn tfport_port_id( + kind: TfportKind, + port: u8, +) -> Result { + let port_name = format!("{}{port}", kind.token()); + match kind { + TfportKind::Qsfp => types::Qsfp::try_from(port_name.as_str()) + .map(types::PortId::Qsfp) + .map_err(|e| format!("bad qsfp port name {port_name}: {e}")), + TfportKind::Rear => types::Rear::try_from(port_name.as_str()) + .map(types::PortId::Rear) + .map_err(|e| format!("bad rear port name {port_name}: {e}")), + } +} + +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the DPD +/// ([`PortId`], [`LinkId`]) pair that names the switch port and link. +/// +/// # Errors +/// +/// Returns an error if `ifname` is not a valid tfport datalink name or its +/// kind and port do not form a valid DPD port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +/// [`LinkId`]: dpd_client::types::LinkId +pub fn port_link_from_ifname( + ifname: &str, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + port_link(&parse_tfport_name(ifname)?) +} + +fn port_link( + tfport: &TfportName, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + let port_id = tfport_port_id(tfport.kind, tfport.port)?; + // Breakout lanes surface up as distinct datalinks (`tfportqsfp0_0`, + // `tfportqsfp0_1`, ...), sharing one `PortId`. The parsed link is the + // lane, which is the DPD `LinkId`. + let link_id = dpd_client::types::LinkId(tfport.link); + Ok((port_id, link_id)) +} + +/// Resolve an untagged tfport datalink name to its DPD `(PortId, LinkId)`. +/// +/// Unlike [`port_link_from_ifname`], which ignores a VLAN suffix, this +/// method rejects it. A `(PortId, LinkId)` names the untagged port and link. +/// Resolving a VLAN interface to it would program state with the incorrect +/// tagging. +/// +/// DDM peers operate over untagged links, using this naming convention, while +/// mg-lower keeps the more open convention for next hops on [VLAN uplinks]. +/// +/// # Errors +/// +/// Returns an error if `ifname` carries a VLAN suffix, or for any reason +/// [`port_link_from_ifname`] would. +/// +/// [VLAN uplinks]: https://github.com/oxidecomputer/dendrite/blob/3019e6f0c73ded56be5c36cbaca82dbbc796a346/uplinkd/src/main.rs#L251-L256 +pub fn port_link_from_untagged_ifname( + ifname: &str, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + let tfport = parse_tfport_name(ifname)?; + if let Some(vlan) = tfport.vlan { + return Err(format!( + "{ifname} is a vlan {vlan} interface, expected an untagged tfport" + )); + } + port_link(&tfport) +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + fn name( + kind: TfportKind, + port: u8, + link: u8, + vlan: Option, + ) -> TfportName { + TfportName { + kind, + port, + link, + vlan, + } + } + + #[test] + fn test_tfport_parser() { + assert_eq!( + parse_tfport_name("tfportqsfp10_0").unwrap(), + name(TfportKind::Qsfp, 10, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportqsfp10_0.100").unwrap(), + name(TfportKind::Qsfp, 10, 0, Some(100)) + ); + assert_eq!( + parse_tfport_name("tfportqsfp010_00.00100").unwrap(), + name(TfportKind::Qsfp, 10, 0, Some(100)) + ); + assert_eq!( + parse_tfport_name("tfportqsfp1_1").unwrap(), + name(TfportKind::Qsfp, 1, 1, None) + ); + + assert_eq!( + parse_tfport_name("tfportrear0_0").unwrap(), + name(TfportKind::Rear, 0, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportrear31_0.200").unwrap(), + name(TfportKind::Rear, 31, 0, Some(200)) + ); + + // Malformed names. + assert!(parse_tfport_name("fportqsfp10_0").is_err()); + assert!(parse_tfport_name("10_0").is_err()); + assert!(parse_tfport_name("tfportqsfp10").is_err()); + assert!(parse_tfport_name("tfportqsfp_10").is_err()); + assert!(parse_tfport_name("tfportqsfp0_").is_err()); + assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); + assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.100.200").is_err()); + assert!(parse_tfport_name("tfportrear0_0/ll").is_err()); + assert!(parse_tfport_name("tfportrear0_0/v6").is_err()); + + // Unsupported or missing device kind. + assert!(parse_tfport_name("tfportfoo0_0").is_err()); + assert!(parse_tfport_name("tfport0_0").is_err()); + + // Invalid numeric components. + assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); + assert!(parse_tfport_name("tfportqsfp+1_0").is_err()); + assert!(parse_tfport_name("tfportqsfp1_+0").is_err()); + assert!(parse_tfport_name("tfportqsfp1_0.+100").is_err()); + assert!(parse_tfport_name("tfportqsfp1_0.65536").is_err()); + } + + #[test] + fn untagged_resolution_rejects_vlan() { + assert!(port_link_from_untagged_ifname("tfportrear0_0").is_ok()); + assert!(port_link_from_untagged_ifname("tfportrear0_0.100").is_err()); + assert!(port_link_from_ifname("tfportqsfp0_0.100").is_ok()); + } + + #[test] + fn port_and_link_boundaries() { + assert!(port_link_from_untagged_ifname("tfportqsfp31_0").is_ok()); + assert!(port_link_from_untagged_ifname("tfportqsfp32_0").is_err()); + assert!(port_link_from_untagged_ifname("tfportrear31_0").is_ok()); + assert!(port_link_from_untagged_ifname("tfportrear32_0").is_err()); + assert!(parse_tfport_name("tfportqsfp255_0").is_ok()); + assert!(parse_tfport_name("tfportqsfp256_0").is_err()); + assert!(parse_tfport_name("tfportqsfp0_255").is_ok()); + assert!(parse_tfport_name("tfportqsfp0_256").is_err()); + assert!(parse_tfport_name("tfportint0_0").is_err()); + } + + fn near_miss_kind() -> impl Strategy { + prop_oneof![ + Just("qsfp".to_string()), + Just("rear".to_string()), + "[a-z]{0,4}", + ] + } + + fn near_miss_number() -> impl Strategy { + prop_oneof![ + "[0-9]{1,4}", + "\\+[0-9]{1,3}", + Just(String::new()), + "[0-9]{1,2}[a-z/]", + ] + } + + fn near_miss_suffix() -> impl Strategy { + prop_oneof![ + Just(String::new()), + "\\.[0-9]{1,6}", + Just(".".to_string()), + "\\.[0-9]{1,3}\\.[0-9]{0,3}", + "\\.[+-][0-9]{1,3}", + "\\.[\u{0660}-\u{0669}\u{FF10}-\u{FF19}]{1,3}", + "/[a-z0-9]{0,3}", + ] + } + + proptest! { + /// Property: a well-formed name round-trips through the parser. + /// + /// The parser is purely syntactic, exercising the full u8/u16 ranges. + #[test] + fn prop_roundtrip( + is_rear in any::(), + port in any::(), + link in any::(), + vlan in proptest::option::of(any::()), + ) { + let kind = + if is_rear { TfportKind::Rear } else { TfportKind::Qsfp }; + let mut ifname = format!("tfport{}{port}_{link}", kind.token()); + if let Some(vlan) = vlan { + ifname.push_str(&format!(".{vlan}")); + } + prop_assert_eq!( + parse_tfport_name(&ifname).unwrap(), + TfportName { kind, port, link, vlan }, + ); + } + + /// Property: a DPD `PortId` formatted into a tfport name resolves back + /// to the same `(PortId, LinkId)`. + #[test] + fn prop_resolves_dpd_port_names( + is_rear in any::(), + port in 0u8..32, + link in any::(), + ) { + let port_id = if is_rear { + types::PortId::Rear( + types::Rear::try_from(format!("rear{port}").as_str()) + .unwrap(), + ) + } else { + types::PortId::Qsfp( + types::Qsfp::try_from(format!("qsfp{port}").as_str()) + .unwrap(), + ) + }; + let ifname = format!("tfport{port_id}_{link}"); + prop_assert_eq!( + port_link_from_untagged_ifname(&ifname).unwrap(), + (port_id, types::LinkId(link)), + ); + } + + /// Property: parsing arbitrary input never panics. + #[test] + fn prop_never_panics(ifname in ".*") { + let _ = parse_tfport_name(&ifname); + } + + /// Property: canonical formatting preserves the parsed components. + #[test] + fn prop_accepted_names_roundtrip( + kind in near_miss_kind(), + port in near_miss_number(), + sep in prop_oneof![Just(""), Just("_"), Just("__")], + link in near_miss_number(), + suffix in near_miss_suffix(), + ) { + let ifname = format!("tfport{kind}{port}{sep}{link}{suffix}"); + if let Ok(name) = parse_tfport_name(&ifname) { + let mut canonical = format!( + "tfport{}{}_{}", + name.kind.token(), + name.port, + name.link, + ); + if let Some(vlan) = name.vlan { + canonical.push_str(&format!(".{vlan}")); + } + prop_assert_eq!(parse_tfport_name(&canonical), Ok(name)); + } + } + } +} diff --git a/mg-lower/src/ddm.rs b/mg-lower/src/ddm.rs index 8e1744851..d265407f2 100644 --- a/mg-lower/src/ddm.rs +++ b/mg-lower/src/ddm.rs @@ -5,11 +5,12 @@ use crate::log::ddm_log; #[cfg(target_os = "illumos")] use ddm_admin_client::Client; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::Logger; use std::{net::Ipv6Addr, sync::Arc}; +use crate::error::Error; use crate::platform::Ddm; pub(crate) const BOUNDARY_SERVICES_VNI: u32 = 99; @@ -107,6 +108,64 @@ pub(crate) fn remove_tunnel_routes<'a, I: Iterator>( } } +pub(crate) fn add_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) -> Result<(), Error> { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return Ok(()); + } + let resp = rt.block_on(client.advertise_multicast_groups(&routes)); + if let Err(e) = resp { + ddm_log!(log, + error, + "advertise multicast groups error: {e}"; + "error" => format!("{e}"), + "groups" => format!("{routes:#?}") + ); + return Err(e.into()); + } + Ok(()) +} + +pub(crate) fn remove_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) -> Result<(), Error> { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return Ok(()); + } + let resp = rt.block_on(client.withdraw_multicast_groups(&routes)); + match resp { + Err(e) => { + ddm_log!(log, + error, + "withdraw multicast groups error: {e}"; + "groups" => format!("{routes:#?}") + ); + return Err(e.into()); + } + Ok(_) => ddm_log!(log, + debug, + "withdrew multicast groups"; + "groups" => format!("{routes:#?}") + ), + } + Ok(()) +} + #[cfg(target_os = "illumos")] pub fn new_ddm_client(log: &Logger) -> Client { Client::new("http://localhost:8000", log.clone()) diff --git a/mg-lower/src/dendrite.rs b/mg-lower/src/dendrite.rs index 8661ba108..c7ee45f4f 100644 --- a/mg-lower/src/dendrite.rs +++ b/mg-lower/src/dendrite.rs @@ -21,7 +21,6 @@ use std::{ time::Duration, }; -const TFPORT_QSFP_DEVICE_PREFIX: &str = "tfportqsfp"; const UNIT_DPD: &str = "dpd"; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -329,68 +328,17 @@ where Ok(()) } -// Translate a tfport name into the underlying (port, link, vlan) tuple. -// tfportqsfp10_0 would translate to (10, 0, None) -// tfportqsfp10_0.100 would translate to (10, 0, Some(100)) -// TODO this is gross, use link type properties rather than futzing -// around with strings. -fn parse_tfport_name(name: &str) -> Result<(u8, u8, Option), Error> { - let body = - name.strip_prefix(TFPORT_QSFP_DEVICE_PREFIX) - .ok_or(Error::Tfport(format!( - "{} missing expected prefix {}", - name, TFPORT_QSFP_DEVICE_PREFIX - )))?; - let fields: Vec<&str> = body.split('.').collect(); - let (port, link) = fields[0] - .split_once('_') - .ok_or(Error::Tfport(format!("{} has no link id", name)))?; - - let port = port.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid port {}", name, port)) - })?; - - let link = link.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid link id {}", name, link)) - })?; - - let vlan = match fields.len() { - 1 => Ok(None), - 2 => fields[1].parse::().map(Some).map_err(|_| { - Error::Tfport(format!("{} has invalid vlan {}", name, fields[1])) - }), - _ => Err(Error::Tfport(format!( - "{} has multiple vlan deliminators", - name +fn qsfp_port_and_link_from_ifname( + ifname: &str, +) -> Result<(types::PortId, types::LinkId), Error> { + match mg_common::tfport::port_link_from_ifname(ifname) + .map_err(Error::Tfport)? + { + port_link @ (types::PortId::Qsfp(_), _) => Ok(port_link), + (port_id, _) => Err(Error::Tfport(format!( + "{ifname} resolves to {port_id}, expected a qsfp port" ))), - }?; - - Ok((port, link, vlan)) -} - -#[test] -fn test_tfport_parser() { - // Test valid names - assert_eq!(parse_tfport_name("tfportqsfp10_0").unwrap(), (10, 0, None)); - assert_eq!( - parse_tfport_name("tfportqsfp10_0.100").unwrap(), - (10, 0, Some(100)) - ); - assert_eq!(parse_tfport_name("tfportqsfp1_1").unwrap(), (1, 1, None)); - - // test malformed names - assert!(parse_tfport_name("fportqsfp10_0").is_err()); - assert!(parse_tfport_name("10_0").is_err()); - assert!(parse_tfport_name("tfportqsfp10").is_err()); - assert!(parse_tfport_name("tfportqsfp_10").is_err()); - assert!(parse_tfport_name("tfportqsfp0_").is_err()); - assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); - assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); - - // test invalid components - assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); + } } fn get_port_and_link( @@ -402,18 +350,7 @@ fn get_port_and_link( && nh6.is_unicast_link_local() && let Some(ref iface) = path.nexthop_interface { - let (port, link, _vlan) = parse_tfport_name(iface)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(port_name.as_str()) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {iface} port name: {port_name}: {e}", - )) - })?; - // TODO breakout considerations - let link_id = types::LinkId(link); - return Ok((port_id, link_id)); + return qsfp_port_and_link_from_ifname(iface); } // Standard nexthop resolution for numbered peers @@ -437,19 +374,7 @@ fn resolve_port_and_link( } }; - let (port, link, _vlan) = parse_tfport_name(&ifname)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(port_name.as_str()) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {ifname} port name: {port_name}: {e}" - )) - })?; - - // TODO breakout considerations - let link_id = types::LinkId(link); - Ok((port_id, link_id)) + qsfp_port_and_link_from_ifname(&ifname) } pub(crate) fn get_routes_for_prefix( diff --git a/mg-lower/src/lib.rs b/mg-lower/src/lib.rs index 33096b5e1..55153f625 100644 --- a/mg-lower/src/lib.rs +++ b/mg-lower/src/lib.rs @@ -25,7 +25,7 @@ use std::collections::HashSet; use std::net::Ipv6Addr; use std::sync::Arc; use std::sync::mpsc::{RecvTimeoutError, channel}; -use std::thread::sleep; +use std::thread::{Builder, JoinHandle, sleep}; use std::time::Duration; // Re-export production backends so callers (e.g. mgd) can construct them. @@ -40,6 +40,7 @@ mod ddm; mod dendrite; mod error; mod log; +pub mod mrib; mod platform; #[cfg(test)] @@ -51,15 +52,48 @@ const COMPONENT_MG_LOWER: &str = MG_LOWER_TAG; const MOD_SYNC: &str = "sync"; const UNIT_EVENT_LOOP: &str = "event_loop"; +/// Platform backends used for route lookups and synchronization. +pub struct Backends { + /// Programs routes and the tunnel endpoint address into the switch ASIC + /// via DPD. + pub dpd: D, + /// Advertises tunnel endpoints to DDM. + pub ddm: M, + /// Looks up kernel routes in the switch zone in order to resolve + /// next hops to switch ports. + pub sw: S, +} + +/// Run the lower-half loop on a dedicated `mg-lower` thread. +/// +/// # Errors +/// +/// Returns an error if the thread cannot be spawned. +pub fn spawn( + tep: Ipv6Addr, + db: Db, + log: Logger, + stats: Arc, + rt: Arc, + backends: Backends< + impl Dpd + Send + 'static, + impl Ddm + Send + 'static, + impl SwitchZone + Send + 'static, + >, +) -> std::io::Result> { + let Backends { dpd, ddm, sw } = backends; + Builder::new() + .name(MG_LOWER_TAG.to_string()) + .spawn(move || run(tep, db, log, stats, rt, &dpd, &ddm, &sw)) +} + /// This is the primary entry point for the lower half. It loops forever, -/// observing changes in the routing databse and synchronizing them to the +/// observing changes in the routing database and synchronizing them to the /// underlying forwarding platform. The loop sets up a watcher to start /// receiving events, does an initial synchronization, then responds to changes -/// moving foward. The loop runs on the calling thread, so callers are -/// responsible for running this function in a separate thread if asynchronous -/// execution is required. +/// moving forward. #[allow(clippy::too_many_arguments)] -pub fn run( +fn run( tep: Ipv6Addr, //tunnel endpoint address db: Db, log: Logger, diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs new file mode 100644 index 000000000..f3b14229d --- /dev/null +++ b/mg-lower/src/mrib.rs @@ -0,0 +1,420 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! MRIB (Multicast Routing Information Base) lower-half synchronization. +//! +//! Advertises locally originated MRIB multicast groups to the DDM admin API, +//! which distributes them across the underlay to other sleds and racks. This is +//! the multicast analog of the unicast lower-half's tunnel-endpoint +//! origination. +//! +//! Origination reads from the local MRIB (`loc_mrib`) by way of its watcher. On +//! each change notification, or once `MRIB_PERIODIC_SYNC_INTERVAL` elapses +//! without a notification, the loop runs the same full reconciliation +//! of `loc_mrib` against the groups originated from DDM. This works similarly +//! to the unicast lower-half's loop (see [`crate::spawn`]). +//! +//! The inbound membership half (resolving DDM-imported routes into switch +//! replication members in DPD) lives in ddmd (`ddm::mcast`), where its input, +//! the imported set with each route's interface, is owned in-process. +//! +//! ## Data Flow +//! +//! ```text +//! Origination (MRIB -> DDM -> underlay) +//! MRIB (loc_mrib changes) +//! | [MribChangeNotification] +//! v [MulticastOrigin] (diff vs DDM originated) +//! DDM admin API --[DDM exchange]--> other sleds/racks +//! ``` +//! +//! See RFD 488 for the multicast architecture. + +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::mpsc::{Receiver, RecvTimeoutError, channel}; +use std::thread::{Builder, JoinHandle, sleep}; +use std::time::Duration; + +use ddm_api_types_versions::latest::net::{MulticastOrigin, OverlayMulticast}; +use rdb::Db; +use rdb::types::{MribChangeNotification, MulticastRoute}; +use slog::{Logger, error, warn}; + +use crate::ddm::{add_multicast_routes, remove_multicast_routes}; +use crate::platform::Ddm; + +pub(crate) const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; + +/// Interval between periodic MRIB full syncs. +/// +/// This interval covers the case where an MRIB change notification is missed, +/// following from the unicast lower-half's 1s resync cadence (see +/// [`crate::spawn`]). +const MRIB_PERIODIC_SYNC_INTERVAL: Duration = Duration::from_secs(1); + +/// The delay before retrying a failed MRIB sync. +/// +/// This is independent of [`MRIB_PERIODIC_SYNC_INTERVAL`], separating +/// resync configuration from changing the retry rate. +const MRIB_SYNC_RETRY_DELAY: Duration = Duration::from_secs(1); + +/// Convert an MRIB `MulticastRoute` to a DDM `MulticastOrigin`. +/// +/// # Errors +/// +/// Returns an error if the route's group or source fails DDM origin +/// validation. +fn ddm_origin(route: &MulticastRoute) -> Result { + let overlay_group = OverlayMulticast::new(route.key.group().ip()) + .map_err(|e| format!("invalid overlay group: {e}"))?; + + MulticastOrigin::new( + overlay_group, + route.underlay_group, + route.key.vni(), + route.key.source(), + ) + .map_err(|e| format!("invalid multicast origin: {e}")) +} + +/// Run the MRIB origination loop on a dedicated `mg-lower-mrib` thread. +/// +/// # Errors +/// +/// Returns an error if the thread cannot be spawned. +pub fn spawn( + db: Db, + log: Logger, + rt: Arc, + ddm: impl Ddm + Send + 'static, +) -> std::io::Result> { + Builder::new() + .name(MG_LOWER_MRIB_TAG.to_string()) + .spawn(move || run(db, log, rt, &ddm)) +} + +/// Run the MRIB origination loop. +/// +/// This function loops ad infinitum, watching for MRIB changes and advertising +/// locally originated multicast groups to DDM. +fn run(db: Db, log: Logger, rt: Arc, ddm: &impl Ddm) { + let mut rx = watch(&db); + loop { + if let Err(e) = sync_pending(&db, ddm, &log, &rt, &rx) { + error!(log, "MRIB sync failed: {e}"); + sleep(MRIB_SYNC_RETRY_DELAY); + continue; + } + + match rx.recv_timeout(MRIB_PERIODIC_SYNC_INTERVAL) { + Ok(_) | Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => { + error!(log, "MRIB watcher disconnected"); + rx = watch(&db); + } + } + } +} + +fn watch(db: &Db) -> Receiver { + let (tx, rx) = channel(); + // Registering under the same tag replaces any watcher left over from a + // previous registration. + db.watch_mrib(MG_LOWER_MRIB_TAG.into(), tx); + rx +} + +/// Drain all pending notifications and reconcile the current MRIB with DDM. +/// +/// `sync` reads the fully selected MRIB, discarding individual notifications. +/// +/// # Errors +/// +/// Propagates errors from `sync`. +fn sync_pending( + db: &Db, + ddm: &impl Ddm, + log: &Logger, + rt: &Arc, + rx: &Receiver, +) -> Result<(), String> { + rx.try_iter().for_each(std::mem::drop); + sync(db, ddm, log, rt) +} + +/// Synchronize MRIB origination to DDM. +/// +/// This compares the current MRIB (`loc_mrib`) with what DDM has advertised, +/// reconciling any differences. +/// +/// # Errors +/// +/// Returns an error if DDM's originated groups cannot be read. +pub(crate) fn sync( + db: &Db, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + let wanted: HashSet = db + .loc_mrib(None) + .values() + .filter_map(|route| match ddm_origin(route) { + Ok(origin) => Some(origin), + Err(e) => { + warn!(log, "skipping MRIB route {}: {e}", route.key); + None + } + }) + .collect(); + + let current: HashSet = rt + .block_on(ddm.get_originated_multicast_groups()) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + add_multicast_routes(ddm, wanted.difference(¤t), rt, log) + .map_err(|e| format!("failed to advertise multicast groups: {e}"))?; + remove_multicast_routes(ddm, current.difference(&wanted), rt, log) + .map_err(|e| format!("failed to withdraw multicast groups: {e}"))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::net::{Ipv4Addr, Ipv6Addr}; + use std::sync::mpsc::TryRecvError; + + use rdb::test::get_test_db; + use rdb::types::{ + MulticastAddr, MulticastRouteKey, MulticastSourceProtocol, + UnderlayMulticastIpv6, Vni, + }; + + use super::*; + use crate::platform::test::TestDdm; + + fn discard_logger() -> Logger { + Logger::root(slog::Discard, slog::o!()) + } + + /// Runtime handle for `sync`'s internal `block_on`. Tests run on a std + /// thread (not a tokio worker). + fn runtime() -> (tokio::runtime::Runtime, Arc) { + let rt = tokio::runtime::Runtime::new().expect("build runtime"); + let handle = Arc::new(rt.handle().clone()); + (rt, handle) + } + + fn vni() -> Vni { + Vni::new(77).expect("valid VNI") + } + + fn test_underlay() -> UnderlayMulticastIpv6 { + UnderlayMulticastIpv6::new(Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1)) + .expect("valid underlay address") + } + + fn origin(route: &MulticastRoute) -> MulticastOrigin { + ddm_origin(route).expect("valid origin") + } + + /// Build an any-source (*,G) static route for the given IPv4 group. + fn asm_route(a: u8, b: u8, c: u8, d: u8) -> MulticastRoute { + let group = MulticastAddr::new(Ipv4Addr::new(a, b, c, d).into()) + .expect("valid group"); + let key = MulticastRouteKey::any_source(group, vni()).expect("key"); + MulticastRoute::new( + key, + test_underlay(), + MulticastSourceProtocol::Static, + ) + } + + /// Build an any-source (*,G) static route for an IPv6 group in + /// `ff0e::/16`, with a distinct underlay (1:1 overlay-to-underlay + /// mapping). + fn asm_route_v6(last: u16) -> MulticastRoute { + let group = MulticastAddr::new( + Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, last).into(), + ) + .expect("valid group"); + let underlay = UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0x1, last, + )) + .expect("valid underlay address"); + let key = MulticastRouteKey::any_source(group, vni()).expect("key"); + MulticastRoute::new(key, underlay, MulticastSourceProtocol::Static) + } + + #[test] + fn sync_pending_reconciles_after_dropping_notifications() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_pending", log.clone()).expect("db"); + let rx = watch(&db); + let removed = asm_route(225, 1, 1, 1); + let added = asm_route_v6(1); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&removed)); + + db.add_static_mcast_routes(std::slice::from_ref(&removed)) + .expect("add initial route"); + db.add_static_mcast_routes(std::slice::from_ref(&added)) + .expect("add replacement route"); + db.remove_static_mcast_routes(&[removed.key]) + .expect("remove initial route"); + rx.try_recv().expect("queued change notification"); + + sync_pending(&db, &ddm, &log, &handle, &rx).expect("sync burst"); + + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + assert_eq!( + *ddm.multicast_originated.lock().unwrap(), + vec![origin(&added)] + ); + + db.remove_static_mcast_routes(&[added.key]) + .expect("remove remaining route"); + rx.try_recv().expect("subsequent change notification"); + + sync_pending(&db, &ddm, &log, &handle, &rx) + .expect("sync subsequent change"); + + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn sync_advertises_groups_missing_from_ddm() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(*originated, vec![origin(&route)]); + } + + #[test] + fn sync_advertises_v4_and_v6_overlay_groups() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_v6", log.clone()).expect("db"); + let v4 = asm_route(225, 1, 1, 1); + let v6 = asm_route_v6(1); + db.add_static_mcast_routes(&[v4.clone(), v6.clone()]) + .expect("add routes"); + + let ddm = TestDdm::default(); + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 2); + assert!(originated.contains(&origin(&v4))); + assert!(originated.contains(&origin(&v6))); + } + + #[test] + fn sync_withdraws_groups_absent_from_mrib() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_remove", log.clone()).expect("db"); + + let stale = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&stale)); + + sync(&db, &ddm, &log, &handle).expect("sync"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn sync_in_sync_makes_no_changes() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&route)); + + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + *originated, + vec![origin(&route)], + "in-sync group must not be re-added" + ); + } + + #[test] + fn sync_adds_and_withdraws_together() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_mixed", log.clone()).expect("db"); + let keep = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&keep)) + .expect("add route"); + + let ddm = TestDdm::default(); + let stale = asm_route(225, 2, 2, 2); + ddm.multicast_originated + .lock() + .unwrap() + .push(origin(&stale)); + + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(*originated, vec![origin(&keep)]); + } + + #[test] + fn sync_withdraws_removed_route() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_sync_withdraw", log.clone()).expect("db"); + let removed = asm_route(225, 1, 1, 1); + let other = asm_route(225, 2, 2, 2); + db.add_static_mcast_routes(&[removed.clone(), other.clone()]) + .expect("add routes"); + + let ddm = TestDdm::default(); + sync(&db, &ddm, &log, &handle).expect("sync"); + + db.remove_static_mcast_routes(&[removed.key]) + .expect("remove route"); + sync(&db, &ddm, &log, &handle).expect("sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + *originated, + vec![origin(&other)], + "unrelated group must remain advertised" + ); + } +} diff --git a/mg-lower/src/platform.rs b/mg-lower/src/platform.rs index 8350f245b..effe380ff 100644 --- a/mg-lower/src/platform.rs +++ b/mg-lower/src/platform.rs @@ -8,7 +8,7 @@ use std::net::IpAddr; use std::time::Duration; use ddm_admin_client::types::Error as DdmError; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use dpd_client::types::{Error as DpdError, *}; // Use the crate-specific re-exports of progenitor_client types to avoid // version conflicts: dpd_client re-exports from progenitor-client 0.11 (via @@ -217,6 +217,47 @@ pub trait Ddm { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, >; + + /// Get multicast group subscriptions originated by this router. + /// + /// Each `MulticastOrigin` pairs an overlay group address with its + /// underlay mapping (ff04::/64) along with an optional source for + /// (S,G) routes. + /// + /// Method names follow the DDM admin API convention + /// (`originated_multicast_groups`, not `originated_multicast_origins`). + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + >; + + /// Advertise multicast group subscriptions to DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; + + /// Withdraw multicast group subscriptions from DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; } /// This trait wraps the methods that have expectations about switch zone @@ -406,6 +447,35 @@ impl Ddm for ProductionDdm { > { self.client.withdraw_tunnel_endpoints(body).await } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + self.client.get_originated_multicast_groups().await + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.advertise_multicast_groups(body).await + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.withdraw_multicast_groups(body).await + } } /// Production switch zone that uses libnet for route lookups (illumos only). @@ -700,6 +770,7 @@ pub(crate) mod test { pub(crate) struct TestDdm { pub(crate) tunnel_originated: Mutex>, pub(crate) originated: Mutex>, + pub(crate) multicast_originated: Mutex>, } impl Default for TestDdm { @@ -707,6 +778,7 @@ pub(crate) mod test { Self { tunnel_originated: Mutex::new(Vec::default()), originated: Mutex::new(Vec::default()), + multicast_originated: Mutex::new(Vec::default()), } } } @@ -767,6 +839,45 @@ pub(crate) mod test { .retain(|x| !body.contains(x)); Ok(ddm_response_ok!(())) } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + Ok(ddm_response_ok!( + self.multicast_originated.lock().unwrap().clone() + )) + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.multicast_originated + .lock() + .unwrap() + .extend(body.clone()); + Ok(ddm_response_ok!(())) + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.multicast_originated + .lock() + .unwrap() + .retain(|x| !body.contains(x)); + Ok(ddm_response_ok!(())) + } } /// A mock switch zone implementation. diff --git a/mgd/src/main.rs b/mgd/src/main.rs index 15a65c2e6..f19bbc63a 100644 --- a/mgd/src/main.rs +++ b/mgd/src/main.rs @@ -179,14 +179,23 @@ async fn run(args: RunArgs) { client: mg_lower::new_ddm_client(&log), }; let sw = mg_lower::ProductionSwitchZone {}; - Builder::new() - .name("mg-lower".to_string()) - .spawn(move || { - mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); - }) + let backends = mg_lower::Backends { dpd, ddm, sw }; + mg_lower::spawn(ctx.tep, db, log, stats, rt, backends) .expect("failed to start mg-lower"); } + #[cfg(all(feature = "mg-lower", target_os = "illumos"))] + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let log = log.clone(); + let db = context.db.clone(); + let ddm = mg_lower::ProductionDdm { + client: mg_lower::new_ddm_client(&log), + }; + mg_lower::mrib::spawn(db, log, rt, ddm) + .expect("failed to start mg-lower mrib"); + } + start_bgp_routers( context.clone(), db.get_bgp_routers() diff --git a/openapi/ddm-admin/ddm-admin-5.0.0-658a80.json b/openapi/ddm-admin/ddm-admin-5.0.0-658a80.json new file mode 100644 index 000000000..1140bc8fb --- /dev/null +++ b/openapi/ddm-admin/ddm-admin-5.0.0-658a80.json @@ -0,0 +1,965 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "DDM Admin", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "5.0.0" + }, + "paths": { + "/disable-stats": { + "post": { + "operationId": "disable_stats", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/enable-stats": { + "post": { + "operationId": "enable_stats", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnableStatsRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/external_peers": { + "get": { + "operationId": "get_external_peers", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExternalPeers" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "set_external_peers", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExternalPeers" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/multicast_group": { + "put": { + "operationId": "advertise_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/multicast_groups": { + "get": { + "operationId": "get_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_ImportedMulticastRoute", + "type": "array", + "items": { + "$ref": "#/components/schemas/ImportedMulticastRoute" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/originated": { + "get": { + "operationId": "get_originated", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_Ipv6Net", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/originated_multicast_groups": { + "get": { + "operationId": "get_originated_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/originated_tunnel_endpoints": { + "get": { + "operationId": "get_originated_tunnel_endpoints", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelOrigin" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/peers": { + "get": { + "operationId": "get_peers", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_PeerInfo", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/PeerInfo" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/peers/{addr}": { + "delete": { + "operationId": "expire_peer", + "parameters": [ + { + "in": "path", + "name": "addr", + "required": true, + "schema": { + "type": "string", + "format": "ipv6" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/prefix": { + "put": { + "operationId": "advertise_prefixes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_Ipv6Net", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_prefixes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_Ipv6Net", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/prefixes": { + "get": { + "operationId": "get_prefixes", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_Set_of_PathVector", + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PathVector" + }, + "uniqueItems": true + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/sync": { + "put": { + "operationId": "sync", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/tunnel_endpoint": { + "put": { + "operationId": "advertise_tunnel_endpoints", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_tunnel_endpoints", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/tunnel_endpoints": { + "get": { + "operationId": "get_tunnel_endpoints", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_TunnelRoute", + "type": "array", + "items": { + "$ref": "#/components/schemas/TunnelRoute" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Duration": { + "type": "object", + "properties": { + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "nanos", + "secs" + ] + }, + "EnableStatsRequest": { + "type": "object", + "properties": { + "rack_id": { + "type": "string", + "format": "uuid" + }, + "sled_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "rack_id", + "sled_id" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "ExternalMulticastIp": { + "description": "A validated IP address for a customer-visible overlay multicast group.\n\nThe overlay multicast group is the address guests can send to and join, e.g. `233.252.0.1` or `ff0e::1`, and spans both address families. Its rack-internal underlay mapping is validated separately via `UnderlayMulticastIpv6`, with a 1:1 mapping between the two.\n\nJSON uses the ordinary IP-address string form. Deserialization rejects addresses that are not multicast, use a reserved, interface-local, or link-local IPv6 scope, fall in the IPv4 local network control block (`224.0.0.0/24`) or the `ff04::/64` underlay subnet, are a reserved base or SSM null address, or carry malformed prefix-based or embedded-RP flags.", + "type": "string", + "format": "ip" + }, + "ExternalPeers": { + "type": "object", + "properties": { + "address_objects": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": [ + "address_objects" + ] + }, + "ImportedMulticastRoute": { + "description": "A multicast route with the interface on which its peer was discovered.", + "type": "object", + "properties": { + "if_name": { + "description": "Local interface on which the advertising peer was discovered.", + "type": "string" + }, + "route": { + "description": "The imported multicast route.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastRoute" + } + ] + } + }, + "required": [ + "if_name", + "route" + ] + }, + "IpNet": { + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::IpNet", + "version": "0.1.0" + }, + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Net" + } + ] + } + ] + }, + "Ipv4Net": { + "example": "192.168.1.0/24", + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and prefix length", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv4Net", + "version": "0.1.0" + }, + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$" + }, + "Ipv6Net": { + "example": "fd12:3456::/64", + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv6Net", + "version": "0.1.0" + }, + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" + }, + "MulticastOrigin": { + "description": "The multicast group origin as exchanged between DDM peers.", + "type": "object", + "properties": { + "metric": { + "description": "The metric for path selection. Among paths of equal length, a lower value is preferred.", + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "overlay_group": { + "description": "The overlay multicast group address (IPv4 or IPv6).", + "allOf": [ + { + "$ref": "#/components/schemas/ExternalMulticastIp" + } + ] + }, + "source": { + "nullable": true, + "description": "Optional source address for Source-Specific Multicast `(S,G)` routes. `None` for Any-Source Multicast `(*,G)` routes.", + "default": null, + "type": "string", + "format": "ip" + }, + "underlay_group": { + "description": "The underlay multicast group address that the overlay group maps to (1:1).", + "allOf": [ + { + "$ref": "#/components/schemas/UnderlayMulticastIpv6" + } + ] + }, + "vni": { + "description": "VNI identifying the VPC/network context for this group.", + "type": "integer", + "format": "uint32", + "minimum": 0, + "maximum": 16777215 + } + }, + "required": [ + "overlay_group", + "underlay_group", + "vni" + ] + }, + "MulticastPathHop": { + "description": "A single hop in the multicast path.\n\n`router_id` identifies the hop for loop detection, and `underlay_addr` is the address replication toward this router targets.", + "type": "object", + "properties": { + "router_id": { + "description": "Router identifier (i.e., the ddmd router ID, which defaults to the hostname).", + "type": "string" + }, + "underlay_addr": { + "description": "The underlay address of this router (for replication targeting).", + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "router_id", + "underlay_addr" + ] + }, + "MulticastRoute": { + "description": "A multicast route learned via DDM.\n\nCarries a `MulticastOrigin` (overlay group + ff04::/64 underlay mapping) and the path vector from the originating subscriber through intermediate transit routers.", + "type": "object", + "properties": { + "nexthop": { + "description": "Underlay nexthop address (DDM peer that advertised this route). Used to associate the route with a peer for expiration.", + "type": "string", + "format": "ipv6" + }, + "origin": { + "description": "The multicast group origin information.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastOrigin" + } + ] + }, + "path": { + "description": "Path vector from the originating subscriber outward. Each hop records the router that redistributed this subscription announcement. Used for loop detection on pull and for future replication optimization in multi-rack topologies.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastPathHop" + } + } + }, + "required": [ + "nexthop", + "origin" + ] + }, + "PathVector": { + "type": "object", + "properties": { + "destination": { + "$ref": "#/components/schemas/Ipv6Net" + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "destination", + "path" + ] + }, + "PeerInfo": { + "description": "Peer information with an optional interface name.", + "type": "object", + "properties": { + "addr": { + "type": "string", + "format": "ipv6" + }, + "host": { + "type": "string" + }, + "if_name": { + "nullable": true, + "description": "Interface name the peer was discovered on (e.g., \"tfportrear0_0\").", + "default": null, + "type": "string" + }, + "kind": { + "$ref": "#/components/schemas/RouterKind" + }, + "status": { + "$ref": "#/components/schemas/PeerStatus" + } + }, + "required": [ + "addr", + "host", + "kind", + "status" + ] + }, + "PeerStatus": { + "description": "Status of a DDM peer, including how long the peer has been in its current state.", + "oneOf": [ + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Init" + ] + } + }, + "required": [ + "duration", + "type" + ] + }, + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Solicit" + ] + } + }, + "required": [ + "duration", + "type" + ] + }, + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Exchange" + ] + } + }, + "required": [ + "duration", + "type" + ] + }, + { + "type": "object", + "properties": { + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "type": { + "type": "string", + "enum": [ + "Expired" + ] + } + }, + "required": [ + "duration", + "type" + ] + } + ] + }, + "RouterKind": { + "type": "integer", + "enum": [ + 0, + 1 + ] + }, + "TunnelOrigin": { + "type": "object", + "properties": { + "boundary_addr": { + "type": "string", + "format": "ipv6" + }, + "metric": { + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "overlay_prefix": { + "$ref": "#/components/schemas/IpNet" + }, + "vni": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "boundary_addr", + "overlay_prefix", + "vni" + ] + }, + "TunnelRoute": { + "type": "object", + "properties": { + "nexthop": { + "type": "string", + "format": "ipv6" + }, + "origin": { + "$ref": "#/components/schemas/TunnelOrigin" + } + }, + "required": [ + "nexthop", + "origin" + ] + }, + "UnderlayMulticastIpv6": { + "description": "A validated underlay multicast IPv6 address.\n\nUnderlay multicast addresses must be within the subnet allocated by Omicron for rack-internal multicast traffic (ff04::/64). This is a subset of the admin-local scope (ff04::/16) defined in RFC 4291.", + "type": "string", + "format": "ipv6" + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index cc34c49e0..d49eb9945 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-4.0.0-27a7bd.json \ No newline at end of file +ddm-admin-5.0.0-658a80.json \ No newline at end of file diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 682efcce7..4f95d7976 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -8,6 +8,8 @@ client-common.workspace = true zone = { git = "https://github.com/oxidecomputer/zone" } ddm-admin-client = { path = "../ddm-admin-client" } ddm-api-types-versions.workspace = true +dpd-client.workspace = true +dpd-types-versions.workspace = true anyhow.workspace = true slog.workspace = true diff --git a/tests/conf/dpd-ports-dual.toml b/tests/conf/dpd-ports-dual.toml new file mode 100644 index 000000000..17777befc --- /dev/null +++ b/tests/conf/dpd-ports-dual.toml @@ -0,0 +1,17 @@ +[int0] +fec = "None" +speed = "Speed10G" +autoneg = true +kr = true + +[rear0] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true + +[rear1] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true diff --git a/tests/conf/dpd-ports-sextet.toml b/tests/conf/dpd-ports-sextet.toml new file mode 100644 index 000000000..b513cbb8a --- /dev/null +++ b/tests/conf/dpd-ports-sextet.toml @@ -0,0 +1,31 @@ +[int0] +fec = "None" +speed = "Speed10G" +autoneg = true +kr = true + +[rear0] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true + +[rear1] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true + +[qsfp0] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true +allow_ddm_traffic = true + +[qsfp1] +fec = "None" +speed = "Speed100G" +autoneg = true +kr = true +allow_ddm_traffic = true diff --git a/tests/conf/softnpu-dual-sidecar_a.dual.toml b/tests/conf/softnpu-dual-sidecar_a.dual.toml new file mode 100644 index 000000000..3a277802c --- /dev/null +++ b/tests/conf/softnpu-dual-sidecar_a.dual.toml @@ -0,0 +1,5 @@ +p4_program = "/opt/libsidecar_lite.so" +ports = [ + { sidecar = "sw0", scrimlet = "sr0", mtu = 1524 }, + { sidecar = "sw1", scrimlet = "sr1", mtu = 1524 }, +] diff --git a/tests/conf/softnpu-dual-sidecar_b.dual.toml b/tests/conf/softnpu-dual-sidecar_b.dual.toml new file mode 100644 index 000000000..fd2cce475 --- /dev/null +++ b/tests/conf/softnpu-dual-sidecar_b.dual.toml @@ -0,0 +1,5 @@ +p4_program = "/opt/libsidecar_lite.so" +ports = [ + { sidecar = "sw2", scrimlet = "sr2", mtu = 1524 }, + { sidecar = "sw3", scrimlet = "sr3", mtu = 1524 }, +] diff --git a/tests/src/ddm.rs b/tests/src/ddm.rs index 3d61175f7..642019ea4 100644 --- a/tests/src/ddm.rs +++ b/tests/src/ddm.rs @@ -8,14 +8,21 @@ use ddm_admin_client::Client; use ddm_api_types_versions::latest::db::PeerStatus; use ddm_api_types_versions::latest::external_peers::ExternalPeers; use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin, Vni}; +use dpd_types_versions::latest::link::LinkId; +use dpd_types_versions::latest::mcast::{ + Direction, MulticastGroupCreateUnderlayEntry, MulticastGroupMember, + MulticastGroupUnderlayResponse, +}; use oxnet::Ipv6Net; use slog::{Drain, Logger}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::env; use std::net::Ipv6Addr; use std::ops::{Deref, DerefMut}; use std::thread::sleep; use std::time::{Duration, Instant}; +use tokio::sync::oneshot; +use tokio::sync::oneshot::error::TryRecvError; use zone::Zlogin; use ztest::*; @@ -56,12 +63,88 @@ macro_rules! ip6_net { }; } +type MulticastImports = HashSet<(String, MulticastOrigin, u64)>; + +macro_rules! assert_client_state { + ($client:expr, $read:expr, $expected:expr, $period:expr, $count:expr) => {{ + let expected = $expected; + let mut actual = $read.await; + for _ in 0..$count { + if actual.as_ref().ok() == Some(&expected) { + break; + } + sleep(Duration::from_secs($period)); + actual = $read.await; + } + if actual.as_ref().ok() != Some(&expected) { + eprintln_nopipe!( + "{}: expected {expected:#?}, last response: {actual:#?}", + stringify!($client), + ); + dump_client_state(stringify!($client), $client).await; + } + assert_eq!(actual.ok(), Some(expected)); + }}; + ($client:expr, $read:expr, $expected:expr) => { + assert_client_state!($client, $read, $expected, 1, 10) + }; +} + +macro_rules! assert_multicast_imports { + ($client:expr, [$(($iface:expr, $origin:expr)),* $(,)?] + $(, $period:expr, $count:expr)?) => {{ + let expected: MulticastImports = + [$(($iface.to_string(), ($origin).clone(), ($origin).metric())),*] + .into_iter() + .collect(); + assert_client_state!( + $client, + multicast_imports($client), + expected + $(, $period, $count)? + ); + }}; +} + +macro_rules! assert_multicast_origins { + ($client:expr, [$($origin:expr),* $(,)?]) => {{ + let expected: HashSet<(MulticastOrigin, u64)> = + [$((($origin).clone(), ($origin).metric())),*] + .into_iter() + .collect(); + assert_client_state!($client, multicast_origins($client), expected); + }}; +} + const ZONE_BRAND: &str = "omicron1"; /// How long to wait for a stopped ddmd to exit, and how often to check. const DDMD_STOP_TIMEOUT: Duration = Duration::from_secs(30); const DDMD_STOP_POLL: Duration = Duration::from_millis(100); +const MULTICAST_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30); +const MULTICAST_POLL_INTERVAL: Duration = Duration::from_millis(250); + +/// The VNI shared by every multicast origin these tests advertise. +const MULTICAST_VNI: u32 = 77; + +/// The duration for holding multicast members steady after an import change. +const MEMBER_HOLD: Duration = Duration::from_secs(3); + +/// How often the failover observer samples, less coarse than +/// [`MULTICAST_POLL_INTERVAL`] to narrow the window a flap could slip by. +const FAILOVER_POLL_INTERVAL: Duration = Duration::from_millis(100); +/// How long the failover observer keeps watching after the transit converges. +const FAILOVER_SETTLE: Duration = Duration::from_secs(3); + +/// DPD API version for the raw underlay-group requests. +/// +/// This is the version the pinned `dpd-client` is generated from, matching +/// both what `ddmd` sends out and the `latest` types these requests use. +fn dpd_api_version() -> &'static str { + >::api_version() +} + struct SoftnpuZone<'a> { zfs: &'a Zfs, zone: Zone, @@ -496,10 +579,6 @@ macro_rules! run_topo { } #[tokio::test] -async fn test_trio_v3() -> Result<()> { - test_trio().await -} - async fn test_trio() -> Result<()> { // A trio. Two server routers and one transit router. // @@ -580,10 +659,9 @@ async fn run_trio_tests( let s2 = Client::new("http://10.0.0.2:8000", log.clone()); let t1 = Client::new("http://10.0.0.3:8000", log.clone()); - // If we never get a response from a server, return 99 as a sentinel value. - wait_for_eq!(s1.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(s2.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(t1.get_peers().await.map_or(99, |x| x.len()), 2); + assert_client_state!(&s1, peer_count(&s1), 1); + assert_client_state!(&s2, peer_count(&s2), 1); + assert_client_state!(&t1, peer_count(&t1), 2); println_nopipe!("initial peering test passed"); @@ -767,14 +845,14 @@ async fn run_trio_tests( println_nopipe!("tunnel endpoint withdraw passed"); - run_trio_mcast(zs1, &s1, &s2, &t1, &log).await + run_trio_mcast(zs1, zt1, &s1, &s2, &t1, &log).await } /// Multicast group advertise/withdraw across the trio. /// -/// This emulates how mg-lower in the switch zone publishes +/// This emulates how mg-lower operates in the switch zone when publishing /// overlay→underlay multicast bindings: the transit router -/// originates an advertisement, and the server routers learn it via +/// originates an advertisement and the server routers learn it via /// the DDM exchange. /// /// ```text @@ -785,26 +863,32 @@ async fn run_trio_tests( /// (O4) /// ``` /// -/// - t1 originates one IPv4 and one IPv6 group, and both servers import -/// them. t1 imports nothing back (a re-import would be a loop). -/// - s1 restarts and relearns both groups from t1. -/// - t1 withdraws, and both servers clear. -/// - s1 originates O4. t1 redistributes it to s2, and s1 imports nothing -/// in return. s1 withdraws, and t1 and s2 clear. +/// 1. t1 originates an IPv4 and an IPv6 group, and both servers import +/// them. t1 imports nothing in return (a re-import would cause a loop). +/// 2. s1 restarts and re-learns both groups from t1. +/// 3. t1 withdraws, and both servers clear. +/// 4. s1 originates O4. t1 redistributes it to s2, s1 imports nothing in +/// return, and t1 programs rear0 for O4's underlay group. +/// 5. s2 also subscribes to O4. t1 programs rear0 and rear1, then rear1 once +/// s1 withdraws, and drains the group when s2 withdraws. +/// 6. s1 subscribes to an origin whose underlay group doesn't exist yet. The +/// group is still missing until it is created, after which the +/// periodic sweep programs rear0. s1 withdraws, and the group drains. async fn run_trio_mcast( zs1: &RouterZone<'_>, + zt1: &RouterZone<'_>, s1: &Client, s2: &Client, t1: &Client, log: &Logger, ) -> Result<()> { - wait_for_eq!(multicast_originated_count(t1).await?, 0); + assert_multicast_origins!(t1, []); // Cover advertisement and withdrawal for both IP families. let mcast_origin_v4 = MulticastOrigin::new( "233.252.0.1".parse().unwrap(), "ff04::100".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); @@ -812,32 +896,48 @@ async fn run_trio_mcast( let mcast_origin_v6 = MulticastOrigin::new( "ff0e::1".parse().unwrap(), "ff04::101".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); + let ipv4_origin_underlay = create_multicast_underlay_group( + zt1, + mcast_origin_v4.underlay_group().into(), + )?; + let s1_member = [underlay_member("rear0")]; + let s2_member = [underlay_member("rear1")]; + t1.advertise_multicast_groups(&vec![ mcast_origin_v4.clone(), mcast_origin_v6.clone(), ]) .await?; - wait_for_eq!(multicast_originated_count(t1).await?, 2); - wait_for_eq!(multicast_group_count(s1).await?, 2); - wait_for_eq!(multicast_group_count(s2).await?, 2); + assert_multicast_origins!(t1, [mcast_origin_v4, mcast_origin_v6]); + assert_multicast_imports!( + s1, + [("sl0", mcast_origin_v4), ("sl0", mcast_origin_v6)] + ); + assert_multicast_imports!( + s2, + [("sl1", mcast_origin_v4), ("sl1", mcast_origin_v6)] + ); // A looped announcement bouncing back from a server would show up here. - assert_eq!(multicast_group_count(t1).await?, 0); + assert_eq!(multicast_imports(t1).await?, MulticastImports::new()); println_nopipe!("multicast group advertise passed"); - // Server router restart: s1's view of the multicast group must - // converge again after ddmd restarts. `wait_for_eq` tolerates the - // restart window via the `unwrap_or` sentinel. + // Server router restart: s1's view of the multicast group must converge + // again after ddmd restarts. `assert_multicast_imports!` retries through + // the restart window. zs1.stop_router()?; zs1.start_router(false)?; let s1 = Client::new("http://10.0.0.1:8000", log.clone()); - wait_for_eq!(multicast_group_count(&s1).await.unwrap_or(99), 2); + assert_multicast_imports!( + &s1, + [("sl0", mcast_origin_v4), ("sl0", mcast_origin_v6)] + ); println_nopipe!("multicast router restart passed"); @@ -847,31 +947,395 @@ async fn run_trio_mcast( ]) .await?; - wait_for_eq!(multicast_originated_count(t1).await?, 0); + assert_multicast_origins!(t1, []); // A periodic pull can also clear the routes within this deadline. - wait_for_eq!(multicast_group_count(&s1).await?, 0, 1, 5); - wait_for_eq!(multicast_group_count(s2).await?, 0, 1, 5); - assert_eq!(multicast_group_count(t1).await?, 0); + assert_multicast_imports!(&s1, [], 1, 5); + assert_multicast_imports!(s2, [], 1, 5); + assert_eq!(multicast_imports(t1).await?, MulticastImports::new()); println_nopipe!("multicast group withdraw passed"); s1.advertise_multicast_groups(&vec![mcast_origin_v4.clone()]) .await?; - wait_for_eq!(multicast_originated_count(&s1).await?, 1); - wait_for_eq!(multicast_group_count(t1).await?, 1); - wait_for_eq!(multicast_group_count(s2).await?, 1); - assert_eq!(multicast_group_count(&s1).await?, 0); + assert_multicast_origins!(&s1, [mcast_origin_v4]); + assert_multicast_imports!(t1, [("tfportrear0_0", mcast_origin_v4)]); + assert_multicast_imports!(s2, [("sl1", mcast_origin_v4)]); + assert_eq!(multicast_imports(&s1).await?, MulticastImports::new()); + + wait_for_multicast_members(zt1, &ipv4_origin_underlay, &s1_member)?; println_nopipe!("multicast server-originated redistribution passed"); - s1.withdraw_multicast_groups(&vec![mcast_origin_v4]).await?; + let origin = mcast_origin_v4; + let group = &ipv4_origin_underlay; + s2.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + wait_for_multicast_members( + zt1, + group, + &[s1_member[0].clone(), s2_member[0].clone()], + )?; - wait_for_eq!(multicast_originated_count(&s1).await?, 0); - wait_for_eq!(multicast_group_count(t1).await?, 0, 1, 5); - wait_for_eq!(multicast_group_count(s2).await?, 0, 1, 5); + s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_origins!(&s1, []); + assert_multicast_imports!(t1, [("tfportrear1_0", origin)], 1, 5); + wait_for_multicast_members(zt1, group, &s2_member)?; + + s2.withdraw_multicast_groups(&vec![origin]).await?; + assert_multicast_imports!(t1, [], 1, 5); + assert_multicast_imports!(&s1, [], 1, 5); + assert_multicast_imports!(s2, [], 1, 5); + wait_for_multicast_members(zt1, group, &[])?; + + println_nopipe!("multicast underlay replication members passed"); + + let delayed_origin = MulticastOrigin::new( + "233.252.0.2".parse().unwrap(), + "ff04::102".parse().unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), + None, + ) + .unwrap(); + let delayed_group_ip: Ipv6Addr = delayed_origin.underlay_group().into(); + + s1.advertise_multicast_groups(&vec![delayed_origin.clone()]) + .await?; + assert_multicast_imports!(t1, [("tfportrear0_0", delayed_origin)]); + assert_multicast_imports!(s2, [("sl1", delayed_origin)]); + assert_multicast_underlay_group_missing(zt1, delayed_group_ip)?; - println_nopipe!("multicast server-originated withdraw passed"); + let delayed_group = create_multicast_underlay_group(zt1, delayed_group_ip)?; + wait_for_multicast_members(zt1, &delayed_group, &s1_member)?; + + s1.withdraw_multicast_groups(&vec![delayed_origin]).await?; + assert_multicast_imports!(t1, []); + wait_for_multicast_members(zt1, &delayed_group, &[])?; + + println_nopipe!("multicast delayed underlay group creation passed"); + + Ok(()) +} + +fn assert_multicast_underlay_group_missing( + zone: &RouterZone<'_>, + group_ip: Ipv6Addr, +) -> Result<()> { + let version = dpd_api_version(); + let status = zone.zexec(&format!( + "curl --noproxy '*' --globoff --silent --show-error \ + --max-time 5 --output /dev/null --write-out '%{{http_code}}' \ + --header 'api-version: {version}' \ + 'http://[::1]:12224/multicast/underlay-groups/{group_ip}'" + ))?; + if status.trim() != "404" { + return Err(anyhow!( + "underlay group {group_ip}: expected HTTP 404, got {status:?}" + )); + } + Ok(()) +} + +fn create_multicast_underlay_group( + zone: &RouterZone<'_>, + group_ip: Ipv6Addr, +) -> Result { + create_multicast_underlay_group_with(zone, group_ip, &[]) +} + +fn create_multicast_underlay_group_with( + zone: &RouterZone<'_>, + group_ip: Ipv6Addr, + members: &[MulticastGroupMember], +) -> Result { + let body = serde_json::to_string(&MulticastGroupCreateUnderlayEntry { + group_ip: group_ip.try_into()?, + tag: Some(format!("{}:{group_ip}", zone.zone.name)), + members: members.to_vec(), + })? + .replace('\'', "'\\''"); + let version = dpd_api_version(); + let response = zone.zexec(&format!( + "curl --noproxy '*' --globoff --fail --silent --show-error \ + --max-time 5 --request POST \ + --header 'api-version: {version}' \ + --header 'Content-Type: application/json' --data '{body}' \ + 'http://[::1]:12224/multicast/underlay-groups'" + ))?; + Ok(serde_json::from_str(&response)?) +} + +fn get_multicast_underlay_group( + zone: &RouterZone<'_>, + group: &MulticastGroupUnderlayResponse, +) -> Result { + let version = dpd_api_version(); + let body = zone.zexec(&format!( + "curl --noproxy '*' --globoff --fail --silent --show-error \ + --max-time 5 --header 'api-version: {version}' \ + 'http://[::1]:12224/multicast/underlay-groups/{}'", + group.group_ip, + ))?; + Ok(serde_json::from_str(&body)?) +} + +fn same_group( + actual: &MulticastGroupUnderlayResponse, + group: &MulticastGroupUnderlayResponse, +) -> bool { + actual.group_ip == group.group_ip + && actual.tag == group.tag + && actual.external_group_id == group.external_group_id + && actual.underlay_group_id == group.underlay_group_id +} + +fn same_membership( + actual: &[MulticastGroupMember], + members: &[MulticastGroupMember], +) -> bool { + actual.len() == members.len() + && members.iter().all(|member| actual.contains(member)) +} + +fn wait_for_multicast_members( + zone: &RouterZone<'_>, + group: &MulticastGroupUnderlayResponse, + members: &[MulticastGroupMember], +) -> Result<()> { + let start = Instant::now(); + loop { + let response = get_multicast_underlay_group(zone, group); + if let Ok(actual) = &response + && same_group(actual, group) + && same_membership(&actual.members, members) + { + return Ok(()); + } + if start.elapsed() >= MULTICAST_CONVERGENCE_TIMEOUT { + return Err(anyhow!( + "underlay group {} did not converge to {members:?}; \ + last response: {response:?}", + group.group_ip, + )); + } + sleep(MULTICAST_POLL_INTERVAL); + } +} + +fn assert_multicast_members_unchanged( + zone: &RouterZone<'_>, + group: &MulticastGroupUnderlayResponse, + members: &[MulticastGroupMember], + window: Duration, +) -> Result<()> { + let start = Instant::now(); + while start.elapsed() < window { + let actual = get_multicast_underlay_group(zone, group)?; + if !same_group(&actual, group) + || !same_membership(&actual.members, members) + { + return Err(anyhow!( + "underlay group {} left {members:?} within {window:?}: \ + {actual:?}", + group.group_ip, + )); + } + sleep(MULTICAST_POLL_INTERVAL); + } + Ok(()) +} + +#[tokio::test] +async fn test_dual_sidecar() -> Result<()> { + // sidecar A sidecar B + // t1 rear0 --- sr0 | sw0 --- sl0 s1 sl1 --- sw2 | sr2 --- rear0 t2 + // t1 rear1 --- sr1 | sw1 --- sl2 s2 sl3 --- sw3 | sr3 --- rear1 t2 + let tra_sr_0 = SimnetLink::new("tra0", "sr0")?; + let tra_sr_1 = SimnetLink::new("tra1", "sr1")?; + let trb_sr_2 = SimnetLink::new("trb0", "sr2")?; + let trb_sr_3 = SimnetLink::new("trb1", "sr3")?; + + let sl0_sw0 = SimnetLink::new("sl0", "sw0")?; + let sl2_sw1 = SimnetLink::new("sl2", "sw1")?; + let sl1_sw2 = SimnetLink::new("sl1", "sw2")?; + let sl3_sw3 = SimnetLink::new("sl3", "sw3")?; + + let mgmt0 = Etherstub::new("mgmt0")?; + let mg0 = Vnic::new("mg0", &mgmt0.name)?; + let mgd1 = Vnic::new("mgd1", &mgmt0.name)?; + let mgd2 = Vnic::new("mgd2", &mgmt0.name)?; + let mgd3 = Vnic::new("mgd3", &mgmt0.name)?; + let mgd4 = Vnic::new("mgd4", &mgmt0.name)?; + + let _mgip = Ip::new("10.0.0.254/24", &mg0.name, "test")?; + + let zfs = Zfs::new("mgdual")?; + + let sidecar_a = SoftnpuZone::new( + "sidecar_a.dual", + &zfs, + &[ + &tra_sr_0.end_b, + &tra_sr_1.end_b, + &sl0_sw0.end_b, + &sl2_sw1.end_b, + ], + "dual", + )?; + let sidecar_b = SoftnpuZone::new( + "sidecar_b.dual", + &zfs, + &[ + &trb_sr_2.end_b, + &trb_sr_3.end_b, + &sl1_sw2.end_b, + &sl3_sw3.end_b, + ], + "dual", + )?; + + println_nopipe!("start zone s1"); + let s1 = RouterZone::server( + "s1.dual", + &zfs, + &mgd1.name, + &[&sl0_sw0.end_a, &sl1_sw2.end_a], + )?; + println_nopipe!("start zone s2"); + let s2 = RouterZone::server( + "s2.dual", + &zfs, + &mgd2.name, + &[&sl2_sw1.end_a, &sl3_sw3.end_a], + )?; + + println_nopipe!("start zone t1"); + let mut t1 = RouterZone::transit( + "t1.dual", + &zfs, + &mgd3.name, + &[&tra_sr_0.end_a, &tra_sr_1.end_a], + "dual", + "sidecar_a.dual", + )?; + t1.set_port_map(BTreeMap::from([ + ("tra0".into(), "tfportrear0_0".into()), + ("tra1".into(), "tfportrear1_0".into()), + ])); + + println_nopipe!("start zone t2"); + let mut t2 = RouterZone::transit( + "t2.dual", + &zfs, + &mgd4.name, + &[&trb_sr_2.end_a, &trb_sr_3.end_a], + "dual", + "sidecar_b.dual", + )?; + t2.set_port_map(BTreeMap::from([ + ("trb0".into(), "tfportrear0_0".into()), + ("trb1".into(), "tfportrear1_0".into()), + ])); + + println_nopipe!("waiting for zones to come up"); + sleep(Duration::from_secs(10)); + + sidecar_a.setup()?; + sidecar_b.setup()?; + s1.setup(1)?; + s2.setup(2)?; + t1.setup(3)?; + t2.setup(4)?; + + run_topo!(run_dual_sidecar_mcast(&s1, &t1, &t2).await) +} + +async fn run_dual_sidecar_mcast( + zs1: &RouterZone<'_>, + zt1: &RouterZone<'_>, + zt2: &RouterZone<'_>, +) -> Result<()> { + let log = init_logger(); + let s1 = Client::new("http://10.0.0.1:8000", log.clone()); + let s2 = Client::new("http://10.0.0.2:8000", log.clone()); + let t1 = Client::new("http://10.0.0.3:8000", log.clone()); + let t2 = Client::new("http://10.0.0.4:8000", log.clone()); + + assert_client_state!(&s1, peer_count(&s1), 2); + assert_client_state!(&s2, peer_count(&s2), 2); + assert_client_state!(&t1, peer_count(&t1), 2); + assert_client_state!(&t2, peer_count(&t2), 2); + + println_nopipe!("dual sidecar peering passed"); + + let origin = MulticastOrigin::new( + "233.252.0.20".parse().unwrap(), + "ff04::120".parse().unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), + None, + ) + .unwrap(); + let group_ip: Ipv6Addr = origin.underlay_group().into(); + let group_a = create_multicast_underlay_group(zt1, group_ip)?; + let group_b = create_multicast_underlay_group(zt2, group_ip)?; + let s1_member = [underlay_member("rear0")]; + let s2_member = [underlay_member("rear1")]; + let both = [s1_member[0].clone(), s2_member[0].clone()]; + + s1.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(&t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(&t2, [("tfportrear0_0", origin)]); + wait_for_multicast_members(zt1, &group_a, &s1_member)?; + wait_for_multicast_members(zt2, &group_b, &s1_member)?; + + s2.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!( + &t1, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + assert_multicast_imports!( + &t2, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + wait_for_multicast_members(zt1, &group_a, &both)?; + wait_for_multicast_members(zt2, &group_b, &both)?; + assert_multicast_imports!(&s1, [("sl0", origin), ("sl1", origin)]); + + println_nopipe!("dual sidecar subscription passed"); + + zt2.stop_router()?; + zt2.start_router(false)?; + let t2 = Client::new("http://10.0.0.4:8000", log.clone()); + assert_multicast_imports!( + &t2, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + assert_multicast_members_unchanged(zt2, &group_b, &both, MEMBER_HOLD)?; + + println_nopipe!("dual sidecar transit restart passed"); + + zs1.zexec("ipadm delete-addr sl1/v6")?; + assert_client_state!(&s1, exchange_peer_count(&s1), 1, 1, 30); + assert_multicast_imports!(&t2, [("tfportrear1_0", origin)], 1, 30); + assert_multicast_imports!(&s1, [("sl0", origin)], 1, 30); + wait_for_multicast_members(zt2, &group_b, &s2_member)?; + assert_multicast_members_unchanged(zt1, &group_a, &both, MEMBER_HOLD)?; + + zs1.zexec("ipadm create-addr -t -T addrconf sl1/v6")?; + assert_client_state!(&s1, exchange_peer_count(&s1), 2, 1, 30); + assert_multicast_imports!( + &t2, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)], + 1, + 30 + ); + wait_for_multicast_members(zt2, &group_b, &both)?; + assert_multicast_imports!(&s1, [("sl0", origin), ("sl1", origin)], 1, 30); + + println_nopipe!("dual sidecar link failover passed"); Ok(()) } @@ -980,11 +1444,10 @@ async fn run_quartet_tests( let s3 = Client::new("http://10.0.0.3:8000", log.clone()); let t1 = Client::new("http://10.0.0.4:8000", log.clone()); - // If we never get a response from a server, return 99 as a sentinel value. - wait_for_eq!(s1.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(s2.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(s3.get_peers().await.map_or(99, |x| x.len()), 1); - wait_for_eq!(t1.get_peers().await.map_or(99, |x| x.len()), 3); + assert_client_state!(&s1, peer_count(&s1), 1); + assert_client_state!(&s2, peer_count(&s2), 1); + assert_client_state!(&s3, peer_count(&s3), 1); + assert_client_state!(&t1, peer_count(&t1), 3); println_nopipe!("initial peering test passed"); @@ -1038,8 +1501,8 @@ async fn run_quartet_tests( /// (origin O) /// ``` /// -/// - s1 withdraws origin O. t1 still reaches O via s2. s3 holds its route -/// through a 3 s window, shorter than the 10 s periodic pull. +/// - s1 withdraws origin O. t1 still reaches O via s2. The observer checks +/// s3's route through t1's convergence and for [`FAILOVER_SETTLE`] afterward. /// - s1 still holds s2's copy of O via t1, and s2 drops the copy it /// learned from s1. /// - s2 withdraws O, and t1, s1, and s3 clear. @@ -1052,7 +1515,7 @@ async fn run_quartet_mcast_failover( let origin = MulticastOrigin::new( "233.252.0.10".parse().unwrap(), "ff04::110".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); @@ -1060,34 +1523,56 @@ async fn run_quartet_mcast_failover( s1.advertise_multicast_groups(&vec![origin.clone()]).await?; s2.advertise_multicast_groups(&vec![origin.clone()]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 2); - wait_for_eq!(multicast_group_count(s1).await?, 1); - wait_for_eq!(multicast_group_count(s2).await?, 1); - wait_for_eq!(multicast_group_count(s3).await?, 1); + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportrear1_0", origin)] + ); + assert_multicast_imports!(s1, [("sl0", origin)]); + assert_multicast_imports!(s2, [("sl1", origin)]); + assert_multicast_imports!(s3, [("sl2", origin)]); println_nopipe!("multicast dual origination passed"); - s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 1); - - let deadline = Instant::now() + Duration::from_secs(3); - while Instant::now() < deadline { - let count = multicast_group_count(s3).await?; - if count != 1 { - return Err(anyhow!( - "s3 lost the group after s1 withdrew (count {count})" - )); - } - sleep(Duration::from_millis(100)); - } + let expected = MulticastImports::from([( + "sl2".to_string(), + origin.clone(), + origin.metric(), + )]); + let expected_transit = MulticastImports::from([( + "tfportrear1_0".to_string(), + origin.clone(), + origin.metric(), + )]); + let (ready_tx, ready_rx) = oneshot::channel(); + let (done_tx, done_rx) = oneshot::channel(); + + let failover = async { + ready_rx.await?; + s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; + wait_for_imports( + "t1", + t1, + &expected_transit, + MULTICAST_CONVERGENCE_TIMEOUT, + ) + .await?; + tokio::time::sleep(FAILOVER_SETTLE).await; + done_tx + .send(()) + .map_err(|_| anyhow!("s3 observation stopped before completion")) + }; + tokio::try_join!( + observe_imports_unchanged("s3", s3, &expected, ready_tx, done_rx), + failover, + )?; - wait_for_eq!(multicast_group_count(s1).await?, 1); - wait_for_eq!(multicast_group_count(s2).await?, 0); + assert_multicast_imports!(s1, [("sl0", origin)]); + assert_multicast_imports!(s2, []); println_nopipe!("multicast failover passed"); s2.withdraw_multicast_groups(&vec![origin]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 0); - wait_for_eq!(multicast_group_count(s1).await?, 0); - wait_for_eq!(multicast_group_count(s3).await?, 0); + assert_multicast_imports!(t1, []); + assert_multicast_imports!(s1, []); + assert_multicast_imports!(s3, []); println_nopipe!("multicast final withdrawal passed"); Ok(()) @@ -1115,14 +1600,14 @@ async fn run_quartet_mcast_resync( let origin = MulticastOrigin::new( "233.252.0.11".parse().unwrap(), "ff04::111".parse().unwrap(), - Vni::try_from(77u32).unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), None, ) .unwrap(); s1.advertise_multicast_groups(&vec![origin.clone()]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 1); - wait_for_eq!(multicast_group_count(s3).await?, 1); + assert_multicast_imports!(t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(s3, [("sl2", origin)]); let peers = t1.get_peers().await?; let peer = peers @@ -1137,6 +1622,7 @@ async fn run_quartet_mcast_resync( let groups = s3.get_multicast_groups().await?; let route = groups .iter() + .map(|imported| &imported.route) .find(|route| route.origin == origin) .ok_or_else(|| anyhow!("s3 has not imported the test origin"))?; let vector = serde_json::json!({ @@ -1174,27 +1660,33 @@ async fn run_quartet_mcast_resync( }; let started = Instant::now(); - wait_for_eq!( - { + assert_client_state!( + s3, + async { push(false)?; - multicast_group_count(s3).await? + multicast_imports(s3).await }, - 0 + MulticastImports::new() ); - wait_for_eq!(multicast_group_count(s3).await?, 1, 1, 30); + assert_multicast_imports!(s3, [("sl2", origin)], 1, 30); println_nopipe!("multicast resync import passed"); - s1.withdraw_multicast_groups(&vec![origin]).await?; - wait_for_eq!(multicast_group_count(t1).await?, 0); - wait_for_eq!(multicast_group_count(s3).await?, 0); - wait_for_eq!( - { + s1.withdraw_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(t1, []); + assert_multicast_imports!(s3, []); + assert_client_state!( + s3, + async { push(true)?; - multicast_group_count(s3).await? + multicast_imports(s3).await }, - 1 + MulticastImports::from([( + "sl2".to_string(), + origin.clone(), + origin.metric(), + )]) ); - wait_for_eq!(multicast_group_count(s3).await?, 0, 1, 30); + assert_multicast_imports!(s3, [], 1, 30); println_nopipe!("multicast resync withdrawal passed"); let min_uptime = *uptime + started.elapsed(); @@ -1402,8 +1894,8 @@ async fn run_sextet_tests( _zs2: &RouterZone<'_>, _zs3: &RouterZone<'_>, _zs4: &RouterZone<'_>, - _zt1: &RouterZone<'_>, - _zt2: &RouterZone<'_>, + zt1: &RouterZone<'_>, + zt2: &RouterZone<'_>, ) -> Result<()> { let log = init_logger(); @@ -1434,13 +1926,7 @@ async fn run_sextet_tests( let name = self.name.clone(); tokio::task::block_in_place(|| { rt.block_on(async move { - println_nopipe!("{name}:"); - if let Ok(peers) = c.get_peers().await { - println_nopipe!("peers: {peers:#?}"); - } - if let Ok(prefixes) = c.get_prefixes().await { - println_nopipe!("prefixes: {prefixes:#?}"); - } + dump_client_state(&name, &c).await; }); }); } @@ -1506,10 +1992,7 @@ async fn run_sextet_tests( stringify!($client), $count ); - wait_for_eq!( - $client.get_peers().await.map(|x| x.len()).ok(), - Some($count) - ); + assert_client_state!(&$client, peer_count(&$client), $count); }}; } @@ -1531,21 +2014,17 @@ async fn run_sextet_tests( stringify!($client), $reach ); - wait_for_eq!( - $client - .get_prefixes() - .await - .map(|x| x - .values() - .cloned() - .into_iter() - .flat_map(|x| x - .clone() - .into_iter() - .map(|y| y.destination)) - .collect::>()) - .ok(), - Some($reach) + assert_client_state!( + &$client, + async { + $client.get_prefixes().await.map(|x| { + x.values() + .flatten() + .map(|route| route.destination) + .collect::>() + }) + }, + $reach ); }}; } @@ -1637,7 +2116,7 @@ async fn run_sextet_tests( assert_peer_counts!(PeerCounts::default().server(1).transit(4)); // - // Going down to the first peer should result in three peering sessions + // Going down to the 1st peer should result in three peering sessions // per transit router. Note in the model above that qsfp0/qsfp1 are cross // connected between the two transit routers. Here we are connecting // swA/qsfp0 <-> swB/qsfp1 @@ -1791,6 +2270,219 @@ async fn run_sextet_tests( assert_all_nexthops_are_peers!(); } + run_sextet_mcast(zt1, zt2, &s1, &s3, &t1, &t2).await +} + +fn underlay_member(port: &str) -> MulticastGroupMember { + MulticastGroupMember { + port_id: port.parse().unwrap(), + link_id: LinkId(0), + direction: Direction::Underlay, + } +} + +fn external_member(port: &str) -> MulticastGroupMember { + MulticastGroupMember { + port_id: port.parse().unwrap(), + link_id: LinkId(0), + direction: Direction::External, + } +} + +/// Multicast replication members across the two racks of the sextet. +/// +/// Each transit router's `ddmd` programs the underlay members of its own +/// switch from the subscriptions it imports. Subscriptions still cross racks +/// over the qsfp external peers, but only rear ports become members, since +/// members on both parallel qsfp links would circulate replicas between the +/// racks. The two qsfp links are cross-connected, akin to the diagram on +/// `test_external_peer_sextet`. +/// +/// ```text +/// rack A rack B +/// s1 ---- rear0 [t1] qsfp0 ------------------ qsfp1 [t2] rear0 ---- s3 +/// qsfp1 ------------------ qsfp0 +/// ``` +/// +/// This test checks cross-rack subscriptions and local replication membership. +/// Cross-rack underlay replication remains unsupported until inter-rack +/// replication prevents cycles. +/// +/// 1. Peer t1/qsfp0 <-> t2/qsfp1 only, and s1 subscribes. t1 programs rear0; +/// t2 imports the subscription over qsfp1 but doesn't program anything. +/// 2. s3 subscribes. Each switch imports the other rack's subscription over +/// qsfp and programs only its own rear0. +/// 3. Move the peering to t1/qsfp1 <-> t2/qsfp0. The imports follow the link; +/// both switches stay at rear0. +/// 4. Peer over both links. Each switch imports the other rack over two next +/// hops, a case that would cause looping with qsfp members, and stays at +/// rear0. +/// 5. Drop back to t1/qsfp0 <-> t2/qsfp1. Each switch loses only the dropped +/// link's import, remaining at rear0. +/// 6. s3 withdraws. t1 keeps rear0 and t2 drains to empty membership. +/// 7. s1 withdraws. t2 drops its import, and t1 keeps only its `External` +/// member. +async fn run_sextet_mcast( + zt1: &RouterZone<'_>, + zt2: &RouterZone<'_>, + s1: &Client, + s3: &Client, + t1: &Client, + t2: &Client, +) -> Result<()> { + const QSFP0: &str = "tfportqsfp0_0/v6"; + const QSFP1: &str = "tfportqsfp1_0/v6"; + let external_peers = |aobjs: &[&str]| ExternalPeers { + address_objects: aobjs.iter().map(|x| x.to_string()).collect(), + }; + + // + // Nexus creates each underlay group before subscriptions arrive. + // Each switch gets its own copy of the group. + // + + let origin = MulticastOrigin::new( + "233.252.0.2".parse().unwrap(), + "ff04::200".parse().unwrap(), + Vni::try_from(MULTICAST_VNI).unwrap(), + None, + ) + .unwrap(); + let rear0 = [underlay_member("rear0")]; + let t1_external = [external_member("qsfp0")]; + let t1_members = [rear0[0].clone(), t1_external[0].clone()]; + + // Nexus-owned `External` member on t1, which ddmd must never remove. + let group_t1 = create_multicast_underlay_group_with( + zt1, + origin.underlay_group().into(), + &t1_external, + )?; + let group_t2 = + create_multicast_underlay_group(zt2, origin.underlay_group().into())?; + + // + // Peer swA/qsfp0 <-> swB/qsfp1 only, then subscribe s1. t1 learns the + // subscription from s1 on rear0 and redistributes it to t2, which imports + // it over qsfp1. t2 has no local subscriber and leaves its group empty. + // + + t1.set_external_peers(&external_peers(&[QSFP0])).await?; + t2.set_external_peers(&external_peers(&[QSFP1])).await?; + + s1.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(t2, [("tfportqsfp1_0", origin)]); + wait_for_multicast_members(zt1, &group_t1, &t1_members)?; + wait_for_multicast_members(zt2, &group_t2, &[])?; + + println_nopipe!("sextet multicast remote import passed"); + + // + // A 2nd subscriber in rack B provides each switch with a local rear member. + // Each switch also imports the other rack's subscription over qsfp, and the + // exact wait on t2 fails if that import resolves to a qsfp member. + // + + s3.advertise_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportqsfp0_0", origin)] + ); + assert_multicast_imports!( + t2, + [("tfportrear0_0", origin), ("tfportqsfp1_0", origin)] + ); + wait_for_multicast_members(zt2, &group_t2, &rear0)?; + + println_nopipe!("sextet multicast subscribers in both racks passed"); + + // + // Switch to swA/qsfp1 <-> swB/qsfp0. Shutting down the old external peer + // expires its routes, and the imports move to the new link while ddmd + // programs only rear0 on both switches. + // + + t1.set_external_peers(&external_peers(&[QSFP1])).await?; + t2.set_external_peers(&external_peers(&[QSFP0])).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportqsfp1_0", origin)] + ); + assert_multicast_imports!( + t2, + [("tfportrear0_0", origin), ("tfportqsfp0_0", origin)] + ); + + println_nopipe!("sextet multicast external link move passed"); + + // + // Peer over both links. Each switch now learns the other rack's + // subscriber over two nexthops. With qsfp members, replicas would cycle + // between the racks here, so ddmd still programs only rear0. + // + + let both = external_peers(&[QSFP0, QSFP1]); + t1.set_external_peers(&both).await?; + t2.set_external_peers(&both).await?; + for client in [t1, t2] { + assert_multicast_imports!( + client, + [ + ("tfportrear0_0", origin), + ("tfportqsfp0_0", origin), + ("tfportqsfp1_0", origin), + ] + ); + } + wait_for_multicast_members(zt1, &group_t1, &t1_members)?; + wait_for_multicast_members(zt2, &group_t2, &rear0)?; + + println_nopipe!("sextet multicast both external links passed"); + + t1.set_external_peers(&external_peers(&[QSFP0])).await?; + t2.set_external_peers(&external_peers(&[QSFP1])).await?; + assert_multicast_imports!( + t1, + [("tfportrear0_0", origin), ("tfportqsfp0_0", origin)] + ); + assert_multicast_imports!( + t2, + [("tfportrear0_0", origin), ("tfportqsfp1_0", origin)] + ); + wait_for_multicast_members(zt1, &group_t1, &t1_members)?; + wait_for_multicast_members(zt2, &group_t2, &rear0)?; + + println_nopipe!("sextet multicast partial link loss passed"); + + // + // With only s1 subscribed, t1 holds rear0 and t2 drains its rear0. + // + + s3.withdraw_multicast_groups(&vec![origin.clone()]).await?; + assert_multicast_imports!(t1, [("tfportrear0_0", origin)]); + assert_multicast_imports!(t2, [("tfportqsfp1_0", origin)]); + wait_for_multicast_members(zt2, &group_t2, &[])?; + + println_nopipe!("sextet multicast remote withdraw passed"); + + // + // Withdrawing the last subscriber drains both groups. The groups remain + // in DPD for Nexus to delete, and the external peers are cleared for any + // test that runs after this one. + // + + s1.withdraw_multicast_groups(&vec![origin]).await?; + assert_multicast_imports!(t1, []); + assert_multicast_imports!(t2, []); + wait_for_multicast_members(zt1, &group_t1, &t1_external)?; + wait_for_multicast_members(zt2, &group_t2, &[])?; + + t1.set_external_peers(&external_peers(&[])).await?; + t2.set_external_peers(&external_peers(&[])).await?; + + println_nopipe!("sextet multicast withdraw passed"); + Ok(()) } @@ -1802,6 +2494,18 @@ async fn prefix_count(c: &Client) -> Result { .sum::()) } +async fn peer_count(c: &Client) -> Result { + Ok(c.get_peers().await?.len()) +} + +async fn exchange_peer_count(c: &Client) -> Result { + Ok(c.get_peers() + .await? + .values() + .filter(|peer| matches!(peer.status, PeerStatus::Exchange(_))) + .count()) +} + async fn tunnel_endpoint_count(c: &Client) -> Result { Ok(c.get_tunnel_endpoints().await?.len()) } @@ -1810,12 +2514,104 @@ async fn tunnel_originated_endpoint_count(c: &Client) -> Result { Ok(c.get_originated_tunnel_endpoints().await?.len()) } -async fn multicast_group_count(c: &Client) -> Result { - Ok(c.get_multicast_groups().await?.len()) +async fn multicast_imports(c: &Client) -> Result { + Ok(c.get_multicast_groups() + .await? + .iter() + .map(|imported| { + ( + imported.if_name.clone(), + imported.route.origin.clone(), + imported.route.origin.metric(), + ) + }) + .collect()) } -async fn multicast_originated_count(c: &Client) -> Result { - Ok(c.get_originated_multicast_groups().await?.len()) +async fn multicast_origins( + c: &Client, +) -> Result> { + Ok(c.get_originated_multicast_groups() + .await? + .iter() + .map(|origin| (origin.clone(), origin.metric())) + .collect()) +} + +/// Wait for `client`'s multicast imports to equal what's `expected`. +/// +/// Unlike [`assert_multicast_imports!`], this yields between reads, which means +/// it can run alongside another future in the same task. +async fn wait_for_imports( + name: &str, + client: &Client, + expected: &MulticastImports, + timeout: Duration, +) -> Result<()> { + tokio::time::timeout(timeout, async { + while multicast_imports(client).await? != *expected { + tokio::time::sleep(FAILOVER_POLL_INTERVAL).await; + } + Ok(()) + }) + .await + .map_err(|_| anyhow!("{name} imports did not converge to {expected:#?}"))? +} + +/// Fail if any read of `client`'s multicast imports differs from `expected`. +/// +/// This signals `ready` after the 1st matching read, and then keeps sampling +/// until `done` arrives. +async fn observe_imports_unchanged( + name: &str, + client: &Client, + expected: &MulticastImports, + ready: oneshot::Sender<()>, + mut done: oneshot::Receiver<()>, +) -> Result<()> { + let mut ready = Some(ready); + loop { + let imports = multicast_imports(client).await?; + if imports != *expected { + dump_client_state(name, client).await; + return Err(anyhow!("{name} imports changed: {imports:#?}")); + } + if let Some(ready) = ready.take() { + ready + .send(()) + .map_err(|_| anyhow!("stopped before {name} was observed"))?; + } + match done.try_recv() { + Ok(()) => return Ok(()), + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Closed) => { + return Err(anyhow!("stopped before {name} observation ended")); + } + } + tokio::time::sleep(FAILOVER_POLL_INTERVAL).await; + } +} + +async fn dump_client_state(name: &str, c: &Client) { + let dump = async { + println_nopipe!("{name}:"); + println_nopipe!("peers: {:#?}", c.get_peers().await); + println_nopipe!("prefixes: {:#?}", c.get_prefixes().await); + println_nopipe!( + "originated multicast: {:#?}", + c.get_originated_multicast_groups().await + ); + println_nopipe!( + "imported multicast: {:#?}", + c.get_multicast_groups().await + ); + }; + if tokio::time::timeout(Duration::from_secs(10), dump) + .await + .is_err() + { + eprintln_nopipe!("{name}: diagnostic dump timed out"); + } } fn init_logger() -> Logger {