diff --git a/p4/sidecar-lite.p4 b/p4/sidecar-lite.p4 index 0da4a93..1f90b46 100644 --- a/p4/sidecar-lite.p4 +++ b/p4/sidecar-lite.p4 @@ -498,7 +498,10 @@ control local( } action nonlocal() { is_local = false; } - action local() { is_local = true; } + action local(bit<8> rid) { + is_local = true; + ingress.router_id = rid; + } } control attached( @@ -619,7 +622,10 @@ control router_v4_idx( Checksum() csum; table rtr { - key = { dst_addr: lpm; } + key = { + ingress.router_id: exact; + dst_addr: lpm; + } actions = { drop; index; } default_action = drop; } @@ -677,7 +683,10 @@ control router_v6_idx( Checksum() csum; table rtr { - key = { dst_addr: lpm; } + key = { + ingress.router_id: exact; + dst_addr: lpm; + } actions = { drop; index; } default_action = drop; } diff --git a/p4/softnpu.p4 b/p4/softnpu.p4 index b0211ba..0f8a4ac 100644 --- a/p4/softnpu.p4 +++ b/p4/softnpu.p4 @@ -9,6 +9,7 @@ struct ingress_metadata_t { bit<16> port; bit<16> nat_id; bit<16> path_idx; + bit<8> router_id; bool forward_needed; bool lldp; bool allow_source_mcast; diff --git a/scadm/src/main.rs b/scadm/src/main.rs index 4a019f0..6f4f5ff 100644 --- a/scadm/src/main.rs +++ b/scadm/src/main.rs @@ -51,6 +51,10 @@ enum Commands { /// Next Hop nexthop: Ipv4Addr, + + /// Router id to which it belongs. + #[clap(long, default_value_t = 0)] + rid: u8, }, /// Remove a route from the routing table. @@ -60,6 +64,10 @@ enum Commands { /// Subnet mask for the destination. mask: u8, + + /// Router id to which it belongs. + #[clap(long, default_value_t = 0)] + rid: u8, }, /// Add an IPv6 route to the routing table. @@ -75,6 +83,10 @@ enum Commands { /// Next Hop nexthop: Ipv6Addr, + + /// Router id to which it belongs. + #[clap(long, default_value_t = 0)] + rid: u8, }, /// Remove a route from the routing table. @@ -84,12 +96,20 @@ enum Commands { /// Subnet mask for the destination. mask: u8, + + /// Router id to which it belongs. + #[clap(long, default_value_t = 0)] + rid: u8, }, /// Add an IPv6 address to the router. AddAddress6 { /// Address to add. address: Ipv6Addr, + + /// Router id claiming the address. + #[clap(long, default_value_t = 0)] + rid: u8, }, /// Remove an IPv6 address from the router. @@ -389,7 +409,7 @@ enum RouteType { } struct RouteData { - data: BTreeMap, + data: BTreeMap<(u8, Cidr), u16>, } impl RouteData { @@ -397,22 +417,22 @@ impl RouteData { fn extract_route_idx( typ: RouteType, full: &BTreeMap>, - ) -> BTreeMap { + ) -> BTreeMap<(u8, Cidr), u16> { let mut data = BTreeMap::new(); let table = match typ { RouteType::V4 => ROUTER_V4_IDX, RouteType::V6 => ROUTER_V6_IDX, }; for e in full.get(table).unwrap() { - let subnet = match get_addr_subnet(&e.keyset_data) { - Some((address, mask)) => Cidr { address, mask }, + let (rid, subnet) = match get_rid_addr_subnet(&e.keyset_data) { + Some((rid, address, mask)) => (rid, Cidr { address, mask }), _ => continue, }; let idx = match get_idx_slots(&e.parameter_data) { Some((idx, _slots)) => idx, None => continue, }; - data.insert(subnet, idx); + data.insert((rid, subnet), idx); } data } @@ -443,9 +463,9 @@ impl RouteData { } // Given a route destination, find the corresponding table index - pub fn lookup(&self, address: IpAddr, mask: u8) -> Option { + pub fn lookup(&self, rid: u8, address: IpAddr, mask: u8) -> Option { let cidr = Cidr { address, mask }; - self.data.get(&cidr).copied() + self.data.get(&(rid, cidr)).copied() } // Find an open slot in the route table @@ -469,6 +489,7 @@ async fn main() { mask, port, nexthop, + rid, } => { let table = RouteData::load(RouteType::V4, &cli).await; let idx = table.find_available(); @@ -492,8 +513,9 @@ async fn main() { ) .await; - // Now add cidr->idx to the index table - let mut keyset_data: Vec = destination.octets().into(); + // Now add (rid, cidr)->idx to the index table + let mut keyset_data: Vec = vec![rid]; + keyset_data.extend_from_slice(&destination.octets()); keyset_data.push(mask); // Hardcoded slot count. @@ -511,11 +533,17 @@ async fn main() { ) .await; } - Commands::RemoveRoute4 { destination, mask } => { + Commands::RemoveRoute4 { + destination, + mask, + rid, + } => { let table = RouteData::load(RouteType::V4, &cli).await; - if let Some(idx) = table.lookup(IpAddr::V4(destination), mask) { - // Remove the entry from the subnet->idx table - let mut keyset_data: Vec = destination.octets().into(); + if let Some(idx) = table.lookup(rid, IpAddr::V4(destination), mask) + { + // Remove the entry from the (rid, subnet)->idx table + let mut keyset_data: Vec = vec![rid]; + keyset_data.extend_from_slice(&destination.octets()); keyset_data.push(mask); send( @@ -546,6 +574,7 @@ async fn main() { mask, port, nexthop, + rid, } => { let table = RouteData::load(RouteType::V6, &cli).await; let idx = table.find_available(); @@ -569,8 +598,9 @@ async fn main() { ) .await; - // Now add cidr->idx to the index table - let mut keyset_data: Vec = destination.octets().into(); + // Now add (rid, cidr)->idx to the index table + let mut keyset_data: Vec = vec![rid]; + keyset_data.extend_from_slice(&destination.octets()); keyset_data.push(mask); // Hardcoded slot count. @@ -588,11 +618,17 @@ async fn main() { ) .await; } - Commands::RemoveRoute6 { destination, mask } => { + Commands::RemoveRoute6 { + destination, + mask, + rid, + } => { let table = RouteData::load(RouteType::V6, &cli).await; - if let Some(idx) = table.lookup(IpAddr::V6(destination), mask) { - // Remove the entry from the subnet->idx table - let mut keyset_data: Vec = destination.octets().into(); + if let Some(idx) = table.lookup(rid, IpAddr::V6(destination), mask) + { + // Remove the entry from the (rid, subnet)->idx table + let mut keyset_data: Vec = vec![rid]; + keyset_data.extend_from_slice(&destination.octets()); keyset_data.push(mask); send( @@ -733,7 +769,9 @@ async fn main() { table: LOCAL_V4.into(), action: "local".into(), keyset_data, - ..Default::default() + // IPv4 addresses are never tunnel endpoints, so they + // always use the default router 0. + parameter_data: vec![0], }), &cli, ) @@ -752,7 +790,7 @@ async fn main() { .await; } - Commands::AddAddress6 { address } => { + Commands::AddAddress6 { address, rid } => { let mut keyset_data: Vec = address.octets().into(); keyset_data.reverse(); send( @@ -760,7 +798,7 @@ async fn main() { table: LOCAL_V6.into(), action: "local".into(), keyset_data, - ..Default::default() + parameter_data: vec![rid], }), &cli, ) @@ -1410,7 +1448,12 @@ fn dump_tables(table: &BTreeMap>) { println!("local v6:"); for e in table.get(LOCAL_V6).unwrap() { if let Some(a) = get_addr(&e.keyset_data, false) { - println!("{a}") + let rid = e + .parameter_data + .first() + .map(|r| r.to_string()) + .unwrap_or_else(|| "?".into()); + println!("{a} (rid {rid})") } } println!("local v4:"); @@ -1422,8 +1465,8 @@ fn dump_tables(table: &BTreeMap>) { println!("router v6_idx:"); for e in table.get(ROUTER_V6_IDX).unwrap() { - let tgt = match get_addr_subnet(&e.keyset_data) { - Some((a, m)) => format!("{a}/{m}"), + let tgt = match get_rid_addr_subnet(&e.keyset_data) { + Some((rid, a, m)) => format!("rid {rid}: {a}/{m}"), None => "?".into(), }; let idx = match get_idx_slots(&e.parameter_data) { @@ -1457,8 +1500,8 @@ fn dump_tables(table: &BTreeMap>) { println!("router v4_idx:"); for e in table.get(ROUTER_V4_IDX).unwrap() { - let tgt = match get_addr_subnet(&e.keyset_data) { - Some((a, m)) => format!("{a}/{m}"), + let tgt = match get_rid_addr_subnet(&e.keyset_data) { + Some((rid, a, m)) => format!("rid {rid}: {a}/{m}"), None => "?".into(), }; let idx = match get_idx_slots(&e.parameter_data) { @@ -1839,6 +1882,17 @@ fn get_addr_subnet(data: &[u8]) -> Option<(IpAddr, u8)> { } } +fn get_rid_addr_subnet(data: &[u8]) -> Option<(u8, IpAddr, u8)> { + match data.len() { + 6 => Some((data[0], get_addr(&data[1..5], true)?, data[5])), + 18 => Some((data[0], get_addr(&data[1..17], true)?, data[17])), + _ => { + println!("expected [rid, address, subnet], found: {data:x?}"); + None + } + } +} + fn get_addr_vni_mac(data: &[u8]) -> Option<(IpAddr, u32, [u8; 6])> { match data.len() { 13 => Some(( diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 41abb6b..732d2b5 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -278,7 +278,7 @@ fn pipeline_init(pipeline: &mut main_pipeline) { ); // boundary services loopback ip entry - let (key_buf, param_buf) = local6_entry("fd00:99::1"); + let (key_buf, param_buf) = local6_entry(0, "fd00:99::1"); pipeline.add_ingress_local_local_v6_entry("local", &key_buf, ¶m_buf, 0); // resolver entry for upstream gateway @@ -959,7 +959,7 @@ fn ipv6_ttl1_dropped() -> Result<(), anyhow::Error> { Ok(()) } -fn boundary_services_v6(inner_dst: &str, hop_limit: u8) -> Vec { +fn boundary_services_v6(tep: &str, inner_dst: &str, hop_limit: u8) -> Vec { let inner_ip = v6_probe("fd00:2::1", inner_dst, hop_limit); let mut eth_data = vec![0u8; 14 + inner_ip.len()]; let mut eth = MutableEthernetPacket::new(&mut eth_data).unwrap(); @@ -967,7 +967,7 @@ fn boundary_services_v6(inner_dst: &str, hop_limit: u8) -> Vec { eth.set_source(MacAddr::new(0x33, 0x33, 0x33, 0x44, 0x44, 0x44)); eth.set_ethertype(EtherType(ETHERTYPE_IPV6)); eth.set_payload(&inner_ip); - wrap_geneve("fd00:1::1", "fd00:99::1", ð_data, None) + wrap_geneve("fd00:1::1", tep, ð_data, None) } #[test] @@ -981,7 +981,7 @@ fn boundary_services_hop_limit0_to_scrimlet() -> Result<(), anyhow::Error> { npu.run(); - let ip_data = boundary_services_v6("fd00:1::5", 0); + let ip_data = boundary_services_v6("fd00:99::1", "fd00:1::5", 0); phy1.send(&[TxFrame::new( l2_dst(phy0.mac, ETHERTYPE_IPV6, &ip_data), ETHERTYPE_IPV6, @@ -1033,7 +1033,7 @@ fn boundary_services_hop_limit0_nonzero_port_dropped( npu.run(); - let ip_data = boundary_services_v6("fd00:3::99", 0); + let ip_data = boundary_services_v6("fd00:99::1", "fd00:3::99", 0); phy0.send(&[TxFrame::new( l2_dst(phy1.mac, ETHERTYPE_IPV6, &ip_data), ETHERTYPE_IPV6, @@ -3451,9 +3451,21 @@ fn router_idx_entry( idx: u16, slots: u8, ) -> (Vec, Vec) { - let mut key_buf = match dst.parse().unwrap() { - IpAddr::V4(a) => a.octets().to_vec(), - IpAddr::V6(a) => a.octets().to_vec(), + router_idx_entry_rid(0, dst, prefix_len, idx, slots) +} + +// Create an entry for the multipath cidr -> index table of a specific router +fn router_idx_entry_rid( + rid: u8, + dst: &str, + prefix_len: u8, + idx: u16, + slots: u8, +) -> (Vec, Vec) { + let mut key_buf = vec![rid]; + match dst.parse().unwrap() { + IpAddr::V4(a) => key_buf.extend_from_slice(&a.octets()), + IpAddr::V6(a) => key_buf.extend_from_slice(&a.octets()), }; key_buf.push(prefix_len); @@ -3615,12 +3627,12 @@ fn ipv6_mcast_l3(src: &str, dst: &str) -> Vec { ip_data } -fn local6_entry(addr: &str) -> (Vec, Vec) { +fn local6_entry(rid: u8, addr: &str) -> (Vec, Vec) { let addr: Ipv6Addr = addr.parse().unwrap(); let mut key_buf = addr.octets().to_vec(); key_buf.reverse(); - (key_buf, Vec::new()) + (key_buf, vec![rid]) } fn resolver4_entry(addr: &str, mac: [u8; 6]) -> (Vec, Vec) { @@ -3925,3 +3937,120 @@ fn geneve_mcast_v6_inner_pkt( let eth = inner_mcast_v6_frame(inner_src, inner_dst); wrap_geneve(outer_src, outer_dst, ð, None) } + +// Build a geneve-encapped ipv4 packet destined to the given TEP address, +// returning the outer ipv6 packet bytes. +fn geneve_encapped_v4(tep: &str, inner_dst: &str) -> Vec { + let mut ip_data = vec![0u8; 28]; + let mut ip = MutableIpv4Packet::new(&mut ip_data).unwrap(); + ip.set_version(4); + ip.set_header_length(5); + ip.set_total_length(28); + ip.set_source("1.2.3.4".parse().unwrap()); + ip.set_destination(inner_dst.parse().unwrap()); + ip.set_next_level_protocol(IpNextHeaderProtocol::new(17)); + ip.set_ttl(64); + + let mut eth_data = vec![0u8; 14 + ip_data.len()]; + let mut eth = MutableEthernetPacket::new(&mut eth_data).unwrap(); + eth.set_destination(MacAddr::new(0x11, 0x11, 0x11, 0x22, 0x22, 0x22)); + eth.set_source(MacAddr::new(0x33, 0x33, 0x33, 0x44, 0x44, 0x44)); + eth.set_ethertype(EtherType(ETHERTYPE_IPV4)); + eth.set_payload(&ip_data); + wrap_geneve("fd00:1::1", tep, ð_data, None) +} + +#[test] +fn endpoint_selects_route_table() -> Result<(), anyhow::Error> { + let mut pipeline = main_pipeline::new(3); + + // router 1: endpoint fd00:99::2, 8.8.8.0/24 and 2001:db8::/64 out port 1 + let (key_buf, param_buf) = local6_entry(1, "fd00:99::2"); + pipeline.add_ingress_local_local_v6_entry("local", &key_buf, ¶m_buf, 0); + let (key_buf, param_buf) = router_idx_entry_rid(1, "8.8.8.0", 24, 4, 1); + pipeline + .add_ingress_router_v4_idx_rtr_entry("index", &key_buf, ¶m_buf, 0); + let (key_buf, param_buf) = router_forward_entry(4, "1.2.3.1", 1); + pipeline.add_ingress_router_v4_route_rtr_entry( + "forward", &key_buf, ¶m_buf, 0, + ); + let (key_buf, param_buf) = router_idx_entry_rid(1, "2001:db8::", 64, 4, 1); + pipeline + .add_ingress_router_v6_idx_rtr_entry("index", &key_buf, ¶m_buf, 0); + let (key_buf, param_buf) = router_forward_entry(4, "fe80::1", 1); + pipeline.add_ingress_router_v6_route_rtr_entry( + "forward", &key_buf, ¶m_buf, 0, + ); + + // router 2: endpoint fd00:99::3, 0.0.0.0/0 and ::/0 out port 2 + let (key_buf, param_buf) = local6_entry(2, "fd00:99::3"); + pipeline.add_ingress_local_local_v6_entry("local", &key_buf, ¶m_buf, 0); + let (key_buf, param_buf) = router_idx_entry_rid(2, "0.0.0.0", 0, 5, 1); + pipeline + .add_ingress_router_v4_idx_rtr_entry("index", &key_buf, ¶m_buf, 0); + let (key_buf, param_buf) = router_forward_entry(5, "1.2.3.1", 2); + pipeline.add_ingress_router_v4_route_rtr_entry( + "forward", &key_buf, ¶m_buf, 0, + ); + let (key_buf, param_buf) = router_idx_entry_rid(2, "::", 0, 5, 1); + pipeline + .add_ingress_router_v6_idx_rtr_entry("index", &key_buf, ¶m_buf, 0); + let (key_buf, param_buf) = router_forward_entry(5, "fe80::1", 2); + pipeline.add_ingress_router_v6_route_rtr_entry( + "forward", &key_buf, ¶m_buf, 0, + ); + + let (key_buf, param_buf) = + resolver4_entry("1.2.3.1", [8, 9, 10, 11, 12, 13]); + pipeline.add_ingress_resolver_resolver_v4_entry( + "rewrite_dst", + &key_buf, + ¶m_buf, + 0, + ); + let (key_buf, param_buf) = + resolver6_entry("fe80::1", [20, 21, 22, 23, 24, 25]); + pipeline.add_ingress_resolver_resolver_v6_entry( + "rewrite_dst", + &key_buf, + ¶m_buf, + 0, + ); + + let mut npu = SoftNpu::new(3, pipeline, false); + let phy0 = npu.phy(0); + let phy1 = npu.phy(1); + let phy2 = npu.phy(2); + + npu.run(); + + // 8.8.8.8 through router 1 should come out phy1 + let pkt = geneve_encapped_v4("fd00:99::2", "8.8.8.8"); + phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV6, &pkt)])?; + wait_for(|| phy1.recv_buffer_len() > 0, "router 1 frame on port 1"); + let fs = phy1.recv(); + assert_eq!(fs.len(), 1); + + // 8.8.8.8 through router 2 should come out phy2 + let pkt = geneve_encapped_v4("fd00:99::3", "8.8.8.8"); + phy0.send(&[TxFrame::new(phy2.mac, ETHERTYPE_IPV6, &pkt)])?; + wait_for(|| phy2.recv_buffer_len() > 0, "router 2 frame on port 2"); + let fs = phy2.recv(); + assert_eq!(fs.len(), 1); + + // 2001:db8::1 through router 1 should come out phy1 + let pkt = boundary_services_v6("fd00:99::2", "2001:db8::1", 64); + phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV6, &pkt)])?; + wait_for(|| phy1.recv_buffer_len() > 0, "router 1 v6 frame on port 1"); + let fs = phy1.recv(); + assert_eq!(fs.len(), 1); + + // 2001:db8::1 through router 2 should come out phy2 + let pkt = boundary_services_v6("fd00:99::3", "2001:db8::1", 64); + phy0.send(&[TxFrame::new(phy2.mac, ETHERTYPE_IPV6, &pkt)])?; + wait_for(|| phy2.recv_buffer_len() > 0, "router 2 v6 frame on port 2"); + let fs = phy2.recv(); + assert_eq!(fs.len(), 1); + + Ok(()) +}