From 0bf3081518e570bbf0bbc0f6ec1a534ed5b739d8 Mon Sep 17 00:00:00 2001 From: Trey Aspelund Date: Fri, 30 Oct 2026 11:37:41 -0600 Subject: [PATCH 1/8] p4: implement SC_ARP_NEEDED / SC_NEIGHBOR_NEEDED Adds new request_{arp,ndp} actions to control resolver, which are now the default actions for the resolver_v{4,6} tables. Runs the unicast reflection check before the resolver, so a packet routed back out its ingress port is dropped rather than punted. Moves the mac rewrite for packets from userspace into control ingress after fwd_from_scrimlet(), and skips the mac rewrite when a resolver miss punts the packet to the scrimlet. This brings sidecar-lite behavior into alignment with sidecar.p4. Adds unit tests that verify routed IPv4/IPv6 packets whose nexthops are unresolved will trigger a sidecar packet to the scrimlet with the proper sc_code, sc_egress, and sc_payload (nexthop ip) values, that reflected packets are dropped on a resolver miss, and that a punted packet keeps its source MAC until userspace sends it back. Fixes: #25 Signed-off-by: Trey Aspelund --- p4/sidecar-lite.p4 | 51 +++++++++-- softnpu/src/test.rs | 214 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 249 insertions(+), 16 deletions(-) diff --git a/p4/sidecar-lite.p4 b/p4/sidecar-lite.p4 index 0da4a93..8db29c8 100644 --- a/p4/sidecar-lite.p4 +++ b/p4/sidecar-lite.p4 @@ -31,7 +31,11 @@ control ingress( // Check if this is a packet coming from the scrimlet. If so just // forward it out the source port indicated by the sidecar header. // - if (hdr.sidecar.isValid()) { fwd_from_scrimlet(); return; } + if (hdr.sidecar.isValid()) { + fwd_from_scrimlet(); + mac.apply(hdr, egress); + return; + } /// /// Check if we need to proxy arp this packet. @@ -182,13 +186,17 @@ control ingress( } } else { router.apply(hdr, ingress, egress); - resolver.apply(hdr, egress); - mac.apply(hdr, egress); - // Prevent reflection. Multicast ingress-port suppression is // handled in the replication bitmap, so this applies to // unicast only. - if (ingress.port == egress.port) { egress.drop = true; } + if (ingress.port == egress.port) { egress.drop = true; } else { + resolver.apply(hdr, ingress, egress); + // Packets sent to scrimlet don't need their SMAC rewritten. + // If a resolver miss punts the packet and it gets retransmitted + // by tfportd (upon successful ARP/ND resolution), the SMAC is + // rewritten after fwd_from_scrimlet(). + if (egress.port != 16w0) { mac.apply(hdr, egress); } + } } } @@ -539,18 +547,19 @@ control attached( control resolver( inout headers_t hdr, + in ingress_metadata_t ingress, inout egress_metadata_t egress, ) { table resolver_v4 { key = { egress.nexthop_v4: exact; } - actions = { rewrite_dst; drop; } - default_action = drop; + actions = { request_arp; rewrite_dst; drop; } + default_action = request_arp; } table resolver_v6 { key = { egress.nexthop_v6: exact; } - actions = { rewrite_dst; drop; } - default_action = drop; + actions = { request_ndp; rewrite_dst; drop; } + default_action = request_ndp; } apply { @@ -558,7 +567,31 @@ control resolver( if (egress.nexthop_v6 != 128w0) { resolver_v6.apply(); } } + action request_arp() { + hdr.sidecar.setValid(); + hdr.sidecar.sc_ether_type = hdr.ethernet.ether_type; + hdr.ethernet.ether_type = 16w0x0901; // ETHERTYPE_SIDECAR + hdr.sidecar.sc_code = 8w0x03; // SC_ARP_NEEDED + hdr.sidecar.sc_ingress = ingress.port; + hdr.sidecar.sc_egress = egress.port; + hdr.sidecar.sc_payload = 128w0; + hdr.sidecar.sc_payload[31:0] = egress.nexthop_v4; + egress.port = 16w0; // USER_SPACE_SERVICE_PORT + } + + action request_ndp() { + hdr.sidecar.setValid(); + hdr.sidecar.sc_ether_type = hdr.ethernet.ether_type; + hdr.ethernet.ether_type = 16w0x0901; // ETHERTYPE_SIDECAR + hdr.sidecar.sc_code = 8w0x04; // SC_NEIGHBOR_NEEDED + hdr.sidecar.sc_ingress = ingress.port; + hdr.sidecar.sc_egress = egress.port; + hdr.sidecar.sc_payload = egress.nexthop_v6; + egress.port = 16w0; // USER_SPACE_SERVICE_PORT + } + action rewrite_dst(bit<48> dst) { hdr.ethernet.dst = dst; } + action drop() { egress.drop = true; } } diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 41abb6b..0649f21 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -27,6 +27,11 @@ const ETHERTYPE_SIDECAR: u16 = 0x0901; const GENEVE_UDP_PORT: u16 = 6081; const GENEVE_PROTO_ETH: u16 = 0x6558; const OXG_OPTION_CLASS: u16 = 0x0129; +const SC_FWD_FROM_USERSPACE: u8 = 0x00; +const SC_ARP_NEEDED: u8 = 0x0003; +const SC_NEIGHBOR_NEEDED: u8 = 0x0004; +// this is defined as bit<9> in sidecar.p4, so round up to u16 +const USER_SPACE_SERVICE_PORT: u16 = 0x0000; // Multicast tag values matching Dendrite's MULTICAST_TAG_* constants. const MCAST_TAG_EXTERNAL: u8 = 0; @@ -236,30 +241,55 @@ pub struct OxgMcastOpt { payload: Vec, } -fn pipeline_init(pipeline: &mut main_pipeline) { +// Sidecar frame format (following headers.p4) +#[allow(dead_code)] +#[derive(PktDerive)] +pub struct Sidecar { + sc_code: u8, + sc_pad: u8, + sc_ingress: u16be, + sc_egress: u16be, + sc_ether_type: u16be, + #[length = "16"] + sc_payload: Vec, + #[payload] + payload: Vec, // the packet inside the sidecar header +} + +fn router_pipeline_init( + pipeline: &mut main_pipeline, + v4_nh: &str, + v4_nh_port: u16, + v6_nh: &str, + v6_nh_port: u16, +) { // router entry upstream // Add a single path for 0.0.0.0/0 pointing at data in slot 2. let (key_buf, param_buf) = router_idx_entry("0.0.0.0", 0, 2, 1); pipeline .add_ingress_router_v4_idx_rtr_entry("index", &key_buf, ¶m_buf, 0); - // At slot 2, add a forwarding entry gw=1.2.3.1, port=1 - let (key_buf, param_buf) = router_forward_entry(2, "1.2.3.1", 1); + // At slot 2, add a forwarding entry (gw=v4_nh, port=v4_nh_port) + let (key_buf, param_buf) = router_forward_entry(2, v4_nh, v4_nh_port); pipeline.add_ingress_router_v4_route_rtr_entry( "forward", &key_buf, ¶m_buf, 0, ); // router entry downstream - // Add a single path for fd00::1 pointing at data in slot 2. + // Add a single path for fd00::1/64 pointing at data in slot 2. let (key_buf, param_buf) = router_idx_entry("fd00:1::", 64, 2, 1); pipeline .add_ingress_router_v6_idx_rtr_entry("index", &key_buf, ¶m_buf, 0); - // At slot 2, add a forwarding entry gw=fe80::1 port=0 - let (key_buf, param_buf) = router_forward_entry(2, "fe80::1", 0); + // At slot 2, add a forwarding entry (gw=v6_nh port=v6_nh_port) + let (key_buf, param_buf) = router_forward_entry(2, v6_nh, v6_nh_port); pipeline.add_ingress_router_v6_route_rtr_entry( "forward", &key_buf, ¶m_buf, 0, ); +} + +fn pipeline_init(pipeline: &mut main_pipeline) { + router_pipeline_init(pipeline, "1.2.3.1", 1, "fe80::1", 0); // nat entry let (key_buf, param_buf) = nat4_entry( @@ -1252,12 +1282,16 @@ fn mcast_ttl_pipeline() -> main_pipeline { } fn mcast_v4_probe(dst: &str, ttl: u8) -> Vec { + v4_probe("10.0.0.1", dst, ttl) +} + +fn v4_probe(src: &str, dst: &str, ttl: u8) -> Vec { let payload = [0u8; 8]; let mut ip_data: Vec = vec![0; 20 + payload.len()]; let mut ip = MutableIpv4Packet::new(&mut ip_data).unwrap(); ip.set_version(4); ip.set_header_length(5); - ip.set_source("10.0.0.1".parse().unwrap()); + ip.set_source(src.parse().unwrap()); ip.set_destination(dst.parse().unwrap()); ip.set_next_level_protocol(IpNextHeaderProtocol::new(17)); ip.set_total_length(20 + payload.len() as u16); @@ -3444,6 +3478,172 @@ fn mcast_suppression_drops_when_only_group_zeroed() -> Result<(), anyhow::Error> Ok(()) } +#[test] +fn resolver_miss() -> Result<(), anyhow::Error> { + const NEXTHOP4: &str = "1.2.3.1"; + const NEXTHOP6: &str = "fe80::1"; + const DST_PORT: u16 = 0x0001; // Routes installed via DST_PORT + const SRC_PORT: u16 = 0x0002; // Packets injected from SRC_PORT + + let mut pipeline = main_pipeline::new(1); + router_pipeline_init(&mut pipeline, NEXTHOP4, DST_PORT, NEXTHOP6, DST_PORT); + + let mut npu = SoftNpu::new(3, pipeline, false); + let phy0 = npu.phy(USER_SPACE_SERVICE_PORT.into()); // scrimlet port + let phy1 = npu.phy(DST_PORT.into()); + let phy2 = npu.phy(SRC_PORT.into()); + + npu.run(); + + // IPv4 setup + const DST4: &str = "100.64.0.1"; + const SRC4: &str = "10.0.0.1"; + let probe4 = v4_probe(SRC4, DST4, 255); + let probe4_desc = format!("IPv4 packet (dst={DST4}, src={SRC4}, ttl=255)"); + + // Send IPv4 probe from port 1 so scrimlet (port 0) can receive it. + phy2.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &probe4)])?; + + wait_for( + || phy0.recv_buffer_len() > 0, + &format!("scrimlet should receive {probe4_desc}"), + ); + let maybe_sidecar = phy0.recv().remove(0); + let maybe_arp_needed = + SidecarPacket::new(&maybe_sidecar.payload).expect("sidecar header"); + + assert_eq!( + maybe_arp_needed.get_sc_code(), + SC_ARP_NEEDED, + "{probe4_desc} should arrive with sidecar code SC_ARP_NEEDED" + ); + assert_eq!( + maybe_arp_needed.get_sc_egress(), + DST_PORT, + "{probe4_desc} should arrive with sidecar egress {DST_PORT}" + ); + assert_eq!( + Ipv4Addr::from_octets( + // for sc_code=SC_ARP_NEEDED, the lower 4 bytes are the IPv4 nexthop + maybe_arp_needed.get_sc_payload_raw()[12..16] + .try_into() + .unwrap() + ), + NEXTHOP4.parse::().unwrap(), + "{probe4_desc} should arrive with sidecar payload {NEXTHOP4}" + ); + + // IPv6 setup + const DST6: &str = "fd00:1::1"; + const SRC6: &str = "fd00:1::99"; + let probe6 = v6_probe(SRC6, DST6, 255); + let probe6_desc = + format!("IPv6 packet (dst={DST6}, src={SRC6}, hop-limit=255)"); + + // Send IPv6 probe from port 1 so scrimlet (port 0) can receive it. + phy2.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV6, &probe6)])?; + + wait_for( + || phy0.recv_buffer_len() > 0, + &format!("scrimlet should receive {probe6_desc}"), + ); + let maybe_sidecar = phy0.recv().remove(0); + let maybe_ndp_needed = + SidecarPacket::new(&maybe_sidecar.payload).expect("sidecar header"); + + assert_eq!( + maybe_ndp_needed.get_sc_code(), + SC_NEIGHBOR_NEEDED, + "{probe6_desc} should arrive with sidecar code SC_NEIGHBOR_NEEDED" + ); + assert_eq!( + maybe_ndp_needed.get_sc_egress(), + DST_PORT, + "{probe6_desc} should arrive with sidecar egress {DST_PORT}" + ); + assert_eq!( + Ipv6Addr::from_octets( + // for sc_code=SC_NEIGHBOR_NEEDED, the payload is the IPv6 nexthop + maybe_ndp_needed.get_sc_payload_raw().try_into().unwrap() + ), + NEXTHOP6.parse::().unwrap(), + "{probe6_desc} should arrive with sidecar payload {NEXTHOP6}" + ); + + Ok(()) +} + +// Route everything out port 1 via nexthops with no resolver entries, +// so every resolver lookup misses. +fn resolver_miss_pipeline() -> main_pipeline { + let mut pipeline = main_pipeline::new(3); + router_pipeline_init(&mut pipeline, "1.2.3.1", 1, "fe80::1", 1); + pipeline +} + +// A packet routed back out its ingress port is dropped regardless of the action +// that control resolver takes +#[test] +fn resolver_miss_reflection_dropped() -> Result<(), anyhow::Error> { + let mut pipeline = resolver_miss_pipeline(); + let sentinel_dst = sentinel_route_v4(&mut pipeline, 2); + + let mut npu = SoftNpu::new(3, pipeline, false); + let (phy0, phy1, phy2) = (npu.phy(0), npu.phy(1), npu.phy(2)); + npu.run(); + + let probe = v4_probe("10.0.0.1", "100.64.0.1", 255); + let sentinel = sentinel_v4_to(&sentinel_dst); + phy1.send(&[ + TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &probe), + TxFrame::new(phy2.mac, ETHERTYPE_IPV4, &sentinel), + ])?; + + // sentinel is sent last, so once it's present the probe should be too + recv_until_sentinel(&phy2, "sentinel on port 2"); + assert_eq!(phy0.recv_buffer_len(), 0, "probe punted to the scrimlet"); + assert_eq!(phy1.recv_buffer_len(), 0, "probe reflected out port 1"); + Ok(()) +} + +// A packet punted upon resolver miss keeps the sender's source MAC. When +// userspace sends it back, it gets the egress port's source MAC. +#[test] +fn resolver_miss_smac_rewrite() -> Result<(), anyhow::Error> { + let mut pipeline = resolver_miss_pipeline(); + // Give ports 0 and 1 source MACs 02:00:00:00:00:00 and 02:00:00:00:00:01 + // (mac_rewrite_entry takes the bytes in reverse order). Without an entry + // for port 0, rewriting a punted packet would be a no-op and go unnoticed. + for port in [0u16, 1] { + let (key, param) = mac_rewrite_entry(port, [port as u8, 0, 0, 0, 0, 2]); + pipeline.add_ingress_mac_mac_rewrite_entry("rewrite", &key, ¶m, 0); + } + + let mut npu = SoftNpu::new(3, pipeline, false); + let (phy0, phy1, phy2) = (npu.phy(0), npu.phy(1), npu.phy(2)); + npu.run(); + + let probe = v4_probe("10.0.0.1", "100.64.0.1", 255); + phy2.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &probe)])?; + wait_for(|| phy0.recv_buffer_len() > 0, "punted probe on port 0"); + let punted = phy0.recv().remove(0); + assert_eq!(punted.src, phy2.mac, "punted probe keeps the sender's SMAC"); + + // Send it back the way tfportd does once the nexthop resolves. + let mut frame = punted.payload; + MutableSidecarPacket::new(&mut frame) + .unwrap() + .set_sc_code(SC_FWD_FROM_USERSPACE); + phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &frame)])?; + wait_for( + || phy1.recv_buffer_len() > 0, + "retransmitted probe on port 1", + ); + let forwarded = phy1.recv().remove(0); + assert_eq!(forwarded.src, [2, 0, 0, 0, 0, 1], "port 1's SMAC"); + Ok(()) +} + // Create an entry for the multipath cidr -> index table fn router_idx_entry( dst: &str, From 8c7517497483f8511d1374a60709a852d94d46c0 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 4 Nov 2026 01:32:38 +0000 Subject: [PATCH 2/8] [multicast] restrict scrimlet multicast to IPv6 link-local `0bf3081` (parent) moved `mac.apply()` onto the from-scrimlet path, which would have given scrimlet-sent multicast the port MAC. We keep the userspace MACs for `ff02::/16` multicast traffic from the scrimlet, while dropping IPv4 multicast and other IPv6 multicast on this path. --- p4/sidecar-lite.p4 | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/p4/sidecar-lite.p4 b/p4/sidecar-lite.p4 index 8db29c8..f31b8cb 100644 --- a/p4/sidecar-lite.p4 +++ b/p4/sidecar-lite.p4 @@ -33,6 +33,21 @@ control ingress( // if (hdr.sidecar.isValid()) { fwd_from_scrimlet(); + // TODO: Combine these checks when x4c supports && and ||. + if (hdr.ipv4.isValid()) { + if (hdr.ipv4.dst[31:28] == 4w0xe) { + egress.drop = true; + return; + } + } + if (hdr.ipv6.isValid()) { + // Link-local multicast retains the userspace MACs. + if (hdr.ipv6.dst[127:112] == 16w0xff02) { return; } + if (hdr.ipv6.dst[127:120] == 8w0xff) { + egress.drop = true; + return; + } + } mac.apply(hdr, egress); return; } @@ -99,8 +114,8 @@ control ingress( } else { // Multicast with TTL/hop_limit <= 1 must not be forwarded // (RFC 1112, RFC 8200 section 3). The check lives here, not in - // the parser, because scrimlet transmissions return earlier - // through fwd_from_scrimlet and must not be dropped. + // the parser, because link-local scrimlet transmissions return + // earlier through fwd_from_scrimlet and must not be dropped. if (mcast_dst) { if (hdr.ipv4.isValid()) { if (hdr.ipv4.ttl <= 8w1) { egress.drop = true; return; } From 917fb1b80c10bfb99c01dd8e6941c36f6c77d93c Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Thu, 5 Nov 2026 01:22:57 +0000 Subject: [PATCH 3/8] [test] update tests --- softnpu/src/test.rs | 74 +++++++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 0649f21..9cadfc8 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -1417,6 +1417,12 @@ fn ipv6_mcast_ff02_hop_limit_zero_to_scrimlet() -> Result<(), anyhow::Error> { assert_v6_to_scrimlet(&probe, "ff02:: packet with hop_limit=0") } +#[test] +fn ipv6_mcast_ff02_hop_limit_255_to_scrimlet() -> Result<(), anyhow::Error> { + let probe = mcast_v6_probe("ff02::fb", 255); + assert_v6_to_scrimlet(&probe, "ff02::fb packet with hop_limit=255") +} + fn assert_v6_to_scrimlet( probe: &[u8], what: &str, @@ -1521,12 +1527,10 @@ fn scrimlet_mcast_copies( let frame = sidecar_wrap(1, ethertype, probe); phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &frame)])?; + let sentinel = sidecar_wrap(1, ETHERTYPE_IPV4, &sentinel_v4()); + phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &sentinel)])?; - wait_for( - || phy1.recv_buffer_len() > 0, - "scrimlet multicast on port 1", - ); - let frames = phy1.recv(); + let frames = recv_until_sentinel(&phy1, "scrimlet sentinel on port 1"); for f in &frames { assert_eq!(f.src, phy0.mac, "scrimlet src MAC not rewritten"); assert_eq!(f.dst, phy1.mac, "scrimlet dst MAC not rewritten"); @@ -1534,32 +1538,50 @@ fn scrimlet_mcast_copies( Ok(frames) } -// The scrimlet's own TTL-1 multicast (e.g., IGMP reports) is a -// transmission vs being a forwarding hop. The packet leaves unchanged, while -// the same packet from a front port is dropped. #[test] -fn scrimlet_ipv4_mcast_ttl1_forwarded() -> Result<(), anyhow::Error> { - let probe = mcast_v4_probe("238.1.1.1", 1); - let fs = scrimlet_mcast_copies(ETHERTYPE_IPV4, &probe)?; - assert_eq!(fs.len(), 1, "one copy out the sidecar egress port"); - assert_eq!(fs[0].ethertype, ETHERTYPE_IPV4, "sidecar header stripped"); - let ip = Ipv4Packet::new(&fs[0].payload).unwrap(); - assert_eq!(ip.get_ttl(), 1, "scrimlet transmission keeps its TTL"); +fn scrimlet_ipv4_mcast_dropped() -> Result<(), anyhow::Error> { + for ttl in [0, 1, 64, 255] { + let probe = mcast_v4_probe("238.1.1.1", ttl); + let fs = scrimlet_mcast_copies(ETHERTYPE_IPV4, &probe)?; + assert!( + fs.is_empty(), + "scrimlet IPv4 multicast at TTL {ttl} should be dropped" + ); + } Ok(()) } -// IPv6 counterpart of the above. -// -// The group is ff0e because the parser already admits link-local multicast -// at hop_limit 1 and should not exercise the exemption. #[test] -fn scrimlet_ipv6_mcast_hop_limit1_forwarded() -> Result<(), anyhow::Error> { - let probe = mcast_v6_probe("ff0e::1", 1); - let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; - assert_eq!(fs.len(), 1, "one copy out the sidecar egress port"); - assert_eq!(fs[0].ethertype, ETHERTYPE_IPV6, "sidecar header stripped"); - let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); - assert_eq!(ip.get_hop_limit(), 1, "scrimlet transmission keeps its hop"); +fn scrimlet_ipv6_mcast_dropped() -> Result<(), anyhow::Error> { + for hop_limit in [0, 1, 64, 255] { + let probe = mcast_v6_probe("ff0e::1", hop_limit); + let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; + assert!( + fs.is_empty(), + "scrimlet ff0e:: multicast at hop limit {hop_limit} should be \ + dropped" + ); + } + Ok(()) +} + +#[test] +fn scrimlet_ipv6_link_local_mcast_forwarded() -> Result<(), anyhow::Error> { + for hop_limit in [0, 1, 64, 255] { + let probe = mcast_v6_probe("ff02::1", hop_limit); + let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; + assert_eq!(fs.len(), 1, "one copy out the sidecar egress port"); + assert_eq!( + fs[0].ethertype, ETHERTYPE_IPV6, + "sidecar header stripped" + ); + let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); + assert_eq!( + ip.get_hop_limit(), + hop_limit, + "scrimlet transmission keeps its hop limit" + ); + } Ok(()) } From e0d5b8dc31588bececcf5f0d69e4ea0fbcde82e4 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Thu, 5 Nov 2026 03:15:20 +0000 Subject: [PATCH 4/8] [nit] fix --- softnpu/src/test.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 9cadfc8..36e3731 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -1526,9 +1526,11 @@ fn scrimlet_mcast_copies( npu.run(); let frame = sidecar_wrap(1, ethertype, probe); - phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &frame)])?; let sentinel = sidecar_wrap(1, ETHERTYPE_IPV4, &sentinel_v4()); - phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &sentinel)])?; + phy0.send(&[ + TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &frame), + TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &sentinel), + ])?; let frames = recv_until_sentinel(&phy1, "scrimlet sentinel on port 1"); for f in &frames { From 7e446ef605b2562c32b2f4a3c3a6be4f911e8d69 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Thu, 5 Nov 2026 03:15:20 +0000 Subject: [PATCH 5/8] [nit] fix --- softnpu/src/test.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 9cadfc8..8cbef6d 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -1526,9 +1526,11 @@ fn scrimlet_mcast_copies( npu.run(); let frame = sidecar_wrap(1, ethertype, probe); - phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &frame)])?; let sentinel = sidecar_wrap(1, ETHERTYPE_IPV4, &sentinel_v4()); - phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &sentinel)])?; + phy0.send(&[ + TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &frame), + TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &sentinel), + ])?; let frames = recv_until_sentinel(&phy1, "scrimlet sentinel on port 1"); for f in &frames { @@ -1571,10 +1573,7 @@ fn scrimlet_ipv6_link_local_mcast_forwarded() -> Result<(), anyhow::Error> { let probe = mcast_v6_probe("ff02::1", hop_limit); let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; assert_eq!(fs.len(), 1, "one copy out the sidecar egress port"); - assert_eq!( - fs[0].ethertype, ETHERTYPE_IPV6, - "sidecar header stripped" - ); + assert_eq!(fs[0].ethertype, ETHERTYPE_IPV6, "sidecar header stripped"); let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); assert_eq!( ip.get_hop_limit(), From a3bb85bbd303064bb13c27ee7a90b1ff24e5f74f Mon Sep 17 00:00:00 2001 From: Trey Aspelund Date: Thu, 5 Nov 2026 07:25:50 -0700 Subject: [PATCH 6/8] softnpu: update scrimlet_mcast_copies test comment Signed-off-by: Trey Aspelund --- softnpu/src/test.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 8cbef6d..58a3290 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -1500,9 +1500,9 @@ fn sidecar_wrap(egress: u16, ethertype: u16, l3: &[u8]) -> Vec { // Send `probe` from the scrimlet side (port 0) toward port 1 and collect what // exits port 1. // -// No replication group or NAT entry is installed: ingress returns early through -// fwd_from_scrimlet, which forwards out sc_egress without looking at the -// multicast tables. +// No replication group or NAT entry is installed: ingress returns early after +// fwd_from_scrimlet without looking at the multicast tables. IPv6 link-local +// multicast goes out sc_egress, and all other multicast is dropped. fn scrimlet_mcast_copies( ethertype: u16, probe: &[u8], From c9dd5013aa5375d58628f135454132af470c0ce6 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 10 Nov 2026 15:06:22 +0000 Subject: [PATCH 7/8] [mcast] switch-zone mcast goes out the mcast path Post Ry's review comments, e.g., https://github.com/oxidecomputer/sidecar-lite/pull/181#discussion_r4526014668, we shouldn't have a rigid policy here. Nonetheless, we have to take care of the sidecar header, which sidecar.p4 currently leaves on these packets. Includes: - scrimlet packets w/ `ff02` (IPv6 link-local mcast) destinations go out `sc_egress` w/ the userspace MACs (just like mainline) - other scrimlet mcast destinations get their sidecar header stripped (`decap_sidecar`), and then the usual DMAC/TTL checks, NAT + replication - They are dropped on a DMAC mismatch, TTL <= 1, or no group - the scrimlet worked below the DMAC check (`ff02` gets it too then) - Note: unicast unchanged --- p4/sidecar-lite.p4 | 85 +++++++++-------- softnpu/src/test.rs | 216 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 230 insertions(+), 71 deletions(-) diff --git a/p4/sidecar-lite.p4 b/p4/sidecar-lite.p4 index f31b8cb..74ddc5d 100644 --- a/p4/sidecar-lite.p4 +++ b/p4/sidecar-lite.p4 @@ -27,40 +27,6 @@ control ingress( ipv4_ttl_decrement() ttl_dec; apply { - // - // Check if this is a packet coming from the scrimlet. If so just - // forward it out the source port indicated by the sidecar header. - // - if (hdr.sidecar.isValid()) { - fwd_from_scrimlet(); - // TODO: Combine these checks when x4c supports && and ||. - if (hdr.ipv4.isValid()) { - if (hdr.ipv4.dst[31:28] == 4w0xe) { - egress.drop = true; - return; - } - } - if (hdr.ipv6.isValid()) { - // Link-local multicast retains the userspace MACs. - if (hdr.ipv6.dst[127:112] == 16w0xff02) { return; } - if (hdr.ipv6.dst[127:120] == 8w0xff) { - egress.drop = true; - return; - } - } - mac.apply(hdr, egress); - return; - } - - /// - /// Check if we need to proxy arp this packet. - /// - if (hdr.arp.isValid()) { - bool proxied = false; - pxarp.apply(hdr, ingress, egress, proxied); - if (proxied) { return; } - } - bool mcast_dst = false; if (hdr.ipv6.isValid()) { if (hdr.ipv6.dst[127:120] == 8w0xff) { mcast_dst = true; } @@ -95,6 +61,43 @@ control ingress( } } + // + // Check if this is a packet coming from the scrimlet: + // + // - Unicast is forwarded out the port indicated by the sidecar header, + // with its source MAC rewritten for that specific port. + // + // - IPv6 link-local multicast is forwarded out the same port, keeping + // the MACs set by userspace. + // + // - Other multicast packets have the sidecar header removed and + // continue through to the multicast path below. + // + if (hdr.sidecar.isValid()) { + bool link_local_mcast = false; + if (hdr.ipv6.isValid()) { + if (hdr.ipv6.dst[127:112] == 16w0xff02) { + link_local_mcast = true; + } + } + if (link_local_mcast) { fwd_from_scrimlet(); return; } + if (mcast_dst == false) { + fwd_from_scrimlet(); + mac.apply(hdr, egress); + return; + } + decap_sidecar(); + } + + /// + /// Check if we need to proxy arp this packet. + /// + if (hdr.arp.isValid()) { + bool proxied = false; + pxarp.apply(hdr, ingress, egress, proxied); + if (proxied) { return; } + } + // // If the packet has a local destination, create the sidecar header and // send it to the scrimlet. @@ -115,7 +118,9 @@ control ingress( // Multicast with TTL/hop_limit <= 1 must not be forwarded // (RFC 1112, RFC 8200 section 3). The check lives here, not in // the parser, because link-local scrimlet transmissions return - // earlier through fwd_from_scrimlet and must not be dropped. + // earlier through fwd_from_scrimlet and must not be dropped. Other + // scrimlet multicast gets checked here, just like in dendrite's + // parser. if (mcast_dst) { if (hdr.ipv4.isValid()) { if (hdr.ipv4.ttl <= 8w1) { egress.drop = true; return; } @@ -143,8 +148,9 @@ control ingress( // whose default_action is drop. Each replica's destination and // source MACs are derived in the egress control instead. // - // IPv6 link-local multicast (ff02::/16) never reaches this point; it is - // marked local and handled by the earlier local path. The parser + // IPv6 link-local multicast (ff02::/16) never reaches this point. From + // a front port it is marked local and handled by the earlier local + // path; from the scrimlet it leaves through fwd_from_scrimlet. The parser // rejects interface-local scope multicast, while expired multicast // (TTL/hop_limit <= 1) gets dropped on the non-local path above. // @@ -299,7 +305,10 @@ control ingress( // Direct packets to the sidecar port corresponding to the scrimlet // port they came from. egress.port = hdr.sidecar.sc_egress; - // Decap the sidecar header. + decap_sidecar(); + } + + action decap_sidecar() { hdr.ethernet.ether_type = hdr.sidecar.sc_ether_type; hdr.sidecar.setInvalid(); } diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 58a3290..8bfae67 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -1500,9 +1500,10 @@ fn sidecar_wrap(egress: u16, ethertype: u16, l3: &[u8]) -> Vec { // Send `probe` from the scrimlet side (port 0) toward port 1 and collect what // exits port 1. // -// No replication group or NAT entry is installed: ingress returns early after -// fwd_from_scrimlet without looking at the multicast tables. IPv6 link-local -// multicast goes out sc_egress, and all other multicast is dropped. +// No replication group or NAT entry is installed. IPv6 link-local multicast +// exits through sc_egress, while all other multicast enters the multicast path +// and is dropped, by the TTL check at TTL <= 1 and otherwise on the empty +// bitmap. fn scrimlet_mcast_copies( ethertype: u16, probe: &[u8], @@ -1525,61 +1526,210 @@ fn scrimlet_mcast_copies( npu.run(); + let dst = l2_dst(phy1.mac, ethertype, probe); let frame = sidecar_wrap(1, ethertype, probe); let sentinel = sidecar_wrap(1, ETHERTYPE_IPV4, &sentinel_v4()); phy0.send(&[ - TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &frame), + TxFrame::new(dst, ETHERTYPE_SIDECAR, &frame), TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &sentinel), ])?; let frames = recv_until_sentinel(&phy1, "scrimlet sentinel on port 1"); for f in &frames { - assert_eq!(f.src, phy0.mac, "scrimlet src MAC not rewritten"); - assert_eq!(f.dst, phy1.mac, "scrimlet dst MAC not rewritten"); + assert_eq!(f.src, phy0.mac, "scrimlet src MAC kept"); + assert_eq!(f.dst, dst, "scrimlet dst MAC kept"); } Ok(frames) } #[test] -fn scrimlet_ipv4_mcast_dropped() -> Result<(), anyhow::Error> { - for ttl in [0, 1, 64, 255] { - let probe = mcast_v4_probe("238.1.1.1", ttl); - let fs = scrimlet_mcast_copies(ETHERTYPE_IPV4, &probe)?; - assert!( - fs.is_empty(), - "scrimlet IPv4 multicast at TTL {ttl} should be dropped" +fn scrimlet_ipv6_link_local_mcast_forwarded() -> Result<(), anyhow::Error> { + for hop_limit in [0, 1, 64, 255] { + let probe = mcast_v6_probe("ff02::1", hop_limit); + let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; + assert_eq!(fs.len(), 1, "one copy out sc_egress"); + assert_eq!(fs[0].ethertype, ETHERTYPE_IPV6, "sidecar header stripped"); + let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); + assert_eq!( + ip.get_hop_limit(), + hop_limit, + "scrimlet transmission keeps its hop limit" ); } Ok(()) } #[test] -fn scrimlet_ipv6_mcast_dropped() -> Result<(), anyhow::Error> { - for hop_limit in [0, 1, 64, 255] { - let probe = mcast_v6_probe("ff0e::1", hop_limit); +fn scrimlet_mcast_without_group_dropped() -> Result<(), anyhow::Error> { + for ttl in [64, 255] { + let probe = mcast_v4_probe("238.1.1.1", ttl); + let fs = scrimlet_mcast_copies(ETHERTYPE_IPV4, &probe)?; + assert!(fs.is_empty(), "238.1.1.1 at TTL {ttl} left sc_egress"); + + let probe = mcast_v6_probe("ff0e::1", ttl); let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; - assert!( - fs.is_empty(), - "scrimlet ff0e:: multicast at hop limit {hop_limit} should be \ - dropped" - ); + assert!(fs.is_empty(), "ff0e::1 at hop limit {ttl} left sc_egress"); } Ok(()) } #[test] -fn scrimlet_ipv6_link_local_mcast_forwarded() -> Result<(), anyhow::Error> { - for hop_limit in [0, 1, 64, 255] { - let probe = mcast_v6_probe("ff02::1", hop_limit); - let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; - assert_eq!(fs.len(), 1, "one copy out the sidecar egress port"); - assert_eq!(fs[0].ethertype, ETHERTYPE_IPV6, "sidecar header stripped"); - let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); - assert_eq!( - ip.get_hop_limit(), - hop_limit, - "scrimlet transmission keeps its hop limit" - ); +fn scrimlet_mcast_with_group_replicated() -> Result<(), anyhow::Error> { + let mut pipeline = mcast_ff0e_pipeline(&[1], &[2]); + let port2_sentinel = sentinel_route_v4(&mut pipeline, 2); + let mut src_mac = [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee]; + src_mac.reverse(); + pipeline.add_egress_mcast_src_mac_entry( + "rewrite_src_mac", + &2u16.to_le_bytes(), + &src_mac, + 0, + ); + let mut npu = SoftNpu::new(3, pipeline, false); + let phy0 = npu.phy(0); + let phy1 = npu.phy(1); + let phy2 = npu.phy(2); + + npu.run(); + + let ip_data = geneve_mcast_v4_pkt_repl( + "fd00:1::1", + "ff0e::1", + "10.0.0.1", + "238.1.1.1", + Some(MCAST_TAG_UNDERLAY), + ); + let frame = sidecar_wrap(0, ETHERTYPE_IPV6, &ip_data); + phy0.send(&[TxFrame::new( + l2_dst(phy1.mac, ETHERTYPE_IPV6, &ip_data), + ETHERTYPE_SIDECAR, + &frame, + )])?; + phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &sentinel_v4())])?; + phy0.send(&[TxFrame::new( + phy2.mac, + ETHERTYPE_IPV4, + &sentinel_v4_to(&port2_sentinel), + )])?; + + let port1 = recv_until_sentinel(&phy1, "port 1 sentinel"); + assert!( + port1.is_empty(), + "external copy suppressed by the underlay tag" + ); + + let port2 = recv_until_sentinel(&phy2, "port 2 sentinel"); + assert_eq!(port2.len(), 1, "one underlay copy on port 2"); + assert_eq!( + port2[0].ethertype, ETHERTYPE_IPV6, + "sidecar header stripped" + ); + assert_eq!( + port2[0].src, + [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee], + "port 2's SMAC" + ); + assert_eq!(port2[0].dst, [0x33, 0x33, 0, 0, 0, 1], "DMAC from ff0e::1"); + let ip = Ipv6Packet::new(&port2[0].payload).unwrap(); + assert_eq!( + ip.get_destination(), + "ff0e::1".parse::().unwrap(), + "copy keeps its encapsulation" + ); + assert_eq!(ip.get_hop_limit(), 63, "outer hop limit decremented"); + assert_eq!(phy0.recv_buffer_len(), 0, "sc_egress (port 0) ignored"); + Ok(()) +} + +#[test] +fn scrimlet_mcast_nat_replicated() -> Result<(), anyhow::Error> { + let mut pipeline = mcast_ttl_pipeline(); + let mut src_mac = [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee]; + src_mac.reverse(); + pipeline.add_egress_mcast_src_mac_entry( + "rewrite_src_mac", + &1u16.to_le_bytes(), + &src_mac, + 0, + ); + let mut npu = SoftNpu::new(2, pipeline, false); + let phy0 = npu.phy(0); + let phy1 = npu.phy(1); + + npu.run(); + + let probe = mcast_v4_probe("238.1.1.1", 64); + let frame = sidecar_wrap(1, ETHERTYPE_IPV4, &probe); + phy0.send(&[TxFrame::new( + l2_dst(phy1.mac, ETHERTYPE_IPV4, &probe), + ETHERTYPE_SIDECAR, + &frame, + )])?; + phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &sentinel_v4())])?; + + let port1 = recv_until_sentinel(&phy1, "port 1 sentinel"); + assert_eq!(port1.len(), 1, "one underlay copy on port 1"); + assert_eq!(port1[0].ethertype, ETHERTYPE_IPV6, "NAT-encapsulated copy"); + assert_eq!( + port1[0].src, + [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee], + "port 1's SMAC" + ); + assert_eq!(port1[0].dst, [0x33, 0x33, 0, 0, 0, 1], "DMAC from ff04::1"); + assert_nat_encap(&port1[0].payload, UNDERLAY_MCAST_DST, 77); + assert_eq!(phy0.recv_buffer_len(), 0, "no copy back to the scrimlet"); + Ok(()) +} + +fn scrimlet_mcast_ttl_pipeline_copies( + dst: Option<[u8; 6]>, + ethertype: u16, + probe: &[u8], +) -> Result, anyhow::Error> { + let mut npu = SoftNpu::new(2, mcast_ttl_pipeline(), false); + let phy0 = npu.phy(0); + let phy1 = npu.phy(1); + + npu.run(); + + let dst = dst.unwrap_or_else(|| l2_dst(phy1.mac, ethertype, probe)); + let frame = sidecar_wrap(1, ethertype, probe); + phy0.send(&[TxFrame::new(dst, ETHERTYPE_SIDECAR, &frame)])?; + phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &sentinel_v4())])?; + + let port1 = recv_until_sentinel(&phy1, "port 1 sentinel"); + assert_eq!(phy0.recv_buffer_len(), 0, "no copy back to the scrimlet"); + Ok(port1) +} + +#[test] +fn scrimlet_mcast_bad_dst_mac_dropped() -> Result<(), anyhow::Error> { + let unicast = [0x02, 0, 0, 0, 0, 0x01]; + + let v6 = mcast_v6_probe("ff02::1", 64); + let fs = + scrimlet_mcast_ttl_pipeline_copies(Some(unicast), ETHERTYPE_IPV6, &v6)?; + assert!(fs.is_empty(), "ff02::1 with a unicast DMAC dropped"); + + let v4 = mcast_v4_probe("238.1.1.1", 64); + let fs = + scrimlet_mcast_ttl_pipeline_copies(Some(unicast), ETHERTYPE_IPV4, &v4)?; + assert!(fs.is_empty(), "238.1.1.1 with a unicast DMAC dropped"); + Ok(()) +} + +#[test] +fn scrimlet_mcast_ttl_expired_dropped() -> Result<(), anyhow::Error> { + for (ttl, copies) in [(0, 0), (1, 0), (2, 1)] { + let probe = mcast_v4_probe("238.1.1.1", ttl); + let fs = + scrimlet_mcast_ttl_pipeline_copies(None, ETHERTYPE_IPV4, &probe)?; + assert_eq!(fs.len(), copies, "scrimlet 238.1.1.1 at TTL {ttl}"); + + let probe = mcast_v6_probe("ff0e::1", ttl); + let fs = + scrimlet_mcast_ttl_pipeline_copies(None, ETHERTYPE_IPV6, &probe)?; + assert_eq!(fs.len(), copies, "scrimlet ff0e::1 at hop limit {ttl}"); } Ok(()) } From 840ce97e134c1c40cd5472cbf3b13349b1da4073 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 10 Nov 2026 23:39:13 +0000 Subject: [PATCH 8/8] [mcast, mac] switch-zone traffic out sc_egress; SMAC rewrite @ end of ingress ~ match sidecar.p4 Per Ry's review (giant-NIC approach/model) + chat: - I've placed the scrimlet path back to main's `fwd_from_scrimlet` (i.e., no replication, NAT, TTL or DMAC handling for anything coming from the switch zone), and `decap_sidecar` has been folded back into `fwd_from_scrimlet` as such. - The SMAC rewrite has been moved to the end of the ingress control for non-mcast packets (by IP dst) that aren't tagged to drop and aren't headed to the scrimlet (port 0). - Unlike main, of course, this also covers switch-zone unicast/ARP/LLDP. - This matches up w/ the [sidecar.p4's end-of-Ingress `mac_rewrite`][sidecar], which is also what covers tfportd's re-injected packets [sidecar]: https://github.com/oxiecomputer/dendrite/blob/f8d4418a9a08741a2965eb45e7bf33333f0387a2/dpd/p4/sidecar.p4#L2201-L2205 Note: the body diff under `if (hdr.sidecar.isValid()) { ... } else { ... }` is mostly a reindent (use `-w` in jj for example). --- p4/sidecar-lite.p4 | 310 +++++++++++++++++++++---------------------- softnpu/src/test.rs | 316 +++++++++++++++++++++++--------------------- 2 files changed, 315 insertions(+), 311 deletions(-) diff --git a/p4/sidecar-lite.p4 b/p4/sidecar-lite.p4 index 74ddc5d..868250c 100644 --- a/p4/sidecar-lite.p4 +++ b/p4/sidecar-lite.p4 @@ -27,191 +27,180 @@ control ingress( ipv4_ttl_decrement() ttl_dec; apply { - bool mcast_dst = false; + bool is_mcast = false; + // TODO: Combine these checks when x4c supports && and ||. if (hdr.ipv6.isValid()) { - if (hdr.ipv6.dst[127:120] == 8w0xff) { mcast_dst = true; } + if (hdr.ipv6.dst[127:120] == 8w0xff) { is_mcast = true; } } if (hdr.ipv4.isValid()) { - if (hdr.ipv4.dst[31:28] == 4w0xe) { mcast_dst = true; } - } - - if (mcast_dst) { - if (hdr.ipv4.isValid()) { - // 01:00:5e + zero bit, then the low 23 bits of the - // group (RFC 1112 section 6.4). - if (hdr.ethernet.dst[47:23] != 25w0x0200bc) { - egress.drop = true; - return; - } - if (hdr.ethernet.dst[22:0] != hdr.ipv4.dst[22:0]) { - egress.drop = true; - return; - } - } - - if (hdr.ipv6.isValid()) { - if (hdr.ethernet.dst[47:32] != 16w0x3333) { - egress.drop = true; - return; - } - if (hdr.ethernet.dst[31:0] != hdr.ipv6.dst[31:0]) { - egress.drop = true; - return; - } - } + if (hdr.ipv4.dst[31:28] == 4w0xe) { is_mcast = true; } } // - // Check if this is a packet coming from the scrimlet: - // - // - Unicast is forwarded out the port indicated by the sidecar header, - // with its source MAC rewritten for that specific port. - // - // - IPv6 link-local multicast is forwarded out the same port, keeping - // the MACs set by userspace. - // - // - Other multicast packets have the sidecar header removed and - // continue through to the multicast path below. + // Check if this is a packet coming from the scrimlet. If so, just + // forward it out the source port indicated by the sidecar header. // if (hdr.sidecar.isValid()) { - bool link_local_mcast = false; - if (hdr.ipv6.isValid()) { - if (hdr.ipv6.dst[127:112] == 16w0xff02) { - link_local_mcast = true; - } - } - if (link_local_mcast) { fwd_from_scrimlet(); return; } - if (mcast_dst == false) { - fwd_from_scrimlet(); - mac.apply(hdr, egress); - return; + fwd_from_scrimlet(); + } else { + /// + /// Check if we need to proxy arp this packet. + /// + if (hdr.arp.isValid()) { + bool proxied = false; + pxarp.apply(hdr, ingress, egress, proxied); + if (proxied) { return; } } - decap_sidecar(); - } - /// - /// Check if we need to proxy arp this packet. - /// - if (hdr.arp.isValid()) { - bool proxied = false; - pxarp.apply(hdr, ingress, egress, proxied); - if (proxied) { return; } - } - - // - // If the packet has a local destination, create the sidecar header and - // send it to the scrimlet. - // - bool local_dst = false; - local.apply(ingress, hdr, local_dst); - if (local_dst) { - // If this is boundary services packet, it should have a geneve - // header. Decap and attempt to route. - if (hdr.geneve.isValid()) { - ingress.nat_egress_hit = true; - decap_geneve(); - } - // If there is no geneve header this is a packet from outside the - // rack and should be sent to the scrimlet. - else { fwd_to_scrimlet(); return; } - } else { - // Multicast with TTL/hop_limit <= 1 must not be forwarded - // (RFC 1112, RFC 8200 section 3). The check lives here, not in - // the parser, because link-local scrimlet transmissions return - // earlier through fwd_from_scrimlet and must not be dropped. Other - // scrimlet multicast gets checked here, just like in dendrite's - // parser. - if (mcast_dst) { + if (is_mcast) { if (hdr.ipv4.isValid()) { - if (hdr.ipv4.ttl <= 8w1) { egress.drop = true; return; } + // 01:00:5e + zero bit, then the low 23 bits of the + // group (RFC 1112 section 6.4). + if (hdr.ethernet.dst[47:23] != 25w0x0200bc) { + egress.drop = true; + return; + } + if (hdr.ethernet.dst[22:0] != hdr.ipv4.dst[22:0]) { + egress.drop = true; + return; + } } + if (hdr.ipv6.isValid()) { - if (hdr.ipv6.hop_limit <= 8w1) { + if (hdr.ethernet.dst[47:32] != 16w0x3333) { + egress.drop = true; + return; + } + if (hdr.ethernet.dst[31:0] != hdr.ipv6.dst[31:0]) { egress.drop = true; return; } } - } else { - attached.apply(ingress, hdr); - } - if (hdr.geneve.isValid() == false) { - nat.apply(hdr, ingress, egress); // check for ingress nat } - } - // - // After local and NAT processing, basic packet forwarding happens. - // - // Unicast and multicast are bifurcated, mirroring dendrite's - // sidecar.p4 (the meta.is_mcast metadata check). Multicast traffic - // is replicated and bypasses the unicast nexthop resolver, - // whose default_action is drop. Each replica's destination and - // source MACs are derived in the egress control instead. - // - // IPv6 link-local multicast (ff02::/16) never reaches this point. From - // a front port it is marked local and handled by the earlier local - // path; from the scrimlet it leaves through fwd_from_scrimlet. The parser - // rejects interface-local scope multicast, while expired multicast - // (TTL/hop_limit <= 1) gets dropped on the non-local path above. - // - // IPv4 local network control (224.0.0.0/24, RFC 5771 section 4) is - // not dropped by address. The TTL check covers IGMP, OSPF and - // PIM, but senders such as mDNS and VRRP can use TTL 255. - bool is_mcast = false; - if (hdr.ipv6.isValid()) { - if (hdr.ipv6.dst[127:120] == 8w0xff) { is_mcast = true; } - } - if (hdr.ipv4.isValid()) { - if (hdr.ipv4.dst[31:28] == 4w0xe) { is_mcast = true; } - } - - if (is_mcast) { - if (hdr.vlan.isValid()) { - hdr.ethernet.ether_type = hdr.vlan.ether_type; - hdr.vlan.setInvalid(); - } - mcast.apply(hdr, ingress, egress); - // Packet copies that keep their encapsulation are forwarded on the - // outer IPv6 header, losing a hop like dendrite's - // MulticastRouter6 (RFC 8200 section 3). Decapsulated copies - // have the inner header decremented before decap_geneve executes. - if (ingress.mcast_decap) { - validate_mcast_external(); - if (egress.drop == false) { - if (hdr.inner_ipv4.isValid()) { - ttl_dec.apply(hdr.inner_ipv4); - } - if (hdr.inner_ipv6.isValid()) { - hdr.inner_ipv6.hop_limit = - hdr.inner_ipv6.hop_limit - 8w1; - } + // + // If the packet has a local destination, create the sidecar header and + // send it to the scrimlet. + // + bool local_dst = false; + local.apply(ingress, hdr, local_dst); + if (local_dst) { + // If this is boundary services packet, it should have a geneve + // header. Decap and attempt to route. + if (hdr.geneve.isValid()) { + ingress.nat_egress_hit = true; decap_geneve(); } + // If there is no geneve header this is a packet from outside the + // rack and should be sent to the scrimlet. + else { fwd_to_scrimlet(); return; } } else { - if (hdr.ipv6.isValid()) { - if (hdr.ipv6.hop_limit <= 8w1) { egress.drop = true; } - if (egress.drop == false) { - hdr.ipv6.hop_limit = hdr.ipv6.hop_limit - 8w1; + // Multicast with TTL/hop_limit <= 1 must not be forwarded + // (RFC 1112, RFC 8200 section 3). The check lives here, not in + // the parser, because scrimlet transmissions take the + // fwd_from_scrimlet branch and must not be dropped. + if (is_mcast) { + if (hdr.ipv4.isValid()) { + if (hdr.ipv4.ttl <= 8w1) { egress.drop = true; return; } + } + if (hdr.ipv6.isValid()) { + if (hdr.ipv6.hop_limit <= 8w1) { + egress.drop = true; + return; + } } + } else { + attached.apply(ingress, hdr); + } + if (hdr.geneve.isValid() == false) { + nat.apply(hdr, ingress, egress); // check for ingress nat } } - // An empty bitmap is equivalent to having no multicast group - // configured, and the packet is dropped. + + // + // After local and NAT processing, basic packet forwarding happens. + // + // Unicast and multicast are bifurcated, mirroring dendrite's + // sidecar.p4 (the meta.is_mcast metadata check). Multicast traffic + // is replicated and bypasses the unicast nexthop resolver, + // whose default_action punts to the scrimlet. Each replica's + // destination and source MACs are derived in the egress control + // instead. // - // This matches the default action of dendrite's - // mcast_replication_ipv6 (drop_mcastv6_admin_scoped_no_group). - // Neither pipeline sends an ICMP error for multicast (RFC 1812 - // section 4.3.2.7). - if (egress.port_bitmap == 128w0) { - egress.drop = true; + // IPv6 link-local multicast (ff02::/16) never reaches this point; it is + // marked local and handled by the earlier local path. The parser + // rejects interface-local scope multicast, while expired multicast + // (TTL/hop_limit <= 1) gets dropped on the non-local path above. + // + // IPv4 local network control (224.0.0.0/24, RFC 5771 section 4) is + // not dropped by address. The TTL check covers IGMP, OSPF and + // PIM, but senders such as mDNS and VRRP can use TTL 255. + is_mcast = false; + if (hdr.ipv6.isValid()) { + if (hdr.ipv6.dst[127:120] == 8w0xff) { is_mcast = true; } } - } else { - router.apply(hdr, ingress, egress); - // Prevent reflection. Multicast ingress-port suppression is - // handled in the replication bitmap, so this applies to - // unicast only. - if (ingress.port == egress.port) { egress.drop = true; } else { - resolver.apply(hdr, ingress, egress); + if (hdr.ipv4.isValid()) { + if (hdr.ipv4.dst[31:28] == 4w0xe) { is_mcast = true; } + } + + if (is_mcast) { + if (hdr.vlan.isValid()) { + hdr.ethernet.ether_type = hdr.vlan.ether_type; + hdr.vlan.setInvalid(); + } + mcast.apply(hdr, ingress, egress); + // Packet copies that keep their encapsulation are forwarded on the + // outer IPv6 header, losing a hop like dendrite's + // MulticastRouter6 (RFC 8200 section 3). Decapsulated copies + // have the inner header decremented before decap_geneve executes. + if (ingress.mcast_decap) { + validate_mcast_external(); + if (egress.drop == false) { + if (hdr.inner_ipv4.isValid()) { + ttl_dec.apply(hdr.inner_ipv4); + } + if (hdr.inner_ipv6.isValid()) { + hdr.inner_ipv6.hop_limit = + hdr.inner_ipv6.hop_limit - 8w1; + } + decap_geneve(); + } + } else { + if (hdr.ipv6.isValid()) { + if (hdr.ipv6.hop_limit <= 8w1) { egress.drop = true; } + if (egress.drop == false) { + hdr.ipv6.hop_limit = hdr.ipv6.hop_limit - 8w1; + } + } + } + // An empty bitmap is equivalent to having no multicast group + // configured, and the packet is dropped. + // + // This matches the default action of dendrite's + // mcast_replication_ipv6 (drop_mcastv6_admin_scoped_no_group). + // Neither pipeline sends an ICMP error for multicast (RFC 1812 + // section 4.3.2.7). + if (egress.port_bitmap == 128w0) { + egress.drop = true; + } + } else { + router.apply(hdr, ingress, egress); + // Prevent reflection. Multicast ingress-port suppression is + // handled in the replication bitmap, so this applies to + // unicast only. + if (ingress.port == egress.port) { egress.drop = true; } else { + resolver.apply(hdr, ingress, egress); + } + } + } + + // + // Unicast has its source MAC rewritten for that port, but + // multicast is disregarded, as replicas get their MACs in the + // egress control, and multicast packets from the scrimlet + // continue on as userspace had sent it. + if (egress.drop == false) { + if (is_mcast == false) { // Packets sent to scrimlet don't need their SMAC rewritten. // If a resolver miss punts the packet and it gets retransmitted // by tfportd (upon successful ARP/ND resolution), the SMAC is @@ -305,10 +294,7 @@ control ingress( // Direct packets to the sidecar port corresponding to the scrimlet // port they came from. egress.port = hdr.sidecar.sc_egress; - decap_sidecar(); - } - - action decap_sidecar() { + // Decap the sidecar header. hdr.ethernet.ether_type = hdr.sidecar.sc_ether_type; hdr.sidecar.setInvalid(); } diff --git a/softnpu/src/test.rs b/softnpu/src/test.rs index 8bfae67..047f687 100644 --- a/softnpu/src/test.rs +++ b/softnpu/src/test.rs @@ -1500,10 +1500,9 @@ fn sidecar_wrap(egress: u16, ethertype: u16, l3: &[u8]) -> Vec { // Send `probe` from the scrimlet side (port 0) toward port 1 and collect what // exits port 1. // -// No replication group or NAT entry is installed. IPv6 link-local multicast -// exits through sc_egress, while all other multicast enters the multicast path -// and is dropped, by the TTL check at TTL <= 1 and otherwise on the empty -// bitmap. +// No replication group or NAT entry is installed: ingress takes the +// fwd_from_scrimlet path, which forwards out sc_egress without looking at the +// multicast tables. fn scrimlet_mcast_copies( ethertype: u16, probe: &[u8], @@ -1542,195 +1541,196 @@ fn scrimlet_mcast_copies( Ok(frames) } +// The scrimlet's own multicast (e.g., TTL-1 IGMP reports) acts as a +// transmission versus as a forwarding hop. The packet leaves unchanged at +// any TTL, while the same packet from a front port is dropped at TTL <= 1 +// (except ff02, which is punted to the scrimlet). #[test] -fn scrimlet_ipv6_link_local_mcast_forwarded() -> Result<(), anyhow::Error> { - for hop_limit in [0, 1, 64, 255] { - let probe = mcast_v6_probe("ff02::1", hop_limit); - let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; - assert_eq!(fs.len(), 1, "one copy out sc_egress"); - assert_eq!(fs[0].ethertype, ETHERTYPE_IPV6, "sidecar header stripped"); - let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); - assert_eq!( - ip.get_hop_limit(), - hop_limit, - "scrimlet transmission keeps its hop limit" - ); +fn scrimlet_mcast_forwarded() -> Result<(), anyhow::Error> { + for ttl in [0, 1, 64, 255] { + let probe = mcast_v4_probe("238.1.1.1", ttl); + let fs = scrimlet_mcast_copies(ETHERTYPE_IPV4, &probe)?; + assert_eq!(fs.len(), 1, "238.1.1.1 at TTL {ttl} out sc_egress"); + assert_eq!(fs[0].ethertype, ETHERTYPE_IPV4, "sidecar header stripped"); + let ip = Ipv4Packet::new(&fs[0].payload).unwrap(); + assert_eq!(ip.get_ttl(), ttl, "scrimlet transmission keeps its TTL"); + + for group in ["ff02::1", "ff0e::1"] { + let probe = mcast_v6_probe(group, ttl); + let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; + assert_eq!(fs.len(), 1, "{group} at hop limit {ttl} out sc_egress"); + assert_eq!( + fs[0].ethertype, ETHERTYPE_IPV6, + "sidecar header stripped" + ); + let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); + assert_eq!( + ip.get_hop_limit(), + ttl, + "scrimlet transmission keeps its hop limit" + ); + } } Ok(()) } #[test] -fn scrimlet_mcast_without_group_dropped() -> Result<(), anyhow::Error> { - for ttl in [64, 255] { - let probe = mcast_v4_probe("238.1.1.1", ttl); - let fs = scrimlet_mcast_copies(ETHERTYPE_IPV4, &probe)?; - assert!(fs.is_empty(), "238.1.1.1 at TTL {ttl} left sc_egress"); +fn scrimlet_non_ip_smac_rewritten() -> Result<(), anyhow::Error> { + let arp: [u8; 28] = [ + 0x00, 0x01, 0x08, 0x00, 0x06, 0x04, 0x00, 0x01, // request + 0x02, 0x00, 0x00, 0x00, 0x00, 0x09, 10, 0, 0, 1, // sha, spa + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 10, 0, 0, 2, // tha, tpa + ]; + let lldp: [u8; 8] = [0x02, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00]; + let cases: [(u16, [u8; 6], &[u8]); 2] = [ + (0x0806, [0xff; 6], &arp), + (0x88cc, [0x01, 0x80, 0xc2, 0x00, 0x00, 0x0e], &lldp), + ]; - let probe = mcast_v6_probe("ff0e::1", ttl); - let fs = scrimlet_mcast_copies(ETHERTYPE_IPV6, &probe)?; - assert!(fs.is_empty(), "ff0e::1 at hop limit {ttl} left sc_egress"); + for (ethertype, dst, payload) in cases { + let mut pipeline = main_pipeline::new(2); + pipeline_init(&mut pipeline); + let mut npu = SoftNpu::new(2, pipeline, false); + let phy0 = npu.phy(0); + let phy1 = npu.phy(1); + + npu.run(); + + let frame = sidecar_wrap(1, ethertype, payload); + let sentinel = sidecar_wrap(1, ETHERTYPE_IPV4, &sentinel_v4()); + phy0.send(&[ + TxFrame::new(dst, ETHERTYPE_SIDECAR, &frame), + TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &sentinel), + ])?; + + let fs = recv_until_sentinel(&phy1, "scrimlet sentinel on port 1"); + assert_eq!(fs.len(), 1, "{ethertype:#06x} out sc_egress"); + assert_eq!(fs[0].ethertype, ethertype, "sidecar header stripped"); + assert_eq!(fs[0].src, [1, 2, 3, 4, 5, 6], "port 1's SMAC"); + assert_eq!(fs[0].dst, dst, "DMAC kept"); } Ok(()) } -#[test] -fn scrimlet_mcast_with_group_replicated() -> Result<(), anyhow::Error> { - let mut pipeline = mcast_ff0e_pipeline(&[1], &[2]); - let port2_sentinel = sentinel_route_v4(&mut pipeline, 2); - let mut src_mac = [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee]; - src_mac.reverse(); - pipeline.add_egress_mcast_src_mac_entry( - "rewrite_src_mac", - &2u16.to_le_bytes(), - &src_mac, - 0, - ); - let mut npu = SoftNpu::new(3, pipeline, false); +fn scrimlet_mcast_group_copies( + dst: Option<[u8; 6]>, + ethertype: u16, + probe: &[u8], +) -> Result, anyhow::Error> { + let mut npu = SoftNpu::new(2, mcast_ttl_pipeline(), false); let phy0 = npu.phy(0); let phy1 = npu.phy(1); - let phy2 = npu.phy(2); npu.run(); - let ip_data = geneve_mcast_v4_pkt_repl( - "fd00:1::1", - "ff0e::1", - "10.0.0.1", - "238.1.1.1", - Some(MCAST_TAG_UNDERLAY), - ); - let frame = sidecar_wrap(0, ETHERTYPE_IPV6, &ip_data); - phy0.send(&[TxFrame::new( - l2_dst(phy1.mac, ETHERTYPE_IPV6, &ip_data), - ETHERTYPE_SIDECAR, - &frame, - )])?; + let dst = dst.unwrap_or_else(|| l2_dst(phy1.mac, ethertype, probe)); + let frame = sidecar_wrap(1, ethertype, probe); + phy0.send(&[TxFrame::new(dst, ETHERTYPE_SIDECAR, &frame)])?; phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &sentinel_v4())])?; - phy0.send(&[TxFrame::new( - phy2.mac, - ETHERTYPE_IPV4, - &sentinel_v4_to(&port2_sentinel), - )])?; let port1 = recv_until_sentinel(&phy1, "port 1 sentinel"); - assert!( - port1.is_empty(), - "external copy suppressed by the underlay tag" - ); + assert_eq!(phy0.recv_buffer_len(), 0, "no copy back to the scrimlet"); + Ok(port1) +} - let port2 = recv_until_sentinel(&phy2, "port 2 sentinel"); - assert_eq!(port2.len(), 1, "one underlay copy on port 2"); - assert_eq!( - port2[0].ethertype, ETHERTYPE_IPV6, - "sidecar header stripped" - ); +#[test] +fn scrimlet_mcast_no_nat() -> Result<(), anyhow::Error> { + let probe = mcast_v4_probe("238.1.1.1", 64); + let fs = scrimlet_mcast_group_copies(None, ETHERTYPE_IPV4, &probe)?; + assert_eq!(fs.len(), 1, "one copy out sc_egress"); + assert_eq!(fs[0].ethertype, ETHERTYPE_IPV4, "not NAT-encapsulated"); + let ip = Ipv4Packet::new(&fs[0].payload).unwrap(); assert_eq!( - port2[0].src, - [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee], - "port 2's SMAC" + ip.get_destination(), + "238.1.1.1".parse::().unwrap(), + "original group" ); - assert_eq!(port2[0].dst, [0x33, 0x33, 0, 0, 0, 1], "DMAC from ff0e::1"); - let ip = Ipv6Packet::new(&port2[0].payload).unwrap(); + assert_eq!(ip.get_ttl(), 64, "TTL not decremented"); + + let probe = mcast_v6_probe("ff0e::1", 64); + let fs = scrimlet_mcast_group_copies(None, ETHERTYPE_IPV6, &probe)?; + assert_eq!(fs.len(), 1, "one copy out sc_egress"); + let ip = Ipv6Packet::new(&fs[0].payload).unwrap(); assert_eq!( ip.get_destination(), "ff0e::1".parse::().unwrap(), - "copy keeps its encapsulation" + "not NAT-encapsulated to the underlay group" ); - assert_eq!(ip.get_hop_limit(), 63, "outer hop limit decremented"); - assert_eq!(phy0.recv_buffer_len(), 0, "sc_egress (port 0) ignored"); + assert_eq!(ip.get_hop_limit(), 64, "hop limit not decremented"); Ok(()) } #[test] -fn scrimlet_mcast_nat_replicated() -> Result<(), anyhow::Error> { - let mut pipeline = mcast_ttl_pipeline(); - let mut src_mac = [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee]; - src_mac.reverse(); - pipeline.add_egress_mcast_src_mac_entry( - "rewrite_src_mac", - &1u16.to_le_bytes(), - &src_mac, - 0, - ); - let mut npu = SoftNpu::new(2, pipeline, false); +fn scrimlet_geneve_mcast_no_repl() -> Result<(), anyhow::Error> { + let mut pipeline = mcast_ff0e_pipeline(&[], &[2]); + let mut rewrite_mac = [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee]; + rewrite_mac.reverse(); + for port in [1u16, 2] { + pipeline.add_egress_mcast_src_mac_entry( + "rewrite_src_mac", + &port.to_le_bytes(), + &rewrite_mac, + 0, + ); + } + + let mut npu = SoftNpu::new(3, pipeline, false); let phy0 = npu.phy(0); let phy1 = npu.phy(1); + let phy2 = npu.phy(2); npu.run(); - let probe = mcast_v4_probe("238.1.1.1", 64); - let frame = sidecar_wrap(1, ETHERTYPE_IPV4, &probe); - phy0.send(&[TxFrame::new( - l2_dst(phy1.mac, ETHERTYPE_IPV4, &probe), - ETHERTYPE_SIDECAR, - &frame, - )])?; - phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &sentinel_v4())])?; + let ip_data = geneve_mcast_v4_pkt_repl( + "fd00:1::1", + "ff0e::1", + "10.0.0.1", + "238.1.1.1", + Some(MCAST_TAG_UNDERLAY_EXTERNAL), + ); + let dst = l2_dst(phy1.mac, ETHERTYPE_IPV6, &ip_data); + let frame = sidecar_wrap(1, ETHERTYPE_IPV6, &ip_data); + let port1_sentinel = sidecar_wrap(1, ETHERTYPE_IPV4, &sentinel_v4()); + let port2_sentinel = sidecar_wrap(2, ETHERTYPE_IPV4, &sentinel_v4()); + phy0.send(&[ + TxFrame::new(dst, ETHERTYPE_SIDECAR, &frame), + TxFrame::new(phy1.mac, ETHERTYPE_SIDECAR, &port1_sentinel), + TxFrame::new(phy2.mac, ETHERTYPE_SIDECAR, &port2_sentinel), + ])?; - let port1 = recv_until_sentinel(&phy1, "port 1 sentinel"); - assert_eq!(port1.len(), 1, "one underlay copy on port 1"); - assert_eq!(port1[0].ethertype, ETHERTYPE_IPV6, "NAT-encapsulated copy"); + let port1 = recv_until_sentinel(&phy1, "scrimlet sentinel on port 1"); + assert_eq!(port1.len(), 1, "one copy out sc_egress"); assert_eq!( - port1[0].src, - [0x02, 0xaa, 0xbb, 0xcc, 0xdd, 0xee], - "port 1's SMAC" + port1[0].ethertype, ETHERTYPE_IPV6, + "sidecar header stripped" + ); + assert_eq!(port1[0].src, phy0.mac, "scrimlet src MAC kept"); + assert_eq!(port1[0].dst, dst, "scrimlet dst MAC kept"); + assert_eq!(port1[0].vid, None, "no VLAN added"); + assert_eq!( + port1[0].payload, ip_data, + "Geneve encapsulation and TTLs kept" ); - assert_eq!(port1[0].dst, [0x33, 0x33, 0, 0, 0, 1], "DMAC from ff04::1"); - assert_nat_encap(&port1[0].payload, UNDERLAY_MCAST_DST, 77); - assert_eq!(phy0.recv_buffer_len(), 0, "no copy back to the scrimlet"); - Ok(()) -} - -fn scrimlet_mcast_ttl_pipeline_copies( - dst: Option<[u8; 6]>, - ethertype: u16, - probe: &[u8], -) -> Result, anyhow::Error> { - let mut npu = SoftNpu::new(2, mcast_ttl_pipeline(), false); - let phy0 = npu.phy(0); - let phy1 = npu.phy(1); - - npu.run(); - - let dst = dst.unwrap_or_else(|| l2_dst(phy1.mac, ethertype, probe)); - let frame = sidecar_wrap(1, ethertype, probe); - phy0.send(&[TxFrame::new(dst, ETHERTYPE_SIDECAR, &frame)])?; - phy0.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &sentinel_v4())])?; - let port1 = recv_until_sentinel(&phy1, "port 1 sentinel"); + let port2 = recv_until_sentinel(&phy2, "scrimlet sentinel on port 2"); + assert!(port2.is_empty(), "no copy on the replication group port"); assert_eq!(phy0.recv_buffer_len(), 0, "no copy back to the scrimlet"); - Ok(port1) + Ok(()) } #[test] -fn scrimlet_mcast_bad_dst_mac_dropped() -> Result<(), anyhow::Error> { +fn scrimlet_mcast_dst_mac_unchecked() -> Result<(), anyhow::Error> { let unicast = [0x02, 0, 0, 0, 0, 0x01]; let v6 = mcast_v6_probe("ff02::1", 64); - let fs = - scrimlet_mcast_ttl_pipeline_copies(Some(unicast), ETHERTYPE_IPV6, &v6)?; - assert!(fs.is_empty(), "ff02::1 with a unicast DMAC dropped"); + let fs = scrimlet_mcast_group_copies(Some(unicast), ETHERTYPE_IPV6, &v6)?; + assert_eq!(fs.len(), 1, "ff02::1 w/ a unicast DMAC out sc_egress"); + assert_eq!(fs[0].dst, unicast, "DMAC kept"); let v4 = mcast_v4_probe("238.1.1.1", 64); - let fs = - scrimlet_mcast_ttl_pipeline_copies(Some(unicast), ETHERTYPE_IPV4, &v4)?; - assert!(fs.is_empty(), "238.1.1.1 with a unicast DMAC dropped"); - Ok(()) -} - -#[test] -fn scrimlet_mcast_ttl_expired_dropped() -> Result<(), anyhow::Error> { - for (ttl, copies) in [(0, 0), (1, 0), (2, 1)] { - let probe = mcast_v4_probe("238.1.1.1", ttl); - let fs = - scrimlet_mcast_ttl_pipeline_copies(None, ETHERTYPE_IPV4, &probe)?; - assert_eq!(fs.len(), copies, "scrimlet 238.1.1.1 at TTL {ttl}"); - - let probe = mcast_v6_probe("ff0e::1", ttl); - let fs = - scrimlet_mcast_ttl_pipeline_copies(None, ETHERTYPE_IPV6, &probe)?; - assert_eq!(fs.len(), copies, "scrimlet ff0e::1 at hop limit {ttl}"); - } + let fs = scrimlet_mcast_group_copies(Some(unicast), ETHERTYPE_IPV4, &v4)?; + assert_eq!(fs.len(), 1, "238.1.1.1 w/ a unicast DMAC out sc_egress"); + assert_eq!(fs[0].dst, unicast, "DMAC kept"); Ok(()) } @@ -3672,7 +3672,7 @@ fn resolver_miss() -> Result<(), anyhow::Error> { let probe4 = v4_probe(SRC4, DST4, 255); let probe4_desc = format!("IPv4 packet (dst={DST4}, src={SRC4}, ttl=255)"); - // Send IPv4 probe from port 1 so scrimlet (port 0) can receive it. + // Send IPv4 probe from port 2 so scrimlet (port 0) can receive it. phy2.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV4, &probe4)])?; wait_for( @@ -3680,9 +3680,18 @@ fn resolver_miss() -> Result<(), anyhow::Error> { &format!("scrimlet should receive {probe4_desc}"), ); let maybe_sidecar = phy0.recv().remove(0); + assert_eq!( + maybe_sidecar.ethertype, ETHERTYPE_SIDECAR, + "{probe4_desc} should arrive with sidecar EtherType" + ); let maybe_arp_needed = SidecarPacket::new(&maybe_sidecar.payload).expect("sidecar header"); + assert_eq!( + maybe_arp_needed.get_sc_ether_type(), + ETHERTYPE_IPV4, + "{probe4_desc} should retain its IPv4 EtherType" + ); assert_eq!( maybe_arp_needed.get_sc_code(), SC_ARP_NEEDED, @@ -3711,7 +3720,7 @@ fn resolver_miss() -> Result<(), anyhow::Error> { let probe6_desc = format!("IPv6 packet (dst={DST6}, src={SRC6}, hop-limit=255)"); - // Send IPv6 probe from port 1 so scrimlet (port 0) can receive it. + // Send IPv6 probe from port 2 so scrimlet (port 0) can receive it. phy2.send(&[TxFrame::new(phy1.mac, ETHERTYPE_IPV6, &probe6)])?; wait_for( @@ -3719,9 +3728,18 @@ fn resolver_miss() -> Result<(), anyhow::Error> { &format!("scrimlet should receive {probe6_desc}"), ); let maybe_sidecar = phy0.recv().remove(0); + assert_eq!( + maybe_sidecar.ethertype, ETHERTYPE_SIDECAR, + "{probe6_desc} should arrive with sidecar EtherType" + ); let maybe_ndp_needed = SidecarPacket::new(&maybe_sidecar.payload).expect("sidecar header"); + assert_eq!( + maybe_ndp_needed.get_sc_ether_type(), + ETHERTYPE_IPV6, + "{probe6_desc} should retain its IPv6 EtherType" + ); assert_eq!( maybe_ndp_needed.get_sc_code(), SC_NEIGHBOR_NEEDED, @@ -3752,8 +3770,8 @@ fn resolver_miss_pipeline() -> main_pipeline { pipeline } -// A packet routed back out its ingress port is dropped regardless of the action -// that control resolver takes +// A packet routed back out of its ingress port is dropped before the control's +// resolver runs (it is never punted to userspace). #[test] fn resolver_miss_reflection_dropped() -> Result<(), anyhow::Error> { let mut pipeline = resolver_miss_pipeline();