From 7fcaf09a8e860cbe49ecdf18fddaca591b901c08 Mon Sep 17 00:00:00 2001 From: Ryan Goodfellow Date: Fri, 23 Jan 2026 07:31:26 +0000 Subject: [PATCH 1/2] attempt to reproduce packet corruption issue from london --- dpd-client/tests/integration_tests/local.rs | 97 +++++++++++++++++++++ dpd-client/tests/integration_tests/mod.rs | 1 + 2 files changed, 98 insertions(+) create mode 100644 dpd-client/tests/integration_tests/local.rs diff --git a/dpd-client/tests/integration_tests/local.rs b/dpd-client/tests/integration_tests/local.rs new file mode 100644 index 00000000..402cb936 --- /dev/null +++ b/dpd-client/tests/integration_tests/local.rs @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use crate::integration_tests::common::{ + PhysPort, SERVICE_PORT, TestPacket, TestResult, get_switch, +}; +use dpd_client::ClientInfo; +use dpd_client::types::Ipv4Entry; +use packet::{ + Packet, + eth::{ETHER_IPV4, ETHER_SIDECAR}, + sidecar::{SC_FWD_TO_USERSPACE, SidecarHdr}, +}; +use std::sync::Arc; + +// This test attempts to reproduce an issue we saw on london whre a TCP SYN +// packet destined for the switch zone has it's source port altered when +// going through the ASIC. This causes a checksum failure at the host and +// torpedoes the SSH session. This condition lasted for nearly half a day +// and then all of a sudden vanished and communications started working. +// +// The before tofino and after tofino packet dumps are the following. +// The source port corruption is highlighted by the carats. +// +// before: +// +// 0000 a8 40 25 05 02 23 aa 00 04 00 ca fe 08 00 45 c0 +// 0010 00 3c da 57 40 00 01 06 28 44 ac 14 0f 1b ac 14 +// 0020 0f 1d ac 31 00 b3 84 f4 27 61 00 00 00 00 a0 02 +// ^^^^^ +// 0030 83 2c a1 20 00 00 02 04 05 b4 04 02 08 0a b9 a7 +// 0040 9a 76 00 00 00 00 01 03 03 01 +// +// after: +// +// 0000 a8 40 25 05 02 23 aa 00 04 00 ca fe 08 00 45 c0 +// 0010 00 3c da 57 40 00 01 06 28 44 ac 14 0f 1b ac 14 +// 0020 0f 1d a0 01 00 b3 84 f4 27 61 00 00 00 00 a0 02 +// ^^^^^ +// 0030 83 2c a1 20 00 00 02 04 05 b4 04 02 08 0a b9 a7 +// 0040 9a 76 00 00 00 00 01 03 03 01 +// +// This test sets up the local switch address 172.20.15.29 and sends the +// packet we had trouble with through the ASIC expecting it to come out +// the other side in tact with a sidecar header. If this test passes it +// is not presenting the issue we saw on london. + +#[tokio::test] +#[ignore] +async fn bgp_syn_of_doom() -> TestResult { + let switch = &*get_switch().await; + let ingress = PhysPort(10); + + let (port_id, link_id) = switch.link_id(ingress).unwrap(); + let entry = Ipv4Entry { + addr: "172.20.15.29".parse().unwrap(), + tag: switch.client.inner().tag.clone(), + }; + switch.client.link_ipv4_create(&port_id, &link_id, &entry).await.unwrap(); + + let syn_of_doom_before = Packet::parse(&[ + 0xa8, 0x40, 0x25, 0x5, 0x2, 0x23, 0xaa, 0x0, 0x4, 0x0, 0xca, 0xfe, 0x8, + 0x0, 0x45, 0xc0, 0x0, 0x3c, 0xda, 0x57, 0x40, 0x0, 0x1, 0x6, 0x28, + 0x44, 0xac, 0x14, 0xf, 0x1b, 0xac, 0x14, 0xf, 0x1d, 0xac, 0x31, 0x0, + 0xb3, 0x84, 0xf4, 0x27, 0x61, 0x0, 0x0, 0x0, 0x0, 0xa0, 0x2, 0x83, + 0x2c, 0xa1, 0x20, 0x0, 0x0, 0x2, 0x4, 0x5, 0xb4, 0x4, 0x2, 0x8, 0xa, + 0xb9, 0xa7, 0x9a, 0x76, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x3, 0x1, + ]) + .unwrap(); + + let mut syn_of_doom_after = syn_of_doom_before.clone(); + if let Some(eth) = &mut syn_of_doom_after.hdrs.eth_hdr { + eth.eth_type = ETHER_SIDECAR; + } + syn_of_doom_after.hdrs.sidecar_hdr = Some(SidecarHdr { + sc_code: SC_FWD_TO_USERSPACE, + sc_pad: 0, + sc_ingress: 0x90, + sc_egress: 0, + sc_ether_type: ETHER_IPV4, + sc_payload: [0u8; 16], + }); + + switch.packet_test( + vec![TestPacket { + packet: Arc::new(syn_of_doom_before), + port: ingress, + }], + vec![TestPacket { + packet: Arc::new(syn_of_doom_after), + port: SERVICE_PORT, + }], + ) +} diff --git a/dpd-client/tests/integration_tests/mod.rs b/dpd-client/tests/integration_tests/mod.rs index 7f1522a8..7744a0fd 100644 --- a/dpd-client/tests/integration_tests/mod.rs +++ b/dpd-client/tests/integration_tests/mod.rs @@ -9,6 +9,7 @@ mod common; mod counters; mod geneve; mod icmp_ipv4; +mod local; mod loopback; mod mcast; mod nat; From 7af968dbafde9092d17cdb57e1cf915e78575ed1 Mon Sep 17 00:00:00 2001 From: Ryan Goodfellow Date: Sun, 25 Jan 2026 07:59:53 +0000 Subject: [PATCH 2/2] add london bridges tests --- dpd-client/tests/integration_tests/nat.rs | 252 ++++++++++++++++++++++ 1 file changed, 252 insertions(+) diff --git a/dpd-client/tests/integration_tests/nat.rs b/dpd-client/tests/integration_tests/nat.rs index 572697e1..1f108f78 100644 --- a/dpd-client/tests/integration_tests/nat.rs +++ b/dpd-client/tests/integration_tests/nat.rs @@ -9,6 +9,7 @@ use std::net::Ipv6Addr; use std::sync::Arc; use anyhow::anyhow; +use oxnet::Ipv4Net; use oxnet::Ipv6Net; use ::common::network::MacAddr; @@ -16,6 +17,7 @@ use ::common::network::Vni; use dpd_client::ClientInfo; use dpd_client::types; use packet::Endpoint; +use packet::Packet; use packet::eth; use packet::geneve; use packet::icmp; @@ -678,3 +680,253 @@ async fn test_ingress_ipv6_tcp() -> TestResult { let switch = &*get_switch().await; test_ingress_ipv6(switch, L4Protocol::Tcp).await } + +#[tokio::test] +#[ignore] +async fn test_london_bridges_falling_down() -> TestResult { + let switch = &*get_switch().await; + let ingress = PhysPort(8); + let egress = PhysPort(9); + + let link_id = types::LinkId(0); + let tag = switch.client.inner().tag.clone(); + + // ADDRESS: Add IPv6 addresses to rear ports + let rear_addrs: [(u8, &str); 30] = [ + (0, "fe80::aa40:25ff:fe05:2203"), + (1, "fe80::aa40:25ff:fe05:2204"), + (2, "fe80::aa40:25ff:fe05:2205"), + (3, "fe80::aa40:25ff:fe05:2206"), + (4, "fe80::aa40:25ff:fe05:2207"), + (5, "fe80::aa40:25ff:fe05:2208"), + (6, "fe80::aa40:25ff:fe05:2209"), + (7, "fe80::aa40:25ff:fe05:220a"), + (8, "fe80::aa40:25ff:fe05:220b"), + (9, "fe80::aa40:25ff:fe05:220c"), + (10, "fe80::aa40:25ff:fe05:220d"), + (11, "fe80::aa40:25ff:fe05:220e"), + (12, "fe80::aa40:25ff:fe05:220f"), + (13, "fe80::aa40:25ff:fe05:2210"), + (14, "fe80::aa40:25ff:fe05:2211"), + (15, "fe80::aa40:25ff:fe05:2212"), + (16, "fe80::aa40:25ff:fe05:2213"), + (17, "fe80::aa40:25ff:fe05:2214"), + (18, "fe80::aa40:25ff:fe05:2215"), + (19, "fe80::aa40:25ff:fe05:2216"), + (20, "fe80::aa40:25ff:fe05:2217"), + (21, "fe80::aa40:25ff:fe05:2218"), + (22, "fe80::aa40:25ff:fe05:2219"), + (23, "fe80::aa40:25ff:fe05:221a"), + (24, "fe80::aa40:25ff:fe05:221b"), + (25, "fe80::aa40:25ff:fe05:221c"), + (26, "fe80::aa40:25ff:fe05:221d"), + (27, "fe80::aa40:25ff:fe05:221e"), + (28, "fe80::aa40:25ff:fe05:221f"), + (29, "fe80::aa40:25ff:fe05:2220"), + ]; + for (rear_num, addr_str) in rear_addrs { + let port_id: types::PortId = + format!("rear{}", rear_num).parse().unwrap(); + let addr: Ipv6Addr = addr_str.parse().unwrap(); + let entry = types::Ipv6Entry { addr, tag: tag.clone() }; + switch + .client + .link_ipv6_create(&port_id, &link_id, &entry) + .await + .unwrap(); + } + + // ADDRESS: Add IPv4 address to qsfp0 + // NOTE: qsfp0 does not appear to be a thing with the simulator setup.... + // so use rear19? + let qsfp0_port_id: types::PortId = "rear19".parse().unwrap(); + let qsfp0_ipv4: Ipv4Addr = "172.20.15.61".parse().unwrap(); + let entry = types::Ipv4Entry { addr: qsfp0_ipv4, tag: tag.clone() }; + switch + .client + .link_ipv4_create(&qsfp0_port_id, &link_id, &entry) + .await + .unwrap(); + + // NAT: Create NAT entry + let nat_ip: Ipv4Addr = "172.20.29.5".parse().unwrap(); + let internal_ip: Ipv6Addr = "fdf1:eb31:4d93:101::1".parse().unwrap(); + let inner_mac: MacAddr = "a8:40:25:ff:c6:69".parse().unwrap(); + let vni = Vni::new(100).unwrap(); + let tgt = types::NatTarget { + internal_ip, + inner_mac: inner_mac.into(), + vni: vni.into(), + }; + switch.client.nat_ipv4_create(&nat_ip, 0, 16383, &tgt).await.unwrap(); + + // ROUTE: IPv4 default route via qsfp0 + let ipv4_cidr: Ipv4Net = "0.0.0.0/0".parse().unwrap(); + let ipv4_route = types::Ipv4RouteUpdate { + cidr: ipv4_cidr, + target: types::Ipv4Route { + port_id: qsfp0_port_id.clone(), + link_id: link_id.clone(), + tgt_ip: "172.20.15.57".parse().unwrap(), + tag: tag.clone(), + vlan_id: None, + }, + replace: false, + }; + switch.client.route_ipv4_set(&ipv4_route).await.unwrap(); + + // ROUTE: IPv6 routes + let ipv6_routes: [(&str, &str, &str); 11] = [ + ("fdb0:a840:2504:351::/64", "rear11", "fe80::aa40:25ff:fe04:351"), + ("fdb0:a840:2504:614::/64", "rear15", "fe80::aa40:25ff:fe04:614"), + ("fdb0:a840:2504:6d3::/64", "rear16", "fe80::aa40:25ff:fe04:6d3"), + ("fdb0:a840:2504:851::/64", "rear17", "fe80::aa40:25ff:fe04:851"), + ("fdf1:eb31:4d93:1::/64", "rear14", "fe80::aa40:25ff:fe04:351"), + ("fdf1:eb31:4d93:2::/64", "rear15", "fe80::aa40:25ff:fe04:614"), + ("fdf1:eb31:4d93:3::/64", "rear16", "fe80::aa40:25ff:fe04:6d3"), + ("fdf1:eb31:4d93:101::/64", "rear11", "fe80::aa40:25ff:fe04:351"), + ("fdf1:eb31:4d93:102::/64", "rear15", "fe80::aa40:25ff:fe04:614"), + ("fdf1:eb31:4d93:103::/64", "rear16", "fe80::aa40:25ff:fe04:6d3"), + ("fdf1:eb31:4d93:104::/64", "rear17", "fe80::aa40:25ff:fe04:851"), + ]; + for (cidr_str, port_str, gw_str) in ipv6_routes { + let cidr: Ipv6Net = cidr_str.parse().unwrap(); + let port_id: types::PortId = port_str.parse().unwrap(); + let tgt_ip: Ipv6Addr = gw_str.parse().unwrap(); + let route = types::Ipv6RouteUpdate { + cidr, + target: types::Ipv6Route { + port_id, + link_id: link_id.clone(), + tgt_ip, + tag: tag.clone(), + vlan_id: None, + }, + replace: false, + }; + switch.client.route_ipv6_set(&route).await.unwrap(); + } + + // ARP: IPv4 entries + let arp_entries: [(&str, &str); 3] = [ + ("172.20.15.57", "aa:00:04:00:ca:fe"), + ("172.20.15.59", "aa:00:04:00:ca:fe"), + ("172.20.15.61", "a8:40:25:05:22:23"), + ]; + for (ip_str, mac_str) in arp_entries { + let ip: Ipv4Addr = ip_str.parse().unwrap(); + let mac: MacAddr = mac_str.parse().unwrap(); + let entry = types::ArpEntry { + ip: ip.into(), + mac: mac.into(), + tag: tag.clone(), + update: String::new(), + }; + switch.client.arp_create(&entry).await.unwrap(); + } + + // NDP: IPv6 entries + let ndp_entries: [(&str, &str); 36] = [ + ("fe80::aa40:25ff:fe04:351", "a8:40:25:04:03:51"), + ("fe80::aa40:25ff:fe04:614", "a8:40:25:04:06:14"), + ("fe80::aa40:25ff:fe04:6d3", "a8:40:25:04:06:d3"), + ("fe80::aa40:25ff:fe04:851", "a8:40:25:04:08:51"), + ("fe80::aa40:25ff:fe05:2202", "a8:40:25:05:22:02"), + ("fe80::aa40:25ff:fe05:2203", "a8:40:25:05:22:03"), + ("fe80::aa40:25ff:fe05:2204", "a8:40:25:05:22:04"), + ("fe80::aa40:25ff:fe05:2205", "a8:40:25:05:22:05"), + ("fe80::aa40:25ff:fe05:2206", "a8:40:25:05:22:06"), + ("fe80::aa40:25ff:fe05:2207", "a8:40:25:05:22:07"), + ("fe80::aa40:25ff:fe05:2208", "a8:40:25:05:22:08"), + ("fe80::aa40:25ff:fe05:2209", "a8:40:25:05:22:09"), + ("fe80::aa40:25ff:fe05:220a", "a8:40:25:05:22:0a"), + ("fe80::aa40:25ff:fe05:220b", "a8:40:25:05:22:0b"), + ("fe80::aa40:25ff:fe05:220c", "a8:40:25:05:22:0c"), + ("fe80::aa40:25ff:fe05:220d", "a8:40:25:05:22:0d"), + ("fe80::aa40:25ff:fe05:220e", "a8:40:25:05:22:0e"), + ("fe80::aa40:25ff:fe05:220f", "a8:40:25:05:22:0f"), + ("fe80::aa40:25ff:fe05:2210", "a8:40:25:05:22:10"), + ("fe80::aa40:25ff:fe05:2211", "a8:40:25:05:22:11"), + ("fe80::aa40:25ff:fe05:2212", "a8:40:25:05:22:12"), + ("fe80::aa40:25ff:fe05:2213", "a8:40:25:05:22:13"), + ("fe80::aa40:25ff:fe05:2214", "a8:40:25:05:22:14"), + ("fe80::aa40:25ff:fe05:2215", "a8:40:25:05:22:15"), + ("fe80::aa40:25ff:fe05:2216", "a8:40:25:05:22:16"), + ("fe80::aa40:25ff:fe05:2217", "a8:40:25:05:22:17"), + ("fe80::aa40:25ff:fe05:2218", "a8:40:25:05:22:18"), + ("fe80::aa40:25ff:fe05:2219", "a8:40:25:05:22:19"), + ("fe80::aa40:25ff:fe05:221a", "a8:40:25:05:22:1a"), + ("fe80::aa40:25ff:fe05:221b", "a8:40:25:05:22:1b"), + ("fe80::aa40:25ff:fe05:221c", "a8:40:25:05:22:1c"), + ("fe80::aa40:25ff:fe05:221d", "a8:40:25:05:22:1d"), + ("fe80::aa40:25ff:fe05:221e", "a8:40:25:05:22:1e"), + ("fe80::aa40:25ff:fe05:221f", "a8:40:25:05:22:1f"), + ("fe80::aa40:25ff:fe05:2220", "a8:40:25:05:22:20"), + ("fe80::aa40:25ff:fe05:2221", "a8:40:25:05:22:21"), + ]; + for (ip_str, mac_str) in ndp_entries { + let ip: Ipv6Addr = ip_str.parse().unwrap(); + let mac: MacAddr = mac_str.parse().unwrap(); + let entry = types::ArpEntry { + ip: ip.into(), + mac: mac.into(), + tag: tag.clone(), + update: String::new(), + }; + switch.client.ndp_create(&entry).await.unwrap(); + } + + let packet_of_doom = Packet::parse(&[ + 0xa8, 0x40, 0x25, 0x05, 0x22, 0x23, 0xaa, 0x00, 0x04, 0x00, 0xca, 0xfe, + 0x08, 0x00, 0x45, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x00, 0x76, 0x01, + 0x6f, 0x84, 0x08, 0x08, 0x04, 0x04, 0xac, 0x14, 0x1d, 0x05, 0x00, 0x00, + 0x8e, 0x1a, 0x3f, 0xfd, 0x00, 0x1a, 0x93, 0xa7, 0x75, 0x69, 0x33, 0xba, + 0x0a, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, + 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, + 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, + 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, + 0x36, 0x37, + ]) + .unwrap(); + + // Build the inner payload: original packet with modified eth header + let inner_payload = { + let mut inner = packet_of_doom.clone(); + let eth = inner.hdrs.eth_hdr.as_mut().unwrap(); + eth.eth_smac = MacAddr::new(0, 0, 0, 0, 0, 0); + eth.eth_dmac = inner_mac; // NAT target inner_mac: a8:40:25:ff:c6:69 + inner.deparse().unwrap().to_vec() + }; + + // Build the encapsulated packet with IPv6/Geneve header + // Outer src: egress port MAC, loopback IP (fdd4:9500:e894:986c::1) + // Outer dst: next-hop MAC (gateway for fdf1:eb31:4d93:101::/64), NAT internal IP + let egress_port_mac = switch.get_port_mac(egress).unwrap().to_string(); + let next_hop_mac = "a8:40:25:04:03:51"; // MAC for gateway fe80::aa40:25ff:fe04:351 + + let mut packet_of_doom_encapsulated = common::gen_external_geneve_packet( + Endpoint::parse(&egress_port_mac, "::", geneve::GENEVE_UDP_PORT) + .unwrap(), + Endpoint::parse( + next_hop_mac, + "fdf1:eb31:4d93:101::1", // NAT internal IP + geneve::GENEVE_UDP_PORT, + ) + .unwrap(), + eth::ETHER_ETHER, + 100, // VNI + &inner_payload, + ); + + // Adjust for routing (decrement hop limit) + ipv6::Ipv6Hdr::adjust_hlim(&mut packet_of_doom_encapsulated, -1); + packet_of_doom_encapsulated.hdrs.udp_hdr.as_mut().unwrap().udp_sum = 0; + + switch.packet_test( + vec![TestPacket { packet: Arc::new(packet_of_doom), port: ingress }], + vec![TestPacket { + packet: Arc::new(packet_of_doom_encapsulated), + port: egress, + }], + ) +}