Skip to content

attempt to reproduce packet corruption issue from london #198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rcgoodfellow wants to merge 2 commits into
base: main
Choose a base branch
from
+350 −0
Draft

attempt to reproduce packet corruption issue from london #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7fcaf09
attempt to reproduce packet corruption issue from london
rcgoodfellow Jan 23, 2026
7af968d
add london bridges tests
rcgoodfellow Jan 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 97 additions & 0 deletions dpd-client/tests/integration_tests/local.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/
//
// Copyright 2026 Oxide Computer Company

use crate::integration_tests::common::{
PhysPort, SERVICE_PORT, TestPacket, TestResult, get_switch,
};
use dpd_client::ClientInfo;
use dpd_client::types::Ipv4Entry;
use packet::{
Packet,
eth::{ETHER_IPV4, ETHER_SIDECAR},
sidecar::{SC_FWD_TO_USERSPACE, SidecarHdr},
};
use std::sync::Arc;

// This test attempts to reproduce an issue we saw on london whre a TCP SYN
// packet destined for the switch zone has it's source port altered when
// going through the ASIC. This causes a checksum failure at the host and
// torpedoes the SSH session. This condition lasted for nearly half a day
// and then all of a sudden vanished and communications started working.
//
// The before tofino and after tofino packet dumps are the following.
// The source port corruption is highlighted by the carats.
//
// before:
//
// 0000 a8 40 25 05 02 23 aa 00 04 00 ca fe 08 00 45 c0
// 0010 00 3c da 57 40 00 01 06 28 44 ac 14 0f 1b ac 14
// 0020 0f 1d ac 31 00 b3 84 f4 27 61 00 00 00 00 a0 02
// ^^^^^
// 0030 83 2c a1 20 00 00 02 04 05 b4 04 02 08 0a b9 a7
// 0040 9a 76 00 00 00 00 01 03 03 01
//
// after:
//
// 0000 a8 40 25 05 02 23 aa 00 04 00 ca fe 08 00 45 c0
// 0010 00 3c da 57 40 00 01 06 28 44 ac 14 0f 1b ac 14
// 0020 0f 1d a0 01 00 b3 84 f4 27 61 00 00 00 00 a0 02
// ^^^^^
// 0030 83 2c a1 20 00 00 02 04 05 b4 04 02 08 0a b9 a7
// 0040 9a 76 00 00 00 00 01 03 03 01
//
// This test sets up the local switch address 172.20.15.29 and sends the
// packet we had trouble with through the ASIC expecting it to come out
// the other side in tact with a sidecar header. If this test passes it
// is not presenting the issue we saw on london.

#[tokio::test]
#[ignore]
async fn bgp_syn_of_doom() -> TestResult {
let switch = &*get_switch().await;
let ingress = PhysPort(10);

let (port_id, link_id) = switch.link_id(ingress).unwrap();
let entry = Ipv4Entry {
addr: "172.20.15.29".parse().unwrap(),
tag: switch.client.inner().tag.clone(),
};
switch.client.link_ipv4_create(&port_id, &link_id, &entry).await.unwrap();

let syn_of_doom_before = Packet::parse(&[
0xa8, 0x40, 0x25, 0x5, 0x2, 0x23, 0xaa, 0x0, 0x4, 0x0, 0xca, 0xfe, 0x8,
0x0, 0x45, 0xc0, 0x0, 0x3c, 0xda, 0x57, 0x40, 0x0, 0x1, 0x6, 0x28,
0x44, 0xac, 0x14, 0xf, 0x1b, 0xac, 0x14, 0xf, 0x1d, 0xac, 0x31, 0x0,
0xb3, 0x84, 0xf4, 0x27, 0x61, 0x0, 0x0, 0x0, 0x0, 0xa0, 0x2, 0x83,
0x2c, 0xa1, 0x20, 0x0, 0x0, 0x2, 0x4, 0x5, 0xb4, 0x4, 0x2, 0x8, 0xa,
0xb9, 0xa7, 0x9a, 0x76, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x3, 0x1,
])
.unwrap();

let mut syn_of_doom_after = syn_of_doom_before.clone();
if let Some(eth) = &mut syn_of_doom_after.hdrs.eth_hdr {
eth.eth_type = ETHER_SIDECAR;
}
syn_of_doom_after.hdrs.sidecar_hdr = Some(SidecarHdr {
sc_code: SC_FWD_TO_USERSPACE,
sc_pad: 0,
sc_ingress: 0x90,
sc_egress: 0,
sc_ether_type: ETHER_IPV4,
sc_payload: [0u8; 16],
});

switch.packet_test(
vec![TestPacket {
packet: Arc::new(syn_of_doom_before),
port: ingress,
}],
vec![TestPacket {
packet: Arc::new(syn_of_doom_after),
port: SERVICE_PORT,
}],
)
}
1 change: 1 addition & 0 deletions dpd-client/tests/integration_tests/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ mod common;
mod counters;
mod geneve;
mod icmp_ipv4;
mod local;
mod loopback;
mod mcast;
mod nat;
Expand Down
252 changes: 252 additions & 0 deletions dpd-client/tests/integration_tests/nat.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ use std::net::Ipv6Addr;
use std::sync::Arc;

use anyhow::anyhow;
use oxnet::Ipv4Net;
use oxnet::Ipv6Net;

use ::common::network::MacAddr;
use ::common::network::Vni;
use dpd_client::ClientInfo;
use dpd_client::types;
use packet::Endpoint;
use packet::Packet;
use packet::eth;
use packet::geneve;
use packet::icmp;
Expand Down Expand Up @@ -678,3 +680,253 @@ async fn test_ingress_ipv6_tcp() -> TestResult {
let switch = &*get_switch().await;
test_ingress_ipv6(switch, L4Protocol::Tcp).await
}

#[tokio::test]
#[ignore]
async fn test_london_bridges_falling_down() -> TestResult {
let switch = &*get_switch().await;
let ingress = PhysPort(8);
let egress = PhysPort(9);

let link_id = types::LinkId(0);
let tag = switch.client.inner().tag.clone();

// ADDRESS: Add IPv6 addresses to rear ports
let rear_addrs: [(u8, &str); 30] = [
(0, "fe80::aa40:25ff:fe05:2203"),
(1, "fe80::aa40:25ff:fe05:2204"),
(2, "fe80::aa40:25ff:fe05:2205"),
(3, "fe80::aa40:25ff:fe05:2206"),
(4, "fe80::aa40:25ff:fe05:2207"),
(5, "fe80::aa40:25ff:fe05:2208"),
(6, "fe80::aa40:25ff:fe05:2209"),
(7, "fe80::aa40:25ff:fe05:220a"),
(8, "fe80::aa40:25ff:fe05:220b"),
(9, "fe80::aa40:25ff:fe05:220c"),
(10, "fe80::aa40:25ff:fe05:220d"),
(11, "fe80::aa40:25ff:fe05:220e"),
(12, "fe80::aa40:25ff:fe05:220f"),
(13, "fe80::aa40:25ff:fe05:2210"),
(14, "fe80::aa40:25ff:fe05:2211"),
(15, "fe80::aa40:25ff:fe05:2212"),
(16, "fe80::aa40:25ff:fe05:2213"),
(17, "fe80::aa40:25ff:fe05:2214"),
(18, "fe80::aa40:25ff:fe05:2215"),
(19, "fe80::aa40:25ff:fe05:2216"),
(20, "fe80::aa40:25ff:fe05:2217"),
(21, "fe80::aa40:25ff:fe05:2218"),
(22, "fe80::aa40:25ff:fe05:2219"),
(23, "fe80::aa40:25ff:fe05:221a"),
(24, "fe80::aa40:25ff:fe05:221b"),
(25, "fe80::aa40:25ff:fe05:221c"),
(26, "fe80::aa40:25ff:fe05:221d"),
(27, "fe80::aa40:25ff:fe05:221e"),
(28, "fe80::aa40:25ff:fe05:221f"),
(29, "fe80::aa40:25ff:fe05:2220"),
];
for (rear_num, addr_str) in rear_addrs {
let port_id: types::PortId =
format!("rear{}", rear_num).parse().unwrap();
let addr: Ipv6Addr = addr_str.parse().unwrap();
let entry = types::Ipv6Entry { addr, tag: tag.clone() };
switch
.client
.link_ipv6_create(&port_id, &link_id, &entry)
.await
.unwrap();
}

// ADDRESS: Add IPv4 address to qsfp0
// NOTE: qsfp0 does not appear to be a thing with the simulator setup....
// so use rear19?
let qsfp0_port_id: types::PortId = "rear19".parse().unwrap();
let qsfp0_ipv4: Ipv4Addr = "172.20.15.61".parse().unwrap();
let entry = types::Ipv4Entry { addr: qsfp0_ipv4, tag: tag.clone() };
switch
.client
.link_ipv4_create(&qsfp0_port_id, &link_id, &entry)
.await
.unwrap();

// NAT: Create NAT entry
let nat_ip: Ipv4Addr = "172.20.29.5".parse().unwrap();
let internal_ip: Ipv6Addr = "fdf1:eb31:4d93:101::1".parse().unwrap();
let inner_mac: MacAddr = "a8:40:25:ff:c6:69".parse().unwrap();
let vni = Vni::new(100).unwrap();
let tgt = types::NatTarget {
internal_ip,
inner_mac: inner_mac.into(),
vni: vni.into(),
};
switch.client.nat_ipv4_create(&nat_ip, 0, 16383, &tgt).await.unwrap();

// ROUTE: IPv4 default route via qsfp0
let ipv4_cidr: Ipv4Net = "0.0.0.0/0".parse().unwrap();
let ipv4_route = types::Ipv4RouteUpdate {
cidr: ipv4_cidr,
target: types::Ipv4Route {
port_id: qsfp0_port_id.clone(),
link_id: link_id.clone(),
tgt_ip: "172.20.15.57".parse().unwrap(),
tag: tag.clone(),
vlan_id: None,
},
replace: false,
};
switch.client.route_ipv4_set(&ipv4_route).await.unwrap();

// ROUTE: IPv6 routes
let ipv6_routes: [(&str, &str, &str); 11] = [
("fdb0:a840:2504:351::/64", "rear11", "fe80::aa40:25ff:fe04:351"),
("fdb0:a840:2504:614::/64", "rear15", "fe80::aa40:25ff:fe04:614"),
("fdb0:a840:2504:6d3::/64", "rear16", "fe80::aa40:25ff:fe04:6d3"),
("fdb0:a840:2504:851::/64", "rear17", "fe80::aa40:25ff:fe04:851"),
("fdf1:eb31:4d93:1::/64", "rear14", "fe80::aa40:25ff:fe04:351"),
("fdf1:eb31:4d93:2::/64", "rear15", "fe80::aa40:25ff:fe04:614"),
("fdf1:eb31:4d93:3::/64", "rear16", "fe80::aa40:25ff:fe04:6d3"),
("fdf1:eb31:4d93:101::/64", "rear11", "fe80::aa40:25ff:fe04:351"),
("fdf1:eb31:4d93:102::/64", "rear15", "fe80::aa40:25ff:fe04:614"),
("fdf1:eb31:4d93:103::/64", "rear16", "fe80::aa40:25ff:fe04:6d3"),
("fdf1:eb31:4d93:104::/64", "rear17", "fe80::aa40:25ff:fe04:851"),
];
for (cidr_str, port_str, gw_str) in ipv6_routes {
let cidr: Ipv6Net = cidr_str.parse().unwrap();
let port_id: types::PortId = port_str.parse().unwrap();
let tgt_ip: Ipv6Addr = gw_str.parse().unwrap();
let route = types::Ipv6RouteUpdate {
cidr,
target: types::Ipv6Route {
port_id,
link_id: link_id.clone(),
tgt_ip,
tag: tag.clone(),
vlan_id: None,
},
replace: false,
};
switch.client.route_ipv6_set(&route).await.unwrap();
}

// ARP: IPv4 entries
let arp_entries: [(&str, &str); 3] = [
("172.20.15.57", "aa:00:04:00:ca:fe"),
("172.20.15.59", "aa:00:04:00:ca:fe"),
("172.20.15.61", "a8:40:25:05:22:23"),
];
for (ip_str, mac_str) in arp_entries {
let ip: Ipv4Addr = ip_str.parse().unwrap();
let mac: MacAddr = mac_str.parse().unwrap();
let entry = types::ArpEntry {
ip: ip.into(),
mac: mac.into(),
tag: tag.clone(),
update: String::new(),
};
switch.client.arp_create(&entry).await.unwrap();
}

// NDP: IPv6 entries
let ndp_entries: [(&str, &str); 36] = [
("fe80::aa40:25ff:fe04:351", "a8:40:25:04:03:51"),
("fe80::aa40:25ff:fe04:614", "a8:40:25:04:06:14"),
("fe80::aa40:25ff:fe04:6d3", "a8:40:25:04:06:d3"),
("fe80::aa40:25ff:fe04:851", "a8:40:25:04:08:51"),
("fe80::aa40:25ff:fe05:2202", "a8:40:25:05:22:02"),
("fe80::aa40:25ff:fe05:2203", "a8:40:25:05:22:03"),
("fe80::aa40:25ff:fe05:2204", "a8:40:25:05:22:04"),
("fe80::aa40:25ff:fe05:2205", "a8:40:25:05:22:05"),
("fe80::aa40:25ff:fe05:2206", "a8:40:25:05:22:06"),
("fe80::aa40:25ff:fe05:2207", "a8:40:25:05:22:07"),
("fe80::aa40:25ff:fe05:2208", "a8:40:25:05:22:08"),
("fe80::aa40:25ff:fe05:2209", "a8:40:25:05:22:09"),
("fe80::aa40:25ff:fe05:220a", "a8:40:25:05:22:0a"),
("fe80::aa40:25ff:fe05:220b", "a8:40:25:05:22:0b"),
("fe80::aa40:25ff:fe05:220c", "a8:40:25:05:22:0c"),
("fe80::aa40:25ff:fe05:220d", "a8:40:25:05:22:0d"),
("fe80::aa40:25ff:fe05:220e", "a8:40:25:05:22:0e"),
("fe80::aa40:25ff:fe05:220f", "a8:40:25:05:22:0f"),
("fe80::aa40:25ff:fe05:2210", "a8:40:25:05:22:10"),
("fe80::aa40:25ff:fe05:2211", "a8:40:25:05:22:11"),
("fe80::aa40:25ff:fe05:2212", "a8:40:25:05:22:12"),
("fe80::aa40:25ff:fe05:2213", "a8:40:25:05:22:13"),
("fe80::aa40:25ff:fe05:2214", "a8:40:25:05:22:14"),
("fe80::aa40:25ff:fe05:2215", "a8:40:25:05:22:15"),
("fe80::aa40:25ff:fe05:2216", "a8:40:25:05:22:16"),
("fe80::aa40:25ff:fe05:2217", "a8:40:25:05:22:17"),
("fe80::aa40:25ff:fe05:2218", "a8:40:25:05:22:18"),
("fe80::aa40:25ff:fe05:2219", "a8:40:25:05:22:19"),
("fe80::aa40:25ff:fe05:221a", "a8:40:25:05:22:1a"),
("fe80::aa40:25ff:fe05:221b", "a8:40:25:05:22:1b"),
("fe80::aa40:25ff:fe05:221c", "a8:40:25:05:22:1c"),
("fe80::aa40:25ff:fe05:221d", "a8:40:25:05:22:1d"),
("fe80::aa40:25ff:fe05:221e", "a8:40:25:05:22:1e"),
("fe80::aa40:25ff:fe05:221f", "a8:40:25:05:22:1f"),
("fe80::aa40:25ff:fe05:2220", "a8:40:25:05:22:20"),
("fe80::aa40:25ff:fe05:2221", "a8:40:25:05:22:21"),
];
for (ip_str, mac_str) in ndp_entries {
let ip: Ipv6Addr = ip_str.parse().unwrap();
let mac: MacAddr = mac_str.parse().unwrap();
let entry = types::ArpEntry {
ip: ip.into(),
mac: mac.into(),
tag: tag.clone(),
update: String::new(),
};
switch.client.ndp_create(&entry).await.unwrap();
}

let packet_of_doom = Packet::parse(&[
0xa8, 0x40, 0x25, 0x05, 0x22, 0x23, 0xaa, 0x00, 0x04, 0x00, 0xca, 0xfe,
0x08, 0x00, 0x45, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x00, 0x76, 0x01,
0x6f, 0x84, 0x08, 0x08, 0x04, 0x04, 0xac, 0x14, 0x1d, 0x05, 0x00, 0x00,
0x8e, 0x1a, 0x3f, 0xfd, 0x00, 0x1a, 0x93, 0xa7, 0x75, 0x69, 0x33, 0xba,
0x0a, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11,
0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d,
0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29,
0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35,
0x36, 0x37,
])
.unwrap();

// Build the inner payload: original packet with modified eth header
let inner_payload = {
let mut inner = packet_of_doom.clone();
let eth = inner.hdrs.eth_hdr.as_mut().unwrap();
eth.eth_smac = MacAddr::new(0, 0, 0, 0, 0, 0);
eth.eth_dmac = inner_mac; // NAT target inner_mac: a8:40:25:ff:c6:69
inner.deparse().unwrap().to_vec()
};

// Build the encapsulated packet with IPv6/Geneve header
// Outer src: egress port MAC, loopback IP (fdd4:9500:e894:986c::1)
// Outer dst: next-hop MAC (gateway for fdf1:eb31:4d93:101::/64), NAT internal IP
let egress_port_mac = switch.get_port_mac(egress).unwrap().to_string();
let next_hop_mac = "a8:40:25:04:03:51"; // MAC for gateway fe80::aa40:25ff:fe04:351

let mut packet_of_doom_encapsulated = common::gen_external_geneve_packet(
Endpoint::parse(&egress_port_mac, "::", geneve::GENEVE_UDP_PORT)
.unwrap(),
Endpoint::parse(
next_hop_mac,
"fdf1:eb31:4d93:101::1", // NAT internal IP
geneve::GENEVE_UDP_PORT,
)
.unwrap(),
eth::ETHER_ETHER,
100, // VNI
&inner_payload,
);

// Adjust for routing (decrement hop limit)
ipv6::Ipv6Hdr::adjust_hlim(&mut packet_of_doom_encapsulated, -1);
packet_of_doom_encapsulated.hdrs.udp_hdr.as_mut().unwrap().udp_sum = 0;

switch.packet_test(
vec![TestPacket { packet: Arc::new(packet_of_doom), port: ingress }],
vec![TestPacket {
packet: Arc::new(packet_of_doom_encapsulated),
port: egress,
}],
)
}