diff --git a/README.org b/README.org index 5d21f54..a487dcc 100644 --- a/README.org +++ b/README.org @@ -145,6 +145,7 @@ module: labn-munet-config | | | +--rw burst? number64 | | +--rw connections* [to] | | +--rw to string + | | +--rw from? string | | +--rw name? string | | +--rw remote-name? string | | +--rw delay? uint64 @@ -1154,14 +1155,19 @@ munet> list connections { key to; description - "Overriding network side configuration for connections to nodes from - this network. If the default network tc parameters are not being - overriden for a specific node connection then this configuration is - not required."; + "Per-node one-way constraints on this switch. to: N is traffic + toward that node (node RX). from: N is traffic from that node + (node TX). Use two entries for both directions."; leaf to { type string; - description "The target of this connection."; + description + "Shape traffic toward this node (switch egress, node RX)."; + } + leaf from { + type string; + description + "Shape traffic from this node (switch ingress, node TX)."; } leaf name { type string; diff --git a/munet/base.py b/munet/base.py index 8ff9c79..c44b2d4 100644 --- a/munet/base.py +++ b/munet/base.py @@ -1783,6 +1783,24 @@ def get_number(c, v, d=None): return netem_args, tbf_args + def _add_tc_qdiscs(self, nsifname, constraints): + netem_args, tbf_args = self.get_linux_tc_args(nsifname, constraints) + if not netem_args and not tbf_args: + return False + count = 1 + selector = f"root handle {count}:" + if netem_args: + self.cmd_raises( + f"tc qdisc add dev {nsifname} {selector} netem {netem_args}" + ) + count += 1 + selector = f"parent {count-1}: handle {count}" + # Place rate limit after delay otherwise limit/burst too complex + if tbf_args: + self.cmd_raises(f"tc qdisc add dev {nsifname} {selector} tbf {tbf_args}") + self.cmd_raises(f"tc qdisc show dev {nsifname}") + return True + def set_intf_constraints(self, ifname, **constraints): """Set interface outbound constraints. @@ -1801,21 +1819,32 @@ def set_intf_constraints(self, ifname, **constraints): rate (int): bits per second, string allows for use of {KMGTKiMiGiTi} prefixes "i" means K == 1024 otherwise K == 1000. """ + self._add_tc_qdiscs(self.get_ns_ifname(ifname), constraints) + + def set_intf_ingress_constraints(self, ifname, **constraints): + """Shape packets arriving on ifname via an IFB in this namespace. + + Used on a switch port so a node's delay/rate/loss still apply to that + node's outbound traffic (same as a qdisc on the node NIC) without + taking the node's root qdisc. + """ nsifname = self.get_ns_ifname(ifname) netem_args, tbf_args = self.get_linux_tc_args(nsifname, constraints) - count = 1 - selector = f"root handle {count}:" - if netem_args: - self.cmd_raises( - f"tc qdisc add dev {nsifname} {selector} netem {netem_args}" - ) - count += 1 - selector = f"parent {count-1}: handle {count}" - # Place rate limit after delay otherwise limit/burst too complex - if tbf_args: - self.cmd_raises(f"tc qdisc add dev {nsifname} {selector} tbf {tbf_args}") - - self.cmd_raises(f"tc qdisc show dev {nsifname}") + if not netem_args and not tbf_args: + return + ifb = "ifb" + re.sub(r"[^A-Za-z0-9]", "", nsifname) + ifb = ifb[:15] or "ifb0" + # Modules are host-global. The munet mount ns often has no + # /lib/modules, so modprobe inside it fails even when ifb is loaded. + commander.cmd_status("modprobe ifb", warn=False) + self.cmd_raises(f"ip link add {ifb} type ifb") + self.cmd_raises(f"ip link set {ifb} up") + self.cmd_raises(f"tc qdisc add dev {nsifname} handle ffff: ingress") + self.cmd_raises( + f"tc filter add dev {nsifname} parent ffff: protocol all " + f"u32 match u32 0 0 action mirred egress redirect dev {ifb}" + ) + self._add_tc_qdiscs(ifb, constraints) class LinuxNamespace(Commander, InterfaceMixin): diff --git a/munet/config.py b/munet/config.py index 8df91df..fe7a34f 100644 --- a/munet/config.py +++ b/munet/config.py @@ -31,8 +31,12 @@ def find_all_with_kv(lst, k, v): return rv -def find_matching_net_config(name, cconf, oconf): - p = find_all_with_kv(oconf.get("connections", {}), "to", name) +def find_matching_net_config(name, cconf, oconf, direction="to"): + """Return the peer's connection entry facing this node. + + direction is "to" (toward the node) or "from" (from the node). + """ + p = find_all_with_kv(oconf.get("connections", {}), direction, name) if not p: return {} diff --git a/munet/munet-schema.json b/munet/munet-schema.json index b61ce5c..9638cd3 100644 --- a/munet/munet-schema.json +++ b/munet/munet-schema.json @@ -484,6 +484,9 @@ "to": { "type": "string" }, + "from": { + "type": "string" + }, "name": { "type": "string" }, diff --git a/munet/native.py b/munet/native.py index bf1bfae..be5b9dc 100644 --- a/munet/native.py +++ b/munet/native.py @@ -3282,15 +3282,26 @@ async def _async_build(self, logger=None): # default tc values for interfaces added to the bridge which aren't # present in `connections`. switch = self.switches[to] - swconf = find_matching_net_config(name, cconf, switch.config) - if not swconf: + sw_to = find_matching_net_config( + name, cconf, switch.config, "to" + ) + sw_from = find_matching_net_config( + name, cconf, switch.config, "from" + ) + if not sw_to: # "name" most important key to leave out, so it gets generated - nontc = ("connections", "external", "ip", "ipv6", "name") - swconf = { - k: v for k, v in switch.config.items() if k not in nontc + nontc = ( + "connections", "external", "ip", "ipv6", "name", + ) + sw_to = { + k: v + for k, v in switch.config.items() + if k not in nontc } - swconf = deepcopy(swconf) - await self.add_native_link(switch, node, swconf, cconf) + sw_to = deepcopy(sw_to) + await self.add_native_link( + switch, node, sw_to, cconf, c1_from=sw_from + ) elif cconf["name"] not in node.intfs: # Only add the p2p interface if not already there. other = self.hosts[to] @@ -3332,8 +3343,40 @@ async def add_dummy_link(self, node1, c1=None): super().add_dummy(node1, if1, **c1) node1.set_dummy_addr(c1) - async def add_native_link(self, node1, node2, c1=None, c2=None): - """Add a link between switch and node or 2 nodes.""" + _TC_KEYS = ( + "delay", + "jitter", + "jitter-correlation", + "loss", + "loss-correlation", + "rate", + ) + + @staticmethod + def _tc_pick(config): + """Copy linux TC keys from config.""" + if not config: + return {} + out = {} + for key in Munet._TC_KEYS: + if config.get(key) is not None: + out[key] = config[key] + return out + + @staticmethod + def _tc_constraints(*configs): + """Merge unprefixed TC keys; later configs override earlier ones.""" + out = {} + for config in configs: + out.update(Munet._tc_pick(config)) + return out + + async def add_native_link(self, node1, node2, c1=None, c2=None, c1_from=None): + """Add a link between switch and node or 2 nodes. + + c1_from is switch ``from:`` (host TX / IFB). It is always the host + node's outbound constraints, even if the node is passed first. + """ isp2p = False c1 = {} if c1 is None else c1 @@ -3392,10 +3435,43 @@ async def add_native_link(self, node1, node2, c1=None, c2=None): if isinstance(node1, ExternalNetwork): pass - elif "physical" not in c1 and not node1.is_vm: - node1.set_intf_constraints(if1, **c1) - if "physical" not in c2 and not node2.is_vm: - node2.set_intf_constraints(if2, **c2) + elif isp2p: + # Both veth ends live inside the nodes. There is no outside + # device to own the "link" unless we insert a mid-netns. + if "physical" not in c1 and not node1.is_vm: + tx = Munet._tc_constraints(c1) + if tx: + self.logger.warning( + "%s: p2p constraints on %s:%s stay inside the node", + self, + node1.name, + if1, + ) + node1.set_intf_constraints(if1, **tx) + if "physical" not in c2 and not node2.is_vm: + tx = Munet._tc_constraints(c2) + if tx: + self.logger.warning( + "%s: p2p constraints on %s:%s stay inside the node", + self, + node2.name, + if2, + ) + node2.set_intf_constraints(if2, **tx) + elif "physical" not in c2: + # Host-to-switch (switch-centric to/from): + # to: r1 + delay → into r1 (node RX, switch egress) + # from: r1 + delay → from r1 (node TX, IFB on switch ingress) + # Node delay still shapes TX if set (same as from:). + node_tx = { + **Munet._tc_constraints(c1_from), + **Munet._tc_constraints(c2), + } + node_rx = Munet._tc_constraints(c1) + if node_rx: + node1.set_intf_constraints(if1, **node_rx) + if node_tx: + node1.set_intf_ingress_constraints(if1, **node_tx) def add_l3_node(self, name, config=None, **kwargs): """Add a node to munet.""" diff --git a/tests/basic/test_basic_constraints.py b/tests/basic/test_basic_constraints.py index d7ce581..61b812b 100644 --- a/tests/basic/test_basic_constraints.py +++ b/tests/basic/test_basic_constraints.py @@ -13,6 +13,7 @@ import pytest from munet import Munet +from munet.config import find_matching_net_config # All tests are coroutines pytestmark = pytest.mark.asyncio @@ -31,6 +32,18 @@ async def unet_(request, rundir_module, pytestconfig): await unet.async_delete() +@pytest.fixture(scope="function", name="unet_case") +async def unet_case_(request, rundir, pytestconfig): + """Per-test munet so switch/node names do not collide with other cases.""" + unshare = bool(request.param) if hasattr(request, "param") else True + logging.info("Creating munet with%s inline unshare", "" if unshare else "out") + unet = Munet(rundir=rundir, unshare_inline=unshare, pytestconfig=pytestconfig) + try: + yield unet + finally: + await unet.async_delete() + + async def ping_average_rtt(r, other, oifname): oip = other.get_intf_addr(oifname).ip await r.async_cmd_raises(f"ping -w1 -c1 {oip}") @@ -99,3 +112,149 @@ async def test_basic_ping(unet): loss = await ping_with_loss(r2, r3, ifname) logging.info("ping loss: %s%%", loss) assert 20 < loss < 40 + + +@pytest.mark.parametrize("unet_case", [False, True], indirect=["unet_case"]) +async def test_switch_constraints_outside_node(unet_case): + """Node delay/rate stay TX-shaped, but the qdisc lives in the switch ns.""" + unet = unet_case + unet.autonumber = True + + r1 = unet.add_l3_node("r1") + r2 = unet.add_l3_node("r2") + sw1 = unet.add_network("sw1", {"ip": "auto"}) + + delay = 50000 + await unet.add_native_link(sw1, r1, {}, {"delay": delay}) + await unet.add_native_link(sw1, r2, {}, {"delay": delay}) + + rif1 = r1.net_intfs[sw1.name] + rif2 = r2.net_intfs[sw1.name] + sif1 = sw1.intfs[0] + sif2 = sw1.intfs[1] + + r1q = await r1.async_cmd_raises(f"tc qdisc show dev {rif1}") + r2q = await r2.async_cmd_raises(f"tc qdisc show dev {rif2}") + s1q = await sw1.async_cmd_raises(f"tc qdisc show dev {sif1}") + s2q = await sw1.async_cmd_raises(f"tc qdisc show dev {sif2}") + swq = await sw1.async_cmd_raises("tc qdisc show") + logging.info("node qdiscs: %s | %s", r1q, r2q) + logging.info("switch port qdiscs: %s | %s", s1q, s2q) + logging.info("switch ns qdiscs: %s", swq) + assert "netem" not in r1q + assert "netem" not in r2q + assert "ingress" in s1q + assert "ingress" in s2q + assert "netem" not in s1q + assert "netem" not in s2q + assert "netem" in swq + + exp_avg = (delay + delay) / 1000 + avg = await ping_average_rtt(r1, r2, rif2) + logging.info("ping average RTT: %s", avg) + assert (exp_avg - 1) < avg < (exp_avg + 2) + + +@pytest.mark.parametrize("unet_case", [False, True], indirect=["unet_case"]) +async def test_switch_to_and_from(unet_case): + """Switch to: is node RX; from: is node TX (IFB). Together they are RTT.""" + unet = unet_case + unet.autonumber = True + + r1 = unet.add_l3_node("r1") + r2 = unet.add_l3_node("r2") + sw1 = unet.add_network("sw1", {"ip": "auto"}) + + delay = 50000 + await unet.add_native_link( + sw1, r1, {"delay": delay}, {}, c1_from={"delay": delay} + ) + await unet.add_native_link( + sw1, r2, {"delay": delay}, {}, c1_from={"delay": delay} + ) + + rif2 = r2.net_intfs[sw1.name] + sif1 = sw1.intfs[0] + sif2 = sw1.intfs[1] + r1q = await r1.async_cmd_raises( + f"tc qdisc show dev {r1.net_intfs[sw1.name]}" + ) + s1q = await sw1.async_cmd_raises(f"tc qdisc show dev {sif1}") + s2q = await sw1.async_cmd_raises(f"tc qdisc show dev {sif2}") + swq = await sw1.async_cmd_raises("tc qdisc show") + logging.info("node qdisc: %s", r1q) + logging.info("switch ports: %s | %s", s1q, s2q) + logging.info("switch ns: %s", swq) + assert "netem" not in r1q + assert "ingress" in s1q and "netem" in s1q + assert "ingress" in s2q and "netem" in s2q + assert "ifb" in swq + + # TX + RX on each hop: 4 * 50ms + exp_avg = (delay * 4) / 1000 + avg = await ping_average_rtt(r1, r2, rif2) + logging.info("ping average RTT: %s", avg) + assert (exp_avg - 1) < avg < (exp_avg + 2) + + +def test_find_matching_net_to_from(): + """Switch connections match separately on to: and from:.""" + sw = { + "connections": [ + {"to": "r1", "delay": 1000, "rate": {"rate": "15M"}}, + {"from": "r1", "delay": 2000, "rate": {"rate": "30M"}}, + {"to": "r2", "delay": 3000}, + ] + } + cconf = {"name": "eth0"} + toward = find_matching_net_config("r1", cconf, sw, "to") + leaving = find_matching_net_config("r1", cconf, sw, "from") + assert toward["delay"] == 1000 + assert toward["rate"]["rate"] == "15M" + assert leaving["delay"] == 2000 + assert leaving["rate"]["rate"] == "30M" + assert find_matching_net_config("r2", cconf, sw, "from") == {} + assert find_matching_net_config("r3", cconf, sw, "to") == {} + assert Munet._tc_constraints(None) == {} + tx = Munet._tc_constraints(leaving) + rx = Munet._tc_constraints(toward) + assert tx["delay"] == 2000 + assert rx["delay"] == 1000 + assert "name" not in tx + # Node delay overrides switch from: on the same TX pipe. + assert Munet._tc_constraints(leaving, {"delay": 100})["delay"] == 100 + + +@pytest.mark.parametrize("unet_case", [False, True], indirect=["unet_case"]) +async def test_switch_to_delay_only(unet_case): + """Switch to: shapes RX on switch egress (no IFB).""" + unet = unet_case + unet.autonumber = True + + r1 = unet.add_l3_node("r1") + r2 = unet.add_l3_node("r2") + sw1 = unet.add_network("sw1", {"ip": "auto"}) + + delay = 50000 + await unet.add_native_link(sw1, r1, {"delay": delay}, {}) + await unet.add_native_link(sw1, r2, {"delay": delay}, {}) + + rif2 = r2.net_intfs[sw1.name] + sif1 = sw1.intfs[0] + sif2 = sw1.intfs[1] + r1q = await r1.async_cmd_raises(f"tc qdisc show dev {r1.net_intfs[sw1.name]}") + s1q = await sw1.async_cmd_raises(f"tc qdisc show dev {sif1}") + s2q = await sw1.async_cmd_raises(f"tc qdisc show dev {sif2}") + swq = await sw1.async_cmd_raises("tc qdisc show") + logging.info("node qdisc: %s", r1q) + logging.info("switch ports: %s | %s", s1q, s2q) + logging.info("switch ns: %s", swq) + assert "netem" not in r1q + assert "ingress" not in s1q and "netem" in s1q + assert "ingress" not in s2q and "netem" in s2q + assert "ifb" not in swq + + exp_avg = (delay + delay) / 1000 + avg = await ping_average_rtt(r1, r2, rif2) + logging.info("ping average RTT: %s", avg) + assert (exp_avg - 1) < avg < (exp_avg + 2) diff --git a/tests/config/qdisc/test_qdisc_cmd.py b/tests/config/qdisc/test_qdisc_cmd.py index 00184e0..f31726c 100644 --- a/tests/config/qdisc/test_qdisc_cmd.py +++ b/tests/config/qdisc/test_qdisc_cmd.py @@ -38,19 +38,31 @@ async def test_config_cmd(unet_share): h1 = unet.hosts["h1"] output = h1.cmd_raises("tc q | grep 'dev eth0'") - logging.debug("qdisc for dev eth0 output found: %s", output) - logging.debug("expects delay='200', jitter='60', loss='70', rate='1600'") + logging.debug("qdisc for h1 eth0 found: %s", output) + logging.debug("node delay is on the switch IFB, not the node NIC") + assert "delay" not in output + assert "loss" not in output + assert "rate" not in output + + net1 = unet.switches["net1"] + output = net1.cmd_raises("tc q | grep 'dev ifbnet1e0'") + logging.debug("qdisc for ifbnet1e0 (h1 TX) found: %s", output) assert re.search(r"delay 200us\s+59us.*loss 70%.*rate 1600bit", output, re.DOTALL) h2 = unet.hosts["h2"] output = h2.cmd_raises("tc q | grep 'dev eth0'") - logging.debug("qdisc for dev eth0 output found: %s", output) - logging.debug("expects delay='209', jitter='60', loss='70', rate='1600'") + logging.debug("qdisc for h2 eth0 found: %s", output) + assert "delay" not in output + assert "loss" not in output + assert "rate" not in output + + output = net1.cmd_raises("tc q | grep 'dev ifbnet1e1'") + logging.debug("qdisc for ifbnet1e1 (h2 TX) found: %s", output) assert re.search(r"delay 209us\s+59us.*loss 70%.*rate 1600bit", output, re.DOTALL) h3 = unet.hosts["h3"] output = h3.cmd_raises("tc q | grep 'dev eth0'") - logging.debug("qdisc for dev eth0 output found: %s", output) + logging.debug("qdisc for h3 eth0 found: %s", output) logging.debug("expects none") assert "delay" not in output assert "loss" not in output