From b979062613e853f3db5132fe7d6c8e64cd183af2 Mon Sep 17 00:00:00 2001 From: Gabriel Lluch Date: Sat, 1 Aug 2026 14:55:58 -0700 Subject: [PATCH] feat: ask devices what they are over mDNS instead of guessing from vendor --- README.md | 15 +- netdiff/audit.py | 2 +- netdiff/cli.py | 26 ++- netdiff/diff.py | 10 +- netdiff/mdns.py | 308 +++++++++++++++++++++++++++++++++++ netdiff/scan.py | 11 +- netdiff/store.py | 46 +++++- tests/test_mdns.py | 280 +++++++++++++++++++++++++++++++ tests/test_scan_and_store.py | 91 ++++++++++- 9 files changed, 772 insertions(+), 17 deletions(-) create mode 100644 netdiff/mdns.py create mode 100644 tests/test_mdns.py diff --git a/README.md b/README.md index fe8a40b..ce321c3 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,10 @@ Then `netdiff audit` asks the question those tools do not: **which of these is r ```console $ netdiff scan 192.168.1.0/24 scan 7: 12 device(s) on 192.168.1.0/24 - 192.168.1.1 00:1d:c9:0a:1b:2c router.local ports 53,80,443 - 192.168.1.23 b8:27:eb:aa:bb:cc Raspberry Pi ports 22 + 192.168.1.1 00:1d:c9:0a:1b:2c router.local ports 53,80,443 + 192.168.1.23 b8:27:eb:aa:bb:cc Raspberry Pi (SSH, Web interface) ports 22 + 192.168.1.64 54:60:09:11:22:33 Google (Chromecast) + 192.168.1.71 d8:3a:dd:aa:bb:cc Mac15,7, AirPlay ... changes since last scan: 1 appeared, 1 port-opened @@ -70,6 +72,8 @@ Findings are recorded alongside scans, so a repeat audit marks what is `[NEW]` s | `upnp-control-open` | medium | The router answered an unauthenticated control request - so would it for anything else on the LAN | | `open-ports-noted` | info | Explicitly **not** a problem. See below. | +**And a vendor is not a device type.** "Espressif" covers a smart plug, a doorbell and someone's weekend project equally, so a MAC lookup alone leaves the most useful column nearly empty. Rather than guess a device type from its open ports - which is how the tool netdiff replaced arrived at "Managed Web Server" for a printer - netdiff asks the network the question every phone on it asks continuously, and reads the answer. Chromecasts, printers, Sonos, HomeKit gear and Apple devices all announce their services over multicast DNS, unprompted, to anyone on the segment. `Mac15,7` in the output is the device's own word for itself, not an inference. A device that announces nothing is left blank, because not knowing is the normal case. + **A port number is not evidence of a protocol either.** Port 23 being open does not prove telnet is behind it. So for services that greet you unprompted - FTP, Telnet, VNC - netdiff will not name the protocol until it has heard the greeting. RTSP and MQTT say nothing until spoken to, so there the evidence line states plainly that the identification is by port assignment, and the `verify` command lets you settle it. **An open port is not a vulnerability.** It is what a working device looks like. Tools that list every open port under a heading like "vulnerabilities found" are counting furniture and calling it a fire, and they train you to ignore the report. netdiff counts open ports and says out loud that they are not findings. A port becomes interesting when the protocol behind it is unencrypted, when it is reachable from outside the network, or when the software behind it is known-broken - and those are the rules above. @@ -106,6 +110,7 @@ Python 3.9+. Nothing else - `pip show netdiff` lists no dependencies, and CI ass ```bash netdiff scan 192.168.1.0/24 # scan, record, report changes netdiff scan 192.168.1.0/24 --no-ports # discovery only, no TCP connections +netdiff scan 192.168.1.0/24 --no-mdns # skip asking devices what they are netdiff audit 192.168.1.0/24 # what this network exposes, and why it matters netdiff inventory # every device ever seen, first and last sighting netdiff history # diff the two most recent scans @@ -154,7 +159,7 @@ History lives in `~/.netdiff/history.db` (override with `--db`). It is a plain S ## Scope -Only scan networks you are responsible for. netdiff is deliberately read-only - it sends empty UDP datagrams, completes TCP handshakes, reads banners services volunteer, and asks the router to list its own port forwards. It never writes to a host, never authenticates, and never changes router configuration. Even so, scanning equipment you do not own is your problem, not the tool's. +Only scan networks you are responsible for. netdiff is deliberately read-only - it sends empty UDP datagrams, completes TCP handshakes, reads banners services volunteer, asks the standard DNS-SD question over multicast and reads the replies, and asks the router to list its own port forwards. It never writes to a host, never authenticates, and never changes router configuration. Even so, scanning equipment you do not own is your problem, not the tool's. ## Development @@ -162,7 +167,9 @@ Only scan networks you are responsible for. netdiff is deliberately read-only - pip install pytest && pytest -q ``` -The tests never touch the network. ARP parsing runs against captured `arp -an` and `ip neigh` output, UPnP parsing against captured router XML, and the one end-to-end test stands up a throwaway HTTP server on loopback. The database is a temp file. +The tests never touch the network. ARP parsing runs against captured `arp -an` and `ip neigh` output, UPnP parsing against captured router XML, mDNS parsing against hand-built packets, and the one end-to-end test stands up a throwaway HTTP server on loopback. The database is a temp file. + +`test_mdns.py` builds its packets with its own helpers rather than with the encoder in `mdns.py`, because a decoder tested only against its own encoder agrees with itself however wrong both are. Half of that file is malformed input - a name pointing at itself, a record claiming to be longer than the packet carrying it - because anything able to send a UDP datagram can send those. `test_diff.py` covers change detection. `test_audit.py` covers the rules, and roughly half of it asserts that something is *not* reported - an open port, an HTTP 200, a missing security header, a connection error. Those are the important half: the failure mode for a tool like this is not missing a finding, it is inventing one. diff --git a/netdiff/audit.py b/netdiff/audit.py index 1d6c10f..671fcff 100644 --- a/netdiff/audit.py +++ b/netdiff/audit.py @@ -283,7 +283,7 @@ def rule_mapping(mapping, devices): ) if mapping.internal_port in device.ports: - label = device.hostname or device.vendor or device.ip + label = device.hostname or device.vendor or device.services or device.ip return finding( "internet-exposed-service", device.ip, diff --git a/netdiff/cli.py b/netdiff/cli.py index f20e53e..1a8f7ca 100644 --- a/netdiff/cli.py +++ b/netdiff/cli.py @@ -11,7 +11,7 @@ import urllib.request from . import audit as audit_rules -from . import oui, store, upnp +from . import mdns, oui, store, upnp from .diff import diff, summarise from .scan import discover, grab_banner @@ -33,6 +33,18 @@ def send_webhook(url: str, payload: dict, timeout: float = 10) -> str: return "" +def device_label(hostname: str, vendor: str, services: str) -> str: + """What to call a device, best evidence first. + + A hostname is what a device was named; its mDNS announcement is what it says + it *is*. Both are worth printing, so the second one goes in parentheses + beside the first - unless it is the only thing we have, in which case it is + the label, and "unknown" is left for devices that told us nothing at all. + """ + main = hostname or vendor or services or "unknown" + return f"{main} ({services})" if services and services != main else main + + def cmd_scan(args) -> int: ports = () if args.no_ports else tuple(args.ports) devices = discover( @@ -40,6 +52,7 @@ def cmd_scan(args) -> int: ports=ports, lookup_vendor=oui.lookup, resolve_names=not args.no_resolve, + services={} if args.no_mdns else mdns.discover(), ) conn = store.connect(args.db) # Read the prior scan before inserting this one, or the "previous" scan @@ -72,7 +85,7 @@ def cmd_scan(args) -> int: else: print(f"scan {scan_id}: {len(devices)} device(s) on {args.subnet}") for device in devices: - label = device.hostname or device.vendor or "unknown" + label = device_label(device.hostname, device.vendor, device.services) open_ports = ( f" ports {','.join(str(p) for p in device.ports)}" if device.ports @@ -165,6 +178,7 @@ def placeholders(text): ports=tuple(args.ports), lookup_vendor=oui.lookup, resolve_names=not args.no_resolve, + services={} if args.no_mdns else mdns.discover(), ) banners = { (device.ip, port): grab_banner(device.ip, port) @@ -235,7 +249,7 @@ def cmd_inventory(args) -> int: return 0 print(f"{len(rows)} device(s) ever seen\n") for row in rows: - label = row["hostname"] or row["vendor"] or "unknown" + label = device_label(row["hostname"], row["vendor"], row["services"]) print(f"{row['ip']:<15} {row['mac']} {label}") print( f" first {row['first_seen']} last {row['last_seen']} seen {row['times_seen']}x" @@ -272,6 +286,9 @@ def build_parser() -> argparse.ArgumentParser: scan.add_argument("--ports", type=int, nargs="*", default=list(DEFAULT_PORTS)) scan.add_argument("--no-ports", action="store_true", help="skip the port scan") scan.add_argument("--no-resolve", action="store_true", help="skip reverse DNS") + scan.add_argument( + "--no-mdns", action="store_true", help="skip asking devices what they are" + ) scan.add_argument("--webhook", help="POST a JSON alert here when anything changed") scan.add_argument( "--fail-on-change", @@ -292,6 +309,9 @@ def build_parser() -> argparse.ArgumentParser: aud.add_argument("subnet", nargs="?", help="CIDR to audit, e.g. 192.168.1.0/24") aud.add_argument("--ports", type=int, nargs="*", default=list(DEFAULT_PORTS)) aud.add_argument("--no-resolve", action="store_true", help="skip reverse DNS") + aud.add_argument( + "--no-mdns", action="store_true", help="skip asking devices what they are" + ) aud.add_argument( "--no-upnp", action="store_true", help="skip the router port-forward check" ) diff --git a/netdiff/diff.py b/netdiff/diff.py index 8517d65..e0dc941 100644 --- a/netdiff/diff.py +++ b/netdiff/diff.py @@ -32,7 +32,15 @@ def severity(self) -> int: return SEVERITY.get(self.kind, 0) def __str__(self) -> str: - label = self.device.hostname or self.device.vendor or self.device.mac + # Display only. `services` is deliberately not compared anywhere in this + # module: one missed mDNS reply would otherwise report a change every + # other scan, and a change that is not real is worse than none. + label = ( + self.device.hostname + or self.device.vendor + or self.device.services + or self.device.mac + ) suffix = f" ({self.detail})" if self.detail else "" return f"[{self.kind}] {label} {self.device.ip} {self.device.mac}{suffix}" diff --git a/netdiff/mdns.py b/netdiff/mdns.py new file mode 100644 index 0000000..49574af --- /dev/null +++ b/netdiff/mdns.py @@ -0,0 +1,308 @@ +"""Ask the network what each device calls itself. + +A MAC vendor tells you who *made* a thing - "Espressif" covers a smart plug, a +doorbell and a hobby project equally. What you actually want to know is what the +thing *is*, and most consumer hardware will simply tell you: Chromecasts, +printers, Sonos, HomeKit gear and Apple devices all announce their services over +multicast DNS, unprompted, to anyone on the segment. + +That makes the identification evidence rather than inference. The alternative - +guessing device types from open port numbers - is what the tool this replaced +did, and it produced "Managed Web Server" for a printer. + +Read-only: this asks the standard DNS-SD question every phone on the network +asks continuously, and reads the answer. + +Joins the multicast group on 5353 where the OS allows it, sharing the port with +the resolver that already owns it (mDNSResponder on macOS, avahi on Linux) via +SO_REUSEPORT. Where that is refused it falls back to an ephemeral port and the +QU unicast-reply bit, which works but sees less: measured on a live network, the +fallback missed a device that answers only to multicast. +""" + +from __future__ import annotations + +import socket +import struct +import time + +MDNS_ADDRESS = ("224.0.0.251", 5353) + +# Records we can do something with. AAAA/NSEC are parsed past, not read. +TYPE_A = 1 +TYPE_PTR = 12 +TYPE_TXT = 16 +TYPE_SRV = 33 + +# The DNS-SD meta-query: "list every service type on this network". Responders +# answer it with PTRs naming their own types, which is how a device we have no +# entry for still shows up as something. +SERVICE_ENUM = "_services._dns-sd._udp.local" + +# What a service type means in words. Keyed by the type label as announced. +SERVICE_LABELS = { + "_googlecast": "Chromecast", + "_androidtvremote2": "Android TV", + "_amzn-wplay": "Amazon Fire TV", + "_roku-rcp": "Roku", + "_airplay": "AirPlay", + "_raop": "AirPlay speaker", + "_mediaremotetv": "Apple TV", + "_touch-able": "Apple TV", + "_companion-link": "Apple device", + "_sleep-proxy": "Apple device", + "_sonos": "Sonos", + "_spotify-connect": "Spotify Connect", + "_hap": "HomeKit accessory", + "_hue": "Philips Hue bridge", + "_esphomelib": "ESPHome device", + "_shelly": "Shelly device", + "_miio": "Xiaomi device", + "_ipp": "Printer", + "_ipps": "Printer", + "_printer": "Printer", + "_pdl-datastream": "Printer", + "_scanner": "Scanner", + "_uscan": "Scanner", + "_smb": "File sharing (SMB)", + "_afpovertcp": "Apple file sharing", + "_nfs": "File sharing (NFS)", + "_rfb": "Screen sharing (VNC)", + "_workstation": "Computer", + "_ssh": "SSH", + "_sftp-ssh": "SFTP", + "_http": "Web interface", + "_nvstream": "NVIDIA GameStream", + "_daap": "Music library", + "_homekit": "HomeKit", +} + +# Queried explicitly as well as via SERVICE_ENUM: some responders answer a +# direct question for their own type but ignore the meta-query. +COMMON_SERVICES = ( + "_googlecast._tcp.local", + "_airplay._tcp.local", + "_raop._tcp.local", + "_ipp._tcp.local", + "_printer._tcp.local", + "_hap._tcp.local", + "_spotify-connect._tcp.local", + "_sonos._tcp.local", + "_workstation._tcp.local", + "_device-info._tcp.local", + "_smb._tcp.local", + "_ssh._tcp.local", + "_http._tcp.local", + "_esphomelib._tcp.local", +) + + +def encode_name(name: str) -> bytes: + """Length-prefixed labels, null terminated.""" + out = b"" + for label in name.rstrip(".").split("."): + raw = label.encode("utf-8")[:63] + out += bytes([len(raw)]) + raw + return out + b"\x00" + + +def encode_query(names, unicast: bool = True) -> bytes: + """One DNS query carrying a question per name. + + QCLASS gets the top bit set to ask for a unicast reply, which is what lets + us listen on an ephemeral port instead of fighting the OS for 5353. + """ + qclass = 0x8001 if unicast else 0x0001 + header = struct.pack("!HHHHHH", 0, 0, len(names), 0, 0, 0) + body = b"".join( + encode_name(n) + struct.pack("!HH", TYPE_PTR, qclass) for n in names + ) + return header + body + + +def decode_name(data: bytes, offset: int): + """Read a possibly compressed name. Returns (name, offset after it). + + Compression pointers can point anywhere, including backwards into a loop, so + the jump budget is a hard stop rather than a guess. + """ + labels = [] + jumps = 0 + after = None + while True: + if offset >= len(data): + break + length = data[offset] + if length == 0: + offset += 1 + break + if length & 0xC0 == 0xC0: # pointer + if offset + 1 >= len(data): + break + jumps += 1 + if jumps > 32: # a pointer loop is malformed input, not a name + break + if after is None: + after = offset + 2 + offset = ((length & 0x3F) << 8) | data[offset + 1] + continue + offset += 1 + labels.append(data[offset : offset + length].decode("utf-8", "replace")) + offset += length + return ".".join(labels), (after if after is not None else offset) + + +def parse_records(data: bytes): + """Every resource record in a message, as (name, rtype, rdata) triples. + + Malformed or truncated input yields what was read so far - a device that + answers badly should cost us one device, not the scan. + """ + records = [] + try: + _, _, qd, an, ns, ar = struct.unpack("!HHHHHH", data[:12]) + except struct.error: + return records + offset = 12 + for _ in range(qd): + _, offset = decode_name(data, offset) + offset += 4 + for _ in range(an + ns + ar): + if offset >= len(data): + break + name, offset = decode_name(data, offset) + if offset + 10 > len(data): + break + rtype, _rclass, _ttl, rdlen = struct.unpack("!HHIH", data[offset : offset + 10]) + offset += 10 + if offset + rdlen > len(data): + # The record says it is longer than the packet holding it. Slicing + # would quietly hand back a short buffer, and a 4-byte address + # truncated to one byte still parses - as the address "192". + break + rdata = data[offset : offset + rdlen] + if rtype in (TYPE_PTR, TYPE_SRV): + # The target may be compressed, so it has to be read against the + # whole message rather than the rdata slice alone. + start = offset + 6 if rtype == TYPE_SRV else offset + rdata, _ = decode_name(data, start) + elif rtype == TYPE_TXT: + rdata = _decode_txt(rdata) + elif rtype == TYPE_A and rdlen == 4: + rdata = ".".join(str(b) for b in rdata) + records.append((name, rtype, rdata)) + offset += rdlen + return records + + +def _decode_txt(rdata: bytes) -> dict: + """TXT rdata is a run of length-prefixed key=value strings.""" + out = {} + i = 0 + while i < len(rdata): + length = rdata[i] + chunk = rdata[i + 1 : i + 1 + length].decode("utf-8", "replace") + key, _, value = chunk.partition("=") + if key: + out[key.lower()] = value + i += 1 + length + return out + + +def service_label(service: str) -> str: + """'_googlecast._tcp.local' -> 'Chromecast', or '' if we have no word.""" + head = service.lstrip(".").split(".")[0] + return SERVICE_LABELS.get(head, "") + + +def describe(records) -> str: + """Turn one device's records into a short human label. + + Prefers the model the device states outright, then the services it offers. + """ + model = "" + services = [] + for name, rtype, rdata in records: + if rtype == TYPE_TXT and isinstance(rdata, dict): + model = model or rdata.get("model") or rdata.get("md") or "" + if rtype == TYPE_PTR: + # The answer to the meta-query names a type; the answer to a type + # query names an instance of it. Both identify the device. + for candidate in (rdata, name): + word = service_label(candidate) + if word and word not in services: + services.append(word) + parts = [] + if model: + parts.append(model) + parts.extend(s for s in services if s != model) + return ", ".join(parts[:3]) + + +def discover(timeout: float = 2.5, sender=None) -> dict: + """Map IP -> short description for everything that answers DNS-SD. + + The datagram's source address is what attributes records to a device: mDNS + responders answer for themselves, so the sender is the subject. + """ + if sender is not None: + replies = sender(timeout) + else: + replies = _query(timeout) + by_ip: dict = {} + for ip, payload in replies: + by_ip.setdefault(ip, []).extend(parse_records(payload)) + return {ip: describe(recs) for ip, recs in by_ip.items() if describe(recs)} + + +def _open_socket(): + """Prefer joining the group on 5353; fall back to an ephemeral port. + + Measured on a live network: joining the group sees devices that the + ephemeral-port route misses, because a responder that ignores the QU bit + answers only to multicast. SO_REUSEPORT is what lets us sit on 5353 + alongside the OS resolver that already owns it. Where that is refused - some + Linux setups, restricted environments - the QU query still works and simply + finds less, which beats finding nothing. + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2) + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, "SO_REUSEPORT"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(("", MDNS_ADDRESS[1])) + sock.setsockopt( + socket.IPPROTO_IP, + socket.IP_ADD_MEMBERSHIP, + struct.pack("4sL", socket.inet_aton(MDNS_ADDRESS[0]), socket.INADDR_ANY), + ) + return sock, False + except OSError: + sock.close() + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM), True + + +def _query(timeout: float): + """Send the DNS-SD questions and collect raw (ip, payload) replies.""" + replies = [] + try: + sock, unicast = _open_socket() + with sock: + sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2) + sock.settimeout(timeout) + sock.sendto( + encode_query((SERVICE_ENUM, *COMMON_SERVICES), unicast=unicast), + MDNS_ADDRESS, + ) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + data, addr = sock.recvfrom(9000) + except socket.timeout: + break + replies.append((addr[0], data)) + except OSError: + # No multicast route is a network without mDNS, not an error. + return [] + return replies diff --git a/netdiff/scan.py b/netdiff/scan.py index 799a04a..b3d2b61 100644 --- a/netdiff/scan.py +++ b/netdiff/scan.py @@ -51,6 +51,7 @@ class Device: ip: str vendor: str = "" hostname: str = "" + services: str = "" ports: tuple[int, ...] = field(default=()) def key(self) -> str: @@ -169,8 +170,15 @@ def discover( lookup_vendor=None, settle: float = ARP_SETTLE_SECONDS, resolve_names: bool = True, + services=None, ) -> list[Device]: - """Scan `subnet` (CIDR) and return the devices found, sorted by IP.""" + """Scan `subnet` (CIDR) and return the devices found, sorted by IP. + + `services` is an optional IP -> description map from `mdns.discover()`, + collected by the caller. Passing it in rather than gathering it here keeps + this module to the one discovery technique it is about, and keeps the map a + plain dict that a test can hand over without a socket. + """ network = ipaddress.ip_network(subnet, strict=False) hosts = [str(h) for h in network.hosts()] nudge(hosts) @@ -189,6 +197,7 @@ def discover( ip=ip, vendor=lookup_vendor(mac) if lookup_vendor else "", hostname=resolve_hostname(ip) if resolve_names else "", + services=(services or {}).get(ip, ""), ports=scan_ports(ip, ports) if ports else (), ) ) diff --git a/netdiff/store.py b/netdiff/store.py index 4f1f665..149ce70 100644 --- a/netdiff/store.py +++ b/netdiff/store.py @@ -25,6 +25,7 @@ ip TEXT NOT NULL, vendor TEXT NOT NULL DEFAULT '', hostname TEXT NOT NULL DEFAULT '', + services TEXT NOT NULL DEFAULT '', ports TEXT NOT NULL DEFAULT '', PRIMARY KEY (scan_id, mac) ); @@ -51,9 +52,32 @@ def connect(path=DEFAULT_PATH) -> sqlite3.Connection: conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.executescript(SCHEMA) + _add_missing_columns(conn) return conn +# Columns added after the first release. `CREATE TABLE IF NOT EXISTS` does +# nothing to a table that already exists, and people have history going back +# months, so a new column has to be added explicitly or every read of it fails +# with "no such column" on exactly the databases worth keeping. +ADDED_COLUMNS = (("observations", "services", "TEXT NOT NULL DEFAULT ''"),) + + +def _add_missing_columns(conn: sqlite3.Connection) -> None: + """Bring an existing database up to the current schema. + + ponytail: a list of columns to add, not a numbered migration ladder. It is + idempotent and order-independent, which is all one added column needs. The + day a migration has to rename or backfill something, that is the day to + build the versioned thing - this will not stretch that far. + """ + with conn: + for table, column, decl in ADDED_COLUMNS: + present = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} + if column not in present: + conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {decl}") + + def _ports_to_text(ports) -> str: return ",".join(str(p) for p in ports) @@ -73,10 +97,19 @@ def record_scan(conn: sqlite3.Connection, subnet: str, devices) -> int: ) scan_id = cursor.lastrowid conn.executemany( - "INSERT INTO observations (scan_id, mac, ip, vendor, hostname, ports)" - " VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO observations" + " (scan_id, mac, ip, vendor, hostname, services, ports)" + " VALUES (?, ?, ?, ?, ?, ?, ?)", [ - (scan_id, d.mac, d.ip, d.vendor, d.hostname, _ports_to_text(d.ports)) + ( + scan_id, + d.mac, + d.ip, + d.vendor, + d.hostname, + d.services, + _ports_to_text(d.ports), + ) for d in devices ], ) @@ -85,7 +118,8 @@ def record_scan(conn: sqlite3.Connection, subnet: str, devices) -> int: def load_scan(conn: sqlite3.Connection, scan_id: int) -> list[Device]: rows = conn.execute( - "SELECT mac, ip, vendor, hostname, ports FROM observations WHERE scan_id = ?", + "SELECT mac, ip, vendor, hostname, services, ports FROM observations" + " WHERE scan_id = ?", (scan_id,), ).fetchall() return [ @@ -94,6 +128,7 @@ def load_scan(conn: sqlite3.Connection, scan_id: int) -> list[Device]: ip=r["ip"], vendor=r["vendor"], hostname=r["hostname"], + services=r["services"], ports=_ports_from_text(r["ports"]), ) for r in rows @@ -188,7 +223,7 @@ def inventory(conn: sqlite3.Connection) -> list[dict]: out = [] for row in rows: latest = conn.execute( - "SELECT ip, vendor, hostname, ports FROM observations o" + "SELECT ip, vendor, hostname, services, ports FROM observations o" " JOIN scans s ON s.id = o.scan_id WHERE o.mac = ?" " ORDER BY s.id DESC LIMIT 1", (row["mac"],), @@ -202,6 +237,7 @@ def inventory(conn: sqlite3.Connection) -> list[dict]: "ip": latest["ip"], "vendor": latest["vendor"], "hostname": latest["hostname"], + "services": latest["services"], "ports": _ports_from_text(latest["ports"]), } ) diff --git a/tests/test_mdns.py b/tests/test_mdns.py new file mode 100644 index 0000000..d7e3db4 --- /dev/null +++ b/tests/test_mdns.py @@ -0,0 +1,280 @@ +"""mDNS parsing against hand-built packets. No network is touched - the same +approach as the ARP fixtures in test_scan_and_store.py and the router XML in +test_upnp.py. + +The packets here are assembled by local helpers rather than by `mdns.encode_name`, +because a decoder tested only against its own encoder agrees with itself no matter +how wrong both are. `test_the_wire_format_is_what_we_think_it_is` pins the helpers +against a literal captured off the wire, so the rest of the file rests on bytes and +not on assumptions. + +Half of these are malformed inputs. A device that answers badly - or hostilely - +must cost us that one device, never the scan. +""" + +import struct + +from netdiff import mdns + +# --- packet builders --------------------------------------------------------- + + +def name(text): + """A DNS name: each label length-prefixed, terminated by a zero byte.""" + out = b"" + for label in text.rstrip(".").split("."): + out += bytes([len(label)]) + label.encode() + return out + b"\x00" + + +def record(owner, rtype, rdata, ttl=120): + """One resource record. `owner` may be raw bytes to place a pointer.""" + head = owner if isinstance(owner, bytes) else name(owner) + return head + struct.pack("!HHIH", rtype, 1, ttl, len(rdata)) + rdata + + +def message(*records, answers=None): + """A response packet. `answers` overrides the count, to lie about it.""" + count = len(records) if answers is None else answers + return struct.pack("!HHHHHH", 0, 0x8400, 0, count, 0, 0) + b"".join(records) + + +def txt(**pairs): + """TXT rdata: a run of length-prefixed `key=value` strings.""" + out = b"" + for key, value in pairs.items(): + chunk = f"{key}={value}".encode() + out += bytes([len(chunk)]) + chunk + return out + + +POINTER_TO_OFFSET_12 = b"\xc0\x0c" + + +def test_the_wire_format_is_what_we_think_it_is(): + """The literal is the first name in a real reply this Mac sent, captured off + the LAN. If the builders above are wrong, everything else here is theatre.""" + assert name("_services._dns-sd._udp.local") == ( + b"\x09_services\x07_dns-sd\x04_udp\x05local\x00" + ) + assert mdns.encode_name("_services._dns-sd._udp.local") == name( + "_services._dns-sd._udp.local" + ) + + +# --- names and compression --------------------------------------------------- + + +def test_a_compressed_name_is_followed_to_where_it_points(): + data = message(record("_airplay._tcp.local", mdns.TYPE_PTR, name("tv.local"))) + assert mdns.decode_name(data, 12)[0] == "_airplay._tcp.local" + + +def test_a_pointer_resumes_after_the_pointer_not_after_the_target(): + """The two-byte pointer is what the reader consumes; the target may sit + anywhere. Getting this wrong desynchronises every record that follows.""" + data = message(record("printer.local", mdns.TYPE_A, bytes([192, 168, 1, 5]))) + _, after = mdns.decode_name(data + POINTER_TO_OFFSET_12, len(data)) + assert after == len(data) + 2 + + +def test_a_pointer_loop_is_abandoned_rather_than_followed_forever(): + """A name that points at itself is malformed input, not a name. Unbounded, + this is a hang triggerable by anything that can send us a packet.""" + header = struct.pack("!HHHHHH", 0, 0x8400, 0, 1, 0, 0) + label, after = mdns.decode_name(header + POINTER_TO_OFFSET_12, 12) + assert label == "" + assert after == 14, "the reader still moves past the pointer" + + +def test_two_pointers_chasing_each_other_also_terminate(): + header = struct.pack("!HHHHHH", 0, 0x8400, 0, 1, 0, 0) + pair = b"\xc0\x0e" + b"\xc0\x0c" # offset 12 -> 14, offset 14 -> 12 + assert mdns.decode_name(header + pair, 12)[0] == "" + + +def test_a_label_running_off_the_end_yields_what_was_there(): + """A label claiming nine bytes with five behind it. Reading what arrived and + stopping beats raising: it costs one name, not the whole reply.""" + assert mdns.decode_name(b"\x09_serv", 0)[0] == "_serv" + + +# --- record parsing ---------------------------------------------------------- + + +def test_records_are_read_past_the_question_section(): + """Our own multicast query comes back to us, questions and all, because we + joined the group we sent to.""" + query = struct.pack("!HHHHHH", 0, 0, 1, 1, 0, 0) + query += name("_airplay._tcp.local") + struct.pack("!HH", mdns.TYPE_PTR, 1) + query += record("_airplay._tcp.local", mdns.TYPE_PTR, name("tv.local")) + + assert mdns.parse_records(query) == [ + ("_airplay._tcp.local", mdns.TYPE_PTR, "tv.local") + ] + + +def test_an_a_record_becomes_a_dotted_address(): + data = message(record("printer.local", mdns.TYPE_A, bytes([192, 168, 1, 5]))) + assert mdns.parse_records(data) == [("printer.local", mdns.TYPE_A, "192.168.1.5")] + + +def test_an_srv_target_is_read_past_its_priority_weight_and_port(): + data = message( + record( + "tv._airplay._tcp.local", + mdns.TYPE_SRV, + struct.pack("!HHH", 0, 0, 7000) + name("tv.local"), + ) + ) + assert mdns.parse_records(data)[0][2] == "tv.local" + + +def test_unreadable_record_types_are_stepped_over_not_choked_on(): + """AAAA and NSEC records arrive in every real reply and mean nothing to us, + but the records after them do.""" + data = message( + record("tv.local", 28, b"\xfe\x80" + b"\x00" * 14), # AAAA + record("tv.local", mdns.TYPE_A, bytes([192, 168, 1, 5])), + ) + assert mdns.parse_records(data)[-1][2] == "192.168.1.5" + + +def test_a_truncated_record_yields_what_was_read_before_it(): + whole = message( + record("printer.local", mdns.TYPE_A, bytes([192, 168, 1, 5])), + record("tv.local", mdns.TYPE_A, bytes([192, 168, 1, 9])), + ) + assert mdns.parse_records(whole[:-3]) == [ + ("printer.local", mdns.TYPE_A, "192.168.1.5") + ] + + +def test_a_header_claiming_more_records_than_it_carries_is_survivable(): + data = message( + record("printer.local", mdns.TYPE_A, bytes([192, 168, 1, 5])), answers=99 + ) + assert len(mdns.parse_records(data)) == 1 + + +def test_a_message_too_short_to_hold_a_header_is_not_an_error(): + assert mdns.parse_records(b"") == [] + assert mdns.parse_records(b"\x00\x01\x02") == [] + + +# --- TXT --------------------------------------------------------------------- + + +def test_txt_pairs_are_split_on_the_first_equals(): + data = message(record("tv.local", mdns.TYPE_TXT, txt(model="Mac15,7", fex="a=b=c"))) + assert mdns.parse_records(data)[0][2] == {"model": "Mac15,7", "fex": "a=b=c"} + + +def test_txt_keys_are_lowercased_because_devices_disagree_on_case(): + data = message(record("tv.local", mdns.TYPE_TXT, txt(MD="Chromecast"))) + assert mdns.parse_records(data)[0][2] == {"md": "Chromecast"} + + +def test_a_txt_string_with_no_value_is_kept_as_an_empty_one(): + """`\\x02id` is a flag, not a malformed pair - it says something by existing.""" + data = message(record("tv.local", mdns.TYPE_TXT, b"\x02id")) + assert mdns.parse_records(data)[0][2] == {"id": ""} + + +def test_an_empty_txt_string_is_dropped_rather_than_keyed_on_nothing(): + data = message(record("tv.local", mdns.TYPE_TXT, b"\x00" + b"\x03a=b")) + assert mdns.parse_records(data)[0][2] == {"a": "b"} + + +# --- describing a device ----------------------------------------------------- + + +def test_a_stated_model_beats_an_inferred_service_label(): + """`model=Mac15,7` is the device answering the question directly. A service + label is us translating what it offers into what it probably is.""" + records = [ + ("mac._device-info._tcp.local", mdns.TYPE_TXT, {"model": "Mac15,7"}), + ("_services._dns-sd._udp.local", mdns.TYPE_PTR, "_airplay._tcp.local"), + ] + assert mdns.describe(records).startswith("Mac15,7") + + +def test_a_service_type_is_named_whether_it_arrives_as_owner_or_target(): + """The meta-query answer names a type in the target; a direct query for that + type names an instance in the owner. Both identify the device.""" + as_target = [ + ("_services._dns-sd._udp.local", mdns.TYPE_PTR, "_googlecast._tcp.local") + ] + as_owner = [ + ("_googlecast._tcp.local", mdns.TYPE_PTR, "living-room._googlecast._tcp.local") + ] + assert mdns.describe(as_target) == "Chromecast" + assert mdns.describe(as_owner) == "Chromecast" + + +def test_a_service_announced_twice_is_only_said_once(): + records = [ + ("_services._dns-sd._udp.local", mdns.TYPE_PTR, "_airplay._tcp.local"), + ("_airplay._tcp.local", mdns.TYPE_PTR, "tv._airplay._tcp.local"), + ] + assert mdns.describe(records) == "AirPlay" + + +def test_a_device_announcing_everything_is_not_described_by_a_paragraph(): + records = [ + ("_services._dns-sd._udp.local", mdns.TYPE_PTR, f"{svc}._tcp.local") + for svc in ("_airplay", "_raop", "_smb", "_ssh", "_http") + ] + assert mdns.describe(records).count(",") == 2, "three parts, not five" + + +def test_a_device_offering_nothing_we_have_a_word_for_describes_as_nothing(): + """Better silent than confidently wrong. The tool this replaced called a + printer a "Managed Web Server" rather than admit it did not know.""" + records = [("_services._dns-sd._udp.local", mdns.TYPE_PTR, "_obscure._tcp.local")] + assert mdns.describe(records) == "" + + +# --- discover ---------------------------------------------------------------- + + +def test_replies_are_attributed_to_the_address_that_sent_them(): + tv = message( + record("_services._dns-sd._udp.local", mdns.TYPE_PTR, name("_googlecast._tcp")) + ) + printer = message( + record("_services._dns-sd._udp.local", mdns.TYPE_PTR, name("_ipp._tcp")) + ) + found = mdns.discover( + sender=lambda t: [("192.168.1.5", tv), ("192.168.1.9", printer)] + ) + assert found == {"192.168.1.5": "Chromecast", "192.168.1.9": "Printer"} + + +def test_several_packets_from_one_device_are_merged_into_one_description(): + """A device with a lot to say sends several datagrams, and the model can + arrive in a different one from the services.""" + info = message( + record("mac._device-info._tcp.local", mdns.TYPE_TXT, txt(model="Mac15,7")) + ) + services = message( + record("_services._dns-sd._udp.local", mdns.TYPE_PTR, name("_airplay._tcp")) + ) + found = mdns.discover( + sender=lambda t: [("192.168.1.5", info), ("192.168.1.5", services)] + ) + assert found == {"192.168.1.5": "Mac15,7, AirPlay"} + + +def test_a_device_we_cannot_describe_is_absent_rather_than_present_and_blank(): + """An empty string here would print as an empty pair of brackets and read as + a bug. Not knowing is the normal case, not a finding.""" + quiet = message(record("thing.local", mdns.TYPE_A, bytes([192, 168, 1, 5]))) + assert mdns.discover(sender=lambda t: [("192.168.1.5", quiet)]) == {} + + +def test_our_own_query_coming_back_to_us_names_no_device(): + """We joined the multicast group we send to, so we receive our own question. + It carries every service name we asked about and must describe nothing.""" + echo = mdns.encode_query((mdns.SERVICE_ENUM, *mdns.COMMON_SERVICES), unicast=False) + assert mdns.discover(sender=lambda t: [("192.168.1.190", echo)]) == {} diff --git a/tests/test_scan_and_store.py b/tests/test_scan_and_store.py index f98a6e1..d3ea530 100644 --- a/tests/test_scan_and_store.py +++ b/tests/test_scan_and_store.py @@ -1,6 +1,8 @@ """ARP parsing and persistence. No network is touched: parsing runs against captured command output, and the database is a temp file.""" +import sqlite3 + import pytest from netdiff import store @@ -92,14 +94,99 @@ def conn(tmp_path): def test_scan_roundtrips_through_the_database(conn): devices = [ Device( - mac="aa:bb:cc:00:00:01", ip="192.168.1.10", vendor="Acme", ports=(22, 80) + mac="aa:bb:cc:00:00:01", + ip="192.168.1.10", + vendor="Acme", + services="Chromecast", + ports=(22, 80), ), Device(mac="aa:bb:cc:00:00:02", ip="192.168.1.11", hostname="nas.local"), ] scan_id = store.record_scan(conn, "192.168.1.0/24", devices) assert store.load_scan(conn, scan_id) == devices, ( - "ports and blanks must survive the trip" + "ports, services and blanks must survive the trip" + ) + + +# The observations table as it shipped before `services` existed. Databases in +# this shape are on real machines with months of history in them. +SCHEMA_BEFORE_SERVICES = """ +CREATE TABLE scans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started TEXT NOT NULL, + subnet TEXT NOT NULL +); +CREATE TABLE observations ( + scan_id INTEGER NOT NULL REFERENCES scans(id) ON DELETE CASCADE, + mac TEXT NOT NULL, + ip TEXT NOT NULL, + vendor TEXT NOT NULL DEFAULT '', + hostname TEXT NOT NULL DEFAULT '', + ports TEXT NOT NULL DEFAULT '', + PRIMARY KEY (scan_id, mac) +); +""" + + +def old_database(path): + """A history file written by the previous release, with one scan in it.""" + legacy = sqlite3.connect(path) + legacy.executescript(SCHEMA_BEFORE_SERVICES) + legacy.execute("INSERT INTO scans (started, subnet) VALUES ('2026-01-01', 'x')") + legacy.execute( + "INSERT INTO observations (scan_id, mac, ip, vendor, hostname, ports)" + " VALUES (1, 'aa:bb:cc:00:00:01', '192.168.1.10', 'Acme', 'nas.local', '22,80')" ) + legacy.commit() + legacy.close() + return path + + +def columns_of(conn, table="observations"): + """Each column as name -> (type, notnull, default). + + Deliberately not a list: `ALTER TABLE` appends and `SCHEMA` inserts, so a + migrated database orders its columns differently from a fresh one. Nothing + in netdiff selects `*`, so that difference is invisible - and pinning the + order here would be asserting something the code does not rely on. + """ + rows = conn.execute(f"PRAGMA table_info({table})").fetchall() + return {r[1]: (r[2], r[3], r[4]) for r in rows} + + +def test_an_existing_database_gains_the_services_column(tmp_path): + """CREATE TABLE IF NOT EXISTS does nothing to a table that already exists, + so without the migration every read of `services` fails on exactly the + databases worth keeping - the ones with history in them.""" + path = old_database(tmp_path / "history.db") + devices = store.load_scan(store.connect(path), 1) + + assert devices == [ + Device( + mac="aa:bb:cc:00:00:01", + ip="192.168.1.10", + vendor="Acme", + hostname="nas.local", + ports=(22, 80), + ) + ], "old rows read back intact, with an empty services" + + +def test_migrating_twice_changes_nothing(tmp_path): + """Every `netdiff` command opens the database, so this runs constantly.""" + path = old_database(tmp_path / "history.db") + store.connect(path).close() + conn = store.connect(path) + + assert "services" in columns_of(conn) + assert store.load_scan(conn, 1)[0].hostname == "nas.local" + + +def test_a_migrated_database_matches_a_fresh_one(tmp_path, conn): + """SCHEMA and ADDED_COLUMNS are two descriptions of one shape, and nothing + stops them drifting apart except this.""" + migrated = store.connect(old_database(tmp_path / "old.db")) + assert columns_of(migrated) == columns_of(conn) def test_previous_devices_is_empty_for_the_very_first_scan(conn):