From f0a231e7c7a8c17a2dddd261fa0f4f306222b787 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 4 Sep 2026 19:57:24 -0400 Subject: [PATCH 1/2] Verify a chart file against the checksum its container holds Both ingest formats hold a checksum for a chart file, and both checksums were discarded, so no step between the bytes on a card and the tiles a mariner navigates by verified that a chart is the one the hydrographic office published. Every zip entry has a CRC32 of its uncompressed bytes, and zip.Iterator.Entry holds it. readAlloc hashes what it read and refuses a mismatch. Raw deflate has no checksum of its own, so a flipped bit inside a literal run inflates to bytes that parse: in an SG2D field that moves a depth-area boundary vertex by up to 25 metres at COMF 1e7, and the chart bakes and draws without an error. S-57 Part 3 3.4 lets the catalogue give a CRC per file in the CATD CRCS subfield. decodeCATD read parts 0 to 6 and left CRCS unread, and CatalogEntry had no field for it. It now holds the text, catalogCrc reads the hex, and a cell opened by catalogue name is checked against it. A producer omitting it is a case the spec allows, and those cells open as before. A test flips one byte in a stored entry and reads it back. --- src/chart.zig | 17 +++++++++++++---- src/s57/s57.zig | 23 ++++++++++++++++++++++- src/zipsrc.zig | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/chart.zig b/src/chart.zig index 0e09ab5..b5f4970 100644 --- a/src/chart.zig +++ b/src/chart.zig @@ -388,12 +388,21 @@ fn cdup(bytes: []const u8) ?[*]u8 { // Peek `relpath`'s bbox+scale; on success append an index-aligned meta + a gpa-owned // copy of the path. Cells that don't read / have no coverage bbox are skipped (both // lists), keeping meta[i] and paths[i] aligned with the streaming cell index. -fn addPathCell(io: std.Io, dir: std.Io.Dir, relpath: []const u8, metas: *std.ArrayList(ChartMeta), paths: *std.ArrayList([]u8)) !bool { +fn addPathCell(io: std.Io, dir: std.Io.Dir, relpath: []const u8, metas: *std.ArrayList(ChartMeta), paths: *std.ArrayList([]u8), want_crc: ?u32) !bool { const bytes = dir.readFileAlloc(io, relpath, gpa, .limited(MAX_CELL_BYTES)) catch { std.debug.print("CHART LOST {s}: cell did not read\n", .{relpath}); return false; }; defer gpa.free(bytes); + // S-57 Part 3 3.4: the catalogue may include a CRC per file. A cell whose + // bytes disagree with it is not the cell the producer published. + if (want_crc) |want| { + const got = std.hash.Crc32.hash(bytes); + if (got != want) { + std.debug.print("CHART LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ relpath, want, got }); + return false; + } + } const m = peekAnyMeta(bytes) orelse { std.debug.print("CHART LOST {s}: cell did not parse\n", .{relpath}); return false; @@ -2576,7 +2585,7 @@ pub const Chart = struct { var skipped: u32 = 0; if (single_file) { - if (!try addPathCell(io, dir, std.fs.path.basename(path), &metas, &paths)) skipped += 1; + if (!try addPathCell(io, dir, std.fs.path.basename(path), &metas, &paths, null)) skipped += 1; } else if (dir.readFileAlloc(io, "CATALOG.031", gpa, .limited(MAX_CELL_BYTES))) |cbytes| { defer gpa.free(cbytes); var carena = std.heap.ArenaAllocator.init(gpa); @@ -2584,7 +2593,7 @@ pub const Chart = struct { if (s57.parseCatalog(carena.allocator(), cbytes)) |entries| { for (entries) |e| { if (e.is_cell) { - if (!try addPathCell(io, dir, e.path, &metas, &paths)) skipped += 1; + if (!try addPathCell(io, dir, e.path, &metas, &paths, s57.catalogCrc(e.crcs))) skipped += 1; } } } @@ -2594,7 +2603,7 @@ pub const Chart = struct { while (try walker.next(io)) |entry| { if (entry.kind != .file) continue; if (!std.mem.endsWith(u8, entry.path, ".000")) continue; - if (!try addPathCell(io, dir, entry.path, &metas, &paths)) skipped += 1; + if (!try addPathCell(io, dir, entry.path, &metas, &paths, null)) skipped += 1; } } if (metas.items.len == 0) return error.OpenFailed; diff --git a/src/s57/s57.zig b/src/s57/s57.zig index 31a30fb..0c4f6ca 100644 --- a/src/s57/s57.zig +++ b/src/s57/s57.zig @@ -1625,8 +1625,21 @@ pub const CatalogEntry = struct { impl: []const u8, // "BIN" (a cell) / "ASC" / "TXT" ("" when absent); allocator-owned bbox: ?[4]f64, // [west, south, east, north]; null for non-cell / no coverage is_cell: bool, // BIN .000 base cell + /// The CRC the catalogue gives for the file (S-57 Part 3 3.4, the CATD + /// CRCS subfield), as the hex string the field holds. "" when the producer + /// gave none, which the spec permits. + crcs: []const u8, }; +/// The CRC32 in a CATD CRCS subfield, or null when it has none or the +/// text is not eight hex digits. S-57 Part 3 7.4.1 table 7.11 types CRCS as +/// `A( )` holding hex, and every producer seen writes it big-endian-first. +pub fn catalogCrc(crcs: []const u8) ?u32 { + const t = std.mem.trim(u8, crcs, " "); + if (t.len != 8) return null; + return std.fmt.parseInt(u32, t, 16) catch null; +} + /// ASCII whitespace set matching Go's strings.TrimSpace over byte data: space, /// tab, LF, VT, FF, CR. The oracle TrimSpace-es every attribute value before /// strconv.ParseFloat (pkg/s57 parseFloat + the portrayal/bake float parses), so @@ -1699,7 +1712,8 @@ fn decodeCATD(a: Allocator, raw_in: []const u8) ?CatalogEntry { }; } const long_name = a.dupe(u8, parts[1]) catch return null; - return .{ .stem = stem, .path = norm, .long_name = long_name, .impl = impl, .bbox = bbox, .is_cell = is_cell }; + const crcs = a.dupe(u8, parts[7]) catch return null; + return .{ .stem = stem, .path = norm, .long_name = long_name, .impl = impl, .bbox = bbox, .is_cell = is_cell, .crcs = crcs }; } /// Parse an S-57 exchange-set catalogue (CATALOG.031): one CATD record per file, @@ -2882,6 +2896,13 @@ test "a catalogue entry naming a file outside the exchange set is dropped" { try std.testing.expectEqualStrings("US5MD12M", e.stem); try std.testing.expect(e.is_cell); + // The CRC the catalogue gives for the file (S-57 Part 3 3.4). + try std.testing.expectEqual(@as(?u32, 0x1A2B3C4D), catalogCrc("1A2B3C4D")); + try std.testing.expectEqual(@as(?u32, 0x1A2B3C4D), catalogCrc(" 1a2b3c4d ")); + try std.testing.expectEqual(@as(?u32, null), catalogCrc("")); // producers may omit it + try std.testing.expectEqual(@as(?u32, null), catalogCrc("1A2B")); + try std.testing.expectEqual(@as(?u32, null), catalogCrc("ZZZZZZZZ")); + try std.testing.expect(safeCatalogPath("ENC_ROOT/A/B.000")); try std.testing.expect(safeCatalogPath("..a/B.000")); try std.testing.expect(!safeCatalogPath("../B.000")); diff --git a/src/zipsrc.zig b/src/zipsrc.zig index 39b3949..54358cd 100644 --- a/src/zipsrc.zig +++ b/src/zipsrc.zig @@ -34,6 +34,8 @@ pub const Error = error{ BadLocalHeader, UnsupportedCompressionMethod, EntryTooLarge, + /// The entry's bytes do not match the CRC32 the archive holds for it. + ChecksumMismatch, }; pub const Entry = struct { @@ -191,6 +193,12 @@ pub const Archive = struct { var w: std.Io.Writer = .fixed(buf); try self.streamEntry(io, i, &w); if (w.end != buf.len) return error.EndOfStream; + // Every zip entry has a CRC32 of its uncompressed bytes. Raw deflate + // has no checksum of its own, so a flipped bit inside a literal run + // inflates to bytes that parse. On a chart that moves a boundary vertex + // and draws a chart nobody published. + const want = e.raw.crc32; + if (want != 0 and std.hash.Crc32.hash(buf) != want) return Error.ChecksumMismatch; return buf; } @@ -652,3 +660,38 @@ test "a file that is not a zip is refused at open" { try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = "0001 this is an S-57 cell, not an archive" }); try testing.expectError(error.ZipNoEndRecord, Archive.open(gpa, io, zpath)); } + +test "a corrupted entry body is refused" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + // Stored, so a flipped byte in the file is a flipped byte in the entry. + // Deflate has no checksum of its own, so the entry CRC32 covers it. + const body = "US5MD12M cell bytes " ** 40; + try writeTestZip(gpa, io, zpath, &.{.{ .name = "ENC_ROOT/US5MD12M/US5MD12M.000", .data = body, .deflate = false }}); + + { + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + const got = try arc.readAlloc(gpa, io, 0, 1 << 20); + defer gpa.free(got); + try testing.expectEqualStrings(body, got); + } + + // Flip one byte inside the stored body and read again. + const raw = try std.Io.Dir.cwd().readFileAlloc(io, zpath, gpa, .unlimited); + defer gpa.free(raw); + const at = std.mem.indexOf(u8, raw, "cell bytes").? + 2; + raw[at] ^= 0x20; + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = raw }); + + var arc2 = try Archive.open(gpa, io, zpath); + defer arc2.deinit(); + try testing.expectError(Error.ChecksumMismatch, arc2.readAlloc(gpa, io, 0, 1 << 20)); +} From 5639dec9e7db0e16dcbacff5b268fc7f8cea7e93 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 4 Sep 2026 20:30:32 -0400 Subject: [PATCH 2/2] Check the catalogue CRC on the update files and on the bake The catalogue check covered the base cells of a directory open. Two gaps were left either side of it. The update files went unchecked on every path. A catalogue gives a CRC per file, updates included, but only .000 entries were compared, so a damaged .001 applied to a chart and the chart drew as though it were the one the hydrographic office published. The open now keeps every CRC the catalogue gives and compares each update as the chain reads it. A mismatch stops the chain and keeps the cell, the policy a corrupt update already follows, and names the file. The bake checked no file at all. It walks for .000 files and never opened the catalogue, so the two ingest paths disagreed about the same bytes: the inventory call refused a set the bake turned into archives without a word. The catalogue is now read once per exchange set and each cell and its updates are compared before the bake reads them. A file with no CRC in the catalogue passes, as before, and a set with no catalogue bakes unchanged. A damaged update fails the whole cell on the bake rather than truncating its chain. The bake writes one archive per cell, and an archive built to an earlier update than the set names cannot be told apart from a complete one afterwards. Checked against an exchange set built from a real cell, with one byte flipped: in the base, and in an update. Both paths now name the file and produce no archive, and the intact set is unaffected. --- src/chart.zig | 142 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 138 insertions(+), 4 deletions(-) diff --git a/src/chart.zig b/src/chart.zig index b5f4970..42ac17c 100644 --- a/src/chart.zig +++ b/src/chart.zig @@ -336,10 +336,21 @@ const PathCtx = struct { io: std.Io, dir: std.Io.Dir, paths: [][]u8, // base .000 path per cell, relative to `dir` + /// CRC per file from the exchange set's catalogue, keyed by the same + /// relative path the reads use. Empty when the set has no catalogue, or + /// when its producer left the CRCs out. Keys are owned. + /// + /// The base cells are verified once at open. The update files are read + /// later, on demand, so their CRCs travel here to be checked at that point + /// rather than costing a second pass over the whole set. + crcs: std.StringHashMapUnmanaged(u32) = .empty, fn deinit(self: *PathCtx) void { for (self.paths) |p| gpa.free(p); gpa.free(self.paths); + var it = self.crcs.keyIterator(); + while (it.next()) |k| gpa.free(k.*); + self.crcs.deinit(gpa); self.dir.close(self.io); self.threaded.deinit(); gpa.destroy(self.threaded); @@ -440,6 +451,16 @@ fn pathRead(user: ?*anyopaque, index: usize, out: *ChartBytes) callconv(.c) bool defer gpa.free(upn); const ub = ctx.dir.readFileAlloc(ctx.io, upn, gpa, .limited(MAX_CELL_BYTES)) catch break; defer gpa.free(ub); + // An update whose bytes disagree with the catalogue is damaged. Stop + // the chain here and keep what applied before it, the policy a corrupt + // update already follows. + if (ctx.crcs.get(upn)) |want| { + const got = std.hash.Crc32.hash(ub); + if (got != want) { + std.debug.print("UPDATE LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ upn, want, got }); + break; + } + } const cub = cdup(ub) orelse break; ulens.append(gpa, ub.len) catch { std.c.free(cub); @@ -1124,6 +1145,11 @@ const BakeFileCtx = struct { done: std.atomic.Value(u32), /// Set when a progress callback returned false; every worker drains out at its next cell. cancel: std.atomic.Value(bool), + /// CRC per file from the exchange set's catalogue, keyed as `in_paths` + /// names them. Empty when the set published none, or when reading from an + /// archive, where every entry is checked against its own CRC as it + /// inflates. Read-only once the workers start. + crcs: std.StringHashMapUnmanaged(u32) = .empty, /// Write the text and pictures each cell references beside its archive. aux: bool = true, }; @@ -1173,12 +1199,15 @@ fn bakeFileWorker(ctx: *BakeFileCtx) void { if (ctx.cancel.load(.monotonic)) return; // a peer's progress callback said stop const i = ctx.next.fetchAdd(1, .monotonic); if (i >= ctx.in_paths.len) return; - bakeOneToFile(ctx, i); + // A file that fails its catalogue CRC has already named itself, so it + // is not baked and does not draw the generic loss line below. + const verified = ctx.crcs.count() == 0 or catalogVerified(ctx.io, ctx.in_paths[i], &ctx.crcs); + if (verified) bakeOneToFile(ctx, i); // The label names a chart that was written. It fired for every cell // before, so a host printed a finished chart for one that failed. if (ctx.ok[i]) { if (ctx.label) |lb| lb(ctx.progress_ctx, @intCast(i)); - } else { + } else if (verified) { std.debug.print("CHART LOST {s}: bake produced no archive\n", .{ctx.in_paths[i]}); } const d = ctx.done.fetchAdd(1, .monotonic) + 1; // attempted count (smooth progress) @@ -1227,7 +1256,11 @@ fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []cons const cell_ms = gpa.alloc(i64, in_paths.len) catch return 0; defer gpa.free(cell_ms); @memset(cell_ms, 0); - var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux }; + // A zip entry is checked against its own CRC as it inflates, so the + // catalogue lookup is for the on-disk bake only. + var crcs = if (zip == null) catalogCrcsFor(io, in_paths) else std.StringHashMapUnmanaged(u32).empty; + defer freeCrcMap(&crcs); + var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux, .crcs = crcs }; var n = @min(@max(workers, 1), in_paths.len); if (n > MAX_BAKE_WORKERS) n = MAX_BAKE_WORKERS; // The comptime lhs prunes the spawn branch on a single-threaded build (wasm). @@ -1279,6 +1312,92 @@ fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []cons /// input (.000 + update chain) is skipped, so a re-run over an unchanged tree bakes nothing — and a /// run that resumes a cancelled one only bakes what the cancel left undone. Returns the count baked /// THIS run; errors if `in_dir` is unreadable. +/// The CRCs an exchange set's catalogue gives, keyed by path the way +/// `in_paths` names each cell, so a bake can check a file against the value +/// the set publishes for it. +/// +/// S-57 Part 3 puts CATALOG.031 at the exchange set's root, with the cells +/// either beside it or one directory below, so a cell's own directory and its +/// parent are the two places to look. Each catalogue found is parsed once, +/// however many cells it covers. An empty result means the set published no +/// catalogue, which S-57 allows, and the bake proceeds unchecked as before. +fn catalogCrcsFor(io: std.Io, in_paths: []const []const u8) std.StringHashMapUnmanaged(u32) { + var out: std.StringHashMapUnmanaged(u32) = .empty; + var seen: std.StringHashMapUnmanaged(void) = .empty; + defer seen.deinit(gpa); + + for (in_paths) |p| { + const d1 = std.fs.path.dirname(p) orelse continue; + const roots = [_][]const u8{ d1, std.fs.path.dirname(d1) orelse d1 }; + for (roots) |root| { + if (seen.contains(root)) continue; + seen.put(gpa, root, {}) catch continue; + const cat = std.fs.path.join(gpa, &.{ root, "CATALOG.031" }) catch continue; + defer gpa.free(cat); + const bytes = std.Io.Dir.cwd().readFileAlloc(io, cat, gpa, .limited(MAX_CELL_BYTES)) catch continue; + defer gpa.free(bytes); + var carena = std.heap.ArenaAllocator.init(gpa); + defer carena.deinit(); + const entries = s57.parseCatalog(carena.allocator(), bytes) orelse continue; + for (entries) |e| { + const want = s57.catalogCrc(e.crcs) orelse continue; + const key = std.fs.path.join(gpa, &.{ root, e.path }) catch continue; + out.put(gpa, key, want) catch gpa.free(key); + } + } + } + return out; +} + +fn freeCrcMap(m: *std.StringHashMapUnmanaged(u32)) void { + var it = m.keyIterator(); + while (it.next()) |k| gpa.free(k.*); + m.deinit(gpa); +} + +/// True when the cell at `path` and every update beside it match the CRC the +/// catalogue gives. A file the catalogue gives no CRC for passes, the case +/// S-57 lets a producer leave out. +/// +/// A damaged update fails the whole cell rather than truncating its chain: the +/// bake writes one archive per cell, and an archive built to an earlier update +/// than the set names cannot be told apart from a complete one afterwards. The +/// serve path keeps the cell and stops the chain instead, because a mariner +/// underway needs the chart already in hand. +fn catalogVerified(io: std.Io, path: []const u8, crcs: *const std.StringHashMapUnmanaged(u32)) bool { + const check = struct { + fn one(io_: std.Io, p: []const u8, m: *const std.StringHashMapUnmanaged(u32)) bool { + const want = m.get(p) orelse return true; + const bytes = std.Io.Dir.cwd().readFileAlloc(io_, p, gpa, .limited(MAX_CELL_BYTES)) catch { + std.debug.print("CHART LOST {s}: did not read\n", .{p}); + return false; + }; + defer gpa.free(bytes); + const got = std.hash.Crc32.hash(bytes); + if (got != want) { + std.debug.print("CHART LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ p, want, got }); + return false; + } + return true; + } + }.one; + + if (!check(io, path, crcs)) return false; + if (!std.mem.endsWith(u8, path, ".000")) return true; + const dir_path = std.fs.path.dirname(path) orelse "."; + var dir = std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }) catch return true; + defer dir.close(io); + var nums = updateNumbersFor(dir, io, std.fs.path.basename(path)) catch return true; + defer nums.deinit(gpa); + const stem = path[0 .. path.len - 4]; // strip ".000" + for (nums.items) |u| { + const upn = std.fmt.allocPrint(gpa, "{s}.{d:0>3}", .{ stem, u }) catch return true; + defer gpa.free(upn); + if (!check(io, upn, crcs)) return false; + } + return true; +} + pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel) !usize { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); @@ -1289,6 +1408,7 @@ pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir: var dir = try std.Io.Dir.cwd().openDir(io, in_dir, .{ .iterate = true }); defer dir.close(io); + var walker = try dir.walk(a); defer walker.deinit(); while (walker.next(io) catch null) |entry| { @@ -2583,6 +2703,12 @@ pub const Chart = struct { // Cells the walk found and could not use. The open succeeds on the // rest, so this is what tells a host the set has a gap. var skipped: u32 = 0; + var crcs: std.StringHashMapUnmanaged(u32) = .empty; + errdefer { + var cit = crcs.keyIterator(); + while (cit.next()) |k| gpa.free(k.*); + crcs.deinit(gpa); + } if (single_file) { if (!try addPathCell(io, dir, std.fs.path.basename(path), &metas, &paths, null)) skipped += 1; @@ -2591,6 +2717,14 @@ pub const Chart = struct { var carena = std.heap.ArenaAllocator.init(gpa); defer carena.deinit(); if (s57.parseCatalog(carena.allocator(), cbytes)) |entries| { + // Keep every CRC the catalogue gives, cells and updates alike. + // A cell is verified below, as its bytes are already read; an + // update is verified when the chain reaches it. + for (entries) |e| { + const want = s57.catalogCrc(e.crcs) orelse continue; + const key = gpa.dupe(u8, e.path) catch continue; + crcs.put(gpa, key, want) catch gpa.free(key); + } for (entries) |e| { if (e.is_cell) { if (!try addPathCell(io, dir, e.path, &metas, &paths, s57.catalogCrc(e.crcs))) skipped += 1; @@ -2615,7 +2749,7 @@ pub const Chart = struct { src.skipped_cells = skipped; const ctx = try gpa.create(PathCtx); errdefer gpa.destroy(ctx); - ctx.* = .{ .threaded = threaded, .io = io, .dir = dir, .paths = try paths.toOwnedSlice(gpa) }; + ctx.* = .{ .threaded = threaded, .io = io, .dir = dir, .paths = try paths.toOwnedSlice(gpa), .crcs = crcs }; src.backend.cells.reader_user = ctx; src.backend.cells.path_ctx = ctx; return src;