Verify a chart file against its container's checksum

src/chart.zig

@@ -336,10 +336,21 @@ const PathCtx = struct {
     io: std.Io,
     dir: std.Io.Dir,
     paths: [][]u8, // base .000 path per cell, relative to `dir`
+    /// CRC per file from the exchange set's catalogue, keyed by the same
+    /// relative path the reads use. Empty when the set has no catalogue, or
+    /// when its producer left the CRCs out. Keys are owned.
+    ///
+    /// The base cells are verified once at open. The update files are read
+    /// later, on demand, so their CRCs travel here to be checked at that point
+    /// rather than costing a second pass over the whole set.
+    crcs: std.StringHashMapUnmanaged(u32) = .empty,
 
     fn deinit(self: *PathCtx) void {
         for (self.paths) |p| gpa.free(p);
         gpa.free(self.paths);
+        var it = self.crcs.keyIterator();
+        while (it.next()) |k| gpa.free(k.*);
+        self.crcs.deinit(gpa);
         self.dir.close(self.io);
         self.threaded.deinit();
         gpa.destroy(self.threaded);
@@ -388,12 +399,21 @@ fn cdup(bytes: []const u8) ?[*]u8 {
 // Peek `relpath`'s bbox+scale; on success append an index-aligned meta + a gpa-owned
 // copy of the path. Cells that don't read / have no coverage bbox are skipped (both
 // lists), keeping meta[i] and paths[i] aligned with the streaming cell index.
-fn addPathCell(io: std.Io, dir: std.Io.Dir, relpath: []const u8, metas: *std.ArrayList(ChartMeta), paths: *std.ArrayList([]u8)) !bool {
+fn addPathCell(io: std.Io, dir: std.Io.Dir, relpath: []const u8, metas: *std.ArrayList(ChartMeta), paths: *std.ArrayList([]u8), want_crc: ?u32) !bool {
     const bytes = dir.readFileAlloc(io, relpath, gpa, .limited(MAX_CELL_BYTES)) catch {
         std.debug.print("CHART LOST {s}: cell did not read\n", .{relpath});
         return false;
     };
     defer gpa.free(bytes);
+    // S-57 Part 3 3.4: the catalogue may include a CRC per file. A cell whose
+    // bytes disagree with it is not the cell the producer published.
+    if (want_crc) |want| {
+        const got = std.hash.Crc32.hash(bytes);
+        if (got != want) {
+            std.debug.print("CHART LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ relpath, want, got });
+            return false;
+        }
+    }
     const m = peekAnyMeta(bytes) orelse {
         std.debug.print("CHART LOST {s}: cell did not parse\n", .{relpath});
         return false;
@@ -431,6 +451,16 @@ fn pathRead(user: ?*anyopaque, index: usize, out: *ChartBytes) callconv(.c) bool
         defer gpa.free(upn);
         const ub = ctx.dir.readFileAlloc(ctx.io, upn, gpa, .limited(MAX_CELL_BYTES)) catch break;
         defer gpa.free(ub);
+        // An update whose bytes disagree with the catalogue is damaged. Stop
+        // the chain here and keep what applied before it, the policy a corrupt
+        // update already follows.
+        if (ctx.crcs.get(upn)) |want| {
+            const got = std.hash.Crc32.hash(ub);
+            if (got != want) {
+                std.debug.print("UPDATE LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ upn, want, got });
+                break;
+            }
+        }
         const cub = cdup(ub) orelse break;
         ulens.append(gpa, ub.len) catch {
             std.c.free(cub);
@@ -1115,6 +1145,11 @@ const BakeFileCtx = struct {
     done: std.atomic.Value(u32),
     /// Set when a progress callback returned false; every worker drains out at its next cell.
     cancel: std.atomic.Value(bool),
+    /// CRC per file from the exchange set's catalogue, keyed as `in_paths`
+    /// names them. Empty when the set published none, or when reading from an
+    /// archive, where every entry is checked against its own CRC as it
+    /// inflates. Read-only once the workers start.
+    crcs: std.StringHashMapUnmanaged(u32) = .empty,
     /// Write the text and pictures each cell references beside its archive.
     aux: bool = true,
 };
@@ -1164,12 +1199,15 @@ fn bakeFileWorker(ctx: *BakeFileCtx) void {
         if (ctx.cancel.load(.monotonic)) return; // a peer's progress callback said stop
         const i = ctx.next.fetchAdd(1, .monotonic);
         if (i >= ctx.in_paths.len) return;
-        bakeOneToFile(ctx, i);
+        // A file that fails its catalogue CRC has already named itself, so it
+        // is not baked and does not draw the generic loss line below.
+        const verified = ctx.crcs.count() == 0 or catalogVerified(ctx.io, ctx.in_paths[i], &ctx.crcs);
+        if (verified) bakeOneToFile(ctx, i);
         // The label names a chart that was written. It fired for every cell
         // before, so a host printed a finished chart for one that failed.
         if (ctx.ok[i]) {
             if (ctx.label) |lb| lb(ctx.progress_ctx, @intCast(i));
-        } else {
+        } else if (verified) {
             std.debug.print("CHART LOST {s}: bake produced no archive\n", .{ctx.in_paths[i]});
         }
         const d = ctx.done.fetchAdd(1, .monotonic) + 1; // attempted count (smooth progress)
@@ -1218,7 +1256,11 @@ fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []cons
     const cell_ms = gpa.alloc(i64, in_paths.len) catch return 0;
     defer gpa.free(cell_ms);
     @memset(cell_ms, 0);
-    var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux };
+    // A zip entry is checked against its own CRC as it inflates, so the
+    // catalogue lookup is for the on-disk bake only.
+    var crcs = if (zip == null) catalogCrcsFor(io, in_paths) else std.StringHashMapUnmanaged(u32).empty;
+    defer freeCrcMap(&crcs);
+    var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux, .crcs = crcs };
     var n = @min(@max(workers, 1), in_paths.len);
     if (n > MAX_BAKE_WORKERS) n = MAX_BAKE_WORKERS;
     // The comptime lhs prunes the spawn branch on a single-threaded build (wasm).
@@ -1270,6 +1312,92 @@ fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []cons
 /// input (.000 + update chain) is skipped, so a re-run over an unchanged tree bakes nothing — and a
 /// run that resumes a cancelled one only bakes what the cancel left undone. Returns the count baked
 /// THIS run; errors if `in_dir` is unreadable.
+/// The CRCs an exchange set's catalogue gives, keyed by path the way
+/// `in_paths` names each cell, so a bake can check a file against the value
+/// the set publishes for it.
+///
+/// S-57 Part 3 puts CATALOG.031 at the exchange set's root, with the cells
+/// either beside it or one directory below, so a cell's own directory and its
+/// parent are the two places to look. Each catalogue found is parsed once,
+/// however many cells it covers. An empty result means the set published no
+/// catalogue, which S-57 allows, and the bake proceeds unchecked as before.
+fn catalogCrcsFor(io: std.Io, in_paths: []const []const u8) std.StringHashMapUnmanaged(u32) {
+    var out: std.StringHashMapUnmanaged(u32) = .empty;
+    var seen: std.StringHashMapUnmanaged(void) = .empty;
+    defer seen.deinit(gpa);
+
+    for (in_paths) |p| {
+        const d1 = std.fs.path.dirname(p) orelse continue;
+        const roots = [_][]const u8{ d1, std.fs.path.dirname(d1) orelse d1 };
+        for (roots) |root| {
+            if (seen.contains(root)) continue;
+            seen.put(gpa, root, {}) catch continue;
+            const cat = std.fs.path.join(gpa, &.{ root, "CATALOG.031" }) catch continue;
+            defer gpa.free(cat);
+            const bytes = std.Io.Dir.cwd().readFileAlloc(io, cat, gpa, .limited(MAX_CELL_BYTES)) catch continue;
+            defer gpa.free(bytes);
+            var carena = std.heap.ArenaAllocator.init(gpa);
+            defer carena.deinit();
+            const entries = s57.parseCatalog(carena.allocator(), bytes) orelse continue;
+            for (entries) |e| {
+                const want = s57.catalogCrc(e.crcs) orelse continue;
+                const key = std.fs.path.join(gpa, &.{ root, e.path }) catch continue;
+                out.put(gpa, key, want) catch gpa.free(key);
+            }
+        }
+    }
+    return out;
+}
+
+fn freeCrcMap(m: *std.StringHashMapUnmanaged(u32)) void {
+    var it = m.keyIterator();
+    while (it.next()) |k| gpa.free(k.*);
+    m.deinit(gpa);
+}
+
+/// True when the cell at `path` and every update beside it match the CRC the
+/// catalogue gives. A file the catalogue gives no CRC for passes, the case
+/// S-57 lets a producer leave out.
+///
+/// A damaged update fails the whole cell rather than truncating its chain: the
+/// bake writes one archive per cell, and an archive built to an earlier update
+/// than the set names cannot be told apart from a complete one afterwards. The
+/// serve path keeps the cell and stops the chain instead, because a mariner
+/// underway needs the chart already in hand.
+fn catalogVerified(io: std.Io, path: []const u8, crcs: *const std.StringHashMapUnmanaged(u32)) bool {
+    const check = struct {
+        fn one(io_: std.Io, p: []const u8, m: *const std.StringHashMapUnmanaged(u32)) bool {
+            const want = m.get(p) orelse return true;
+            const bytes = std.Io.Dir.cwd().readFileAlloc(io_, p, gpa, .limited(MAX_CELL_BYTES)) catch {
+                std.debug.print("CHART LOST {s}: did not read\n", .{p});
+                return false;
+            };
+            defer gpa.free(bytes);
+            const got = std.hash.Crc32.hash(bytes);
+            if (got != want) {
+                std.debug.print("CHART LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ p, want, got });
+                return false;
+            }
+            return true;
+        }
+    }.one;
+
+    if (!check(io, path, crcs)) return false;
+    if (!std.mem.endsWith(u8, path, ".000")) return true;
+    const dir_path = std.fs.path.dirname(path) orelse ".";
+    var dir = std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }) catch return true;
+    defer dir.close(io);
+    var nums = updateNumbersFor(dir, io, std.fs.path.basename(path)) catch return true;
+    defer nums.deinit(gpa);
+    const stem = path[0 .. path.len - 4]; // strip ".000"
+    for (nums.items) |u| {
+        const upn = std.fmt.allocPrint(gpa, "{s}.{d:0>3}", .{ stem, u }) catch return true;
+        defer gpa.free(upn);
+        if (!check(io, upn, crcs)) return false;
+    }
+    return true;
+}
+
 pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel) !usize {
     var arena = std.heap.ArenaAllocator.init(gpa);
     defer arena.deinit();
@@ -1280,6 +1408,7 @@ pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir:
 
     var dir = try std.Io.Dir.cwd().openDir(io, in_dir, .{ .iterate = true });
     defer dir.close(io);
+
     var walker = try dir.walk(a);
     defer walker.deinit();
     while (walker.next(io) catch null) |entry| {
@@ -2574,17 +2703,31 @@ pub const Chart = struct {
         // Cells the walk found and could not use. The open succeeds on the
         // rest, so this is what tells a host the set has a gap.
         var skipped: u32 = 0;
+        var crcs: std.StringHashMapUnmanaged(u32) = .empty;
+        errdefer {
+            var cit = crcs.keyIterator();
+            while (cit.next()) |k| gpa.free(k.*);
+            crcs.deinit(gpa);
+        }
 
         if (single_file) {
-            if (!try addPathCell(io, dir, std.fs.path.basename(path), &metas, &paths)) skipped += 1;
+            if (!try addPathCell(io, dir, std.fs.path.basename(path), &metas, &paths, null)) skipped += 1;
         } else if (dir.readFileAlloc(io, "CATALOG.031", gpa, .limited(MAX_CELL_BYTES))) |cbytes| {
             defer gpa.free(cbytes);
             var carena = std.heap.ArenaAllocator.init(gpa);
             defer carena.deinit();
             if (s57.parseCatalog(carena.allocator(), cbytes)) |entries| {
+                // Keep every CRC the catalogue gives, cells and updates alike.
+                // A cell is verified below, as its bytes are already read; an
+                // update is verified when the chain reaches it.
+                for (entries) |e| {
+                    const want = s57.catalogCrc(e.crcs) orelse continue;
+                    const key = gpa.dupe(u8, e.path) catch continue;
+                    crcs.put(gpa, key, want) catch gpa.free(key);
+                }
                 for (entries) |e| {
                     if (e.is_cell) {
-                        if (!try addPathCell(io, dir, e.path, &metas, &paths)) skipped += 1;
+                        if (!try addPathCell(io, dir, e.path, &metas, &paths, s57.catalogCrc(e.crcs))) skipped += 1;
                     }
                 }
             }
@@ -2594,7 +2737,7 @@ pub const Chart = struct {
             while (try walker.next(io)) |entry| {
                 if (entry.kind != .file) continue;
                 if (!std.mem.endsWith(u8, entry.path, ".000")) continue;
-                if (!try addPathCell(io, dir, entry.path, &metas, &paths)) skipped += 1;
+                if (!try addPathCell(io, dir, entry.path, &metas, &paths, null)) skipped += 1;
             }
         }
         if (metas.items.len == 0) return error.OpenFailed;
@@ -2606,7 +2749,7 @@ pub const Chart = struct {
         src.skipped_cells = skipped;
         const ctx = try gpa.create(PathCtx);
         errdefer gpa.destroy(ctx);
-        ctx.* = .{ .threaded = threaded, .io = io, .dir = dir, .paths = try paths.toOwnedSlice(gpa) };
+        ctx.* = .{ .threaded = threaded, .io = io, .dir = dir, .paths = try paths.toOwnedSlice(gpa), .crcs = crcs };
         src.backend.cells.reader_user = ctx;
         src.backend.cells.path_ctx = ctx;
         return src;

src/s57/s57.zig

@@ -1625,8 +1625,21 @@ pub const CatalogEntry = struct {
     impl: []const u8, // "BIN" (a cell) / "ASC" / "TXT" ("" when absent); allocator-owned
     bbox: ?[4]f64, // [west, south, east, north]; null for non-cell / no coverage
     is_cell: bool, // BIN .000 base cell
+    /// The CRC the catalogue gives for the file (S-57 Part 3 3.4, the CATD
+    /// CRCS subfield), as the hex string the field holds. "" when the producer
+    /// gave none, which the spec permits.
+    crcs: []const u8,
 };
 
+/// The CRC32 in a CATD CRCS subfield, or null when it has none or the
+/// text is not eight hex digits. S-57 Part 3 7.4.1 table 7.11 types CRCS as
+/// `A( )` holding hex, and every producer seen writes it big-endian-first.
+pub fn catalogCrc(crcs: []const u8) ?u32 {
+    const t = std.mem.trim(u8, crcs, " ");
+    if (t.len != 8) return null;
+    return std.fmt.parseInt(u32, t, 16) catch null;
+}
+
 /// ASCII whitespace set matching Go's strings.TrimSpace over byte data: space,
 /// tab, LF, VT, FF, CR. The oracle TrimSpace-es every attribute value before
 /// strconv.ParseFloat (pkg/s57 parseFloat + the portrayal/bake float parses), so
@@ -1699,7 +1712,8 @@ fn decodeCATD(a: Allocator, raw_in: []const u8) ?CatalogEntry {
                     };
     }
     const long_name = a.dupe(u8, parts[1]) catch return null;
-    return .{ .stem = stem, .path = norm, .long_name = long_name, .impl = impl, .bbox = bbox, .is_cell = is_cell };
+    const crcs = a.dupe(u8, parts[7]) catch return null;
+    return .{ .stem = stem, .path = norm, .long_name = long_name, .impl = impl, .bbox = bbox, .is_cell = is_cell, .crcs = crcs };
 }
 
 /// Parse an S-57 exchange-set catalogue (CATALOG.031): one CATD record per file,
@@ -2882,6 +2896,13 @@ test "a catalogue entry naming a file outside the exchange set is dropped" {
     try std.testing.expectEqualStrings("US5MD12M", e.stem);
     try std.testing.expect(e.is_cell);
 
+    // The CRC the catalogue gives for the file (S-57 Part 3 3.4).
+    try std.testing.expectEqual(@as(?u32, 0x1A2B3C4D), catalogCrc("1A2B3C4D"));
+    try std.testing.expectEqual(@as(?u32, 0x1A2B3C4D), catalogCrc(" 1a2b3c4d "));
+    try std.testing.expectEqual(@as(?u32, null), catalogCrc("")); // producers may omit it
+    try std.testing.expectEqual(@as(?u32, null), catalogCrc("1A2B"));
+    try std.testing.expectEqual(@as(?u32, null), catalogCrc("ZZZZZZZZ"));
+
     try std.testing.expect(safeCatalogPath("ENC_ROOT/A/B.000"));
     try std.testing.expect(safeCatalogPath("..a/B.000"));
     try std.testing.expect(!safeCatalogPath("../B.000"));

src/zipsrc.zig

@@ -34,6 +34,8 @@ pub const Error = error{
     BadLocalHeader,
     UnsupportedCompressionMethod,
     EntryTooLarge,
+    /// The entry's bytes do not match the CRC32 the archive holds for it.
+    ChecksumMismatch,
 };
 
 pub const Entry = struct {
@@ -191,6 +193,12 @@ pub const Archive = struct {
         var w: std.Io.Writer = .fixed(buf);
         try self.streamEntry(io, i, &w);
         if (w.end != buf.len) return error.EndOfStream;
+        // Every zip entry has a CRC32 of its uncompressed bytes. Raw deflate
+        // has no checksum of its own, so a flipped bit inside a literal run
+        // inflates to bytes that parse. On a chart that moves a boundary vertex
+        // and draws a chart nobody published.
+        const want = e.raw.crc32;
+        if (want != 0 and std.hash.Crc32.hash(buf) != want) return Error.ChecksumMismatch;
         return buf;
     }
 
@@ -652,3 +660,38 @@ test "a file that is not a zip is refused at open" {
     try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = "0001 this is an S-57 cell, not an archive" });
     try testing.expectError(error.ZipNoEndRecord, Archive.open(gpa, io, zpath));
 }
+
+test "a corrupted entry body is refused" {
+    const gpa = testing.allocator;
+    const io = testIo();
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    const dir = try tmpPath(gpa, &tmp);
+    defer gpa.free(dir);
+    const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" });
+    defer gpa.free(zpath);
+
+    // Stored, so a flipped byte in the file is a flipped byte in the entry.
+    // Deflate has no checksum of its own, so the entry CRC32 covers it.
+    const body = "US5MD12M cell bytes " ** 40;
+    try writeTestZip(gpa, io, zpath, &.{.{ .name = "ENC_ROOT/US5MD12M/US5MD12M.000", .data = body, .deflate = false }});
+
+    {
+        var arc = try Archive.open(gpa, io, zpath);
+        defer arc.deinit();
+        const got = try arc.readAlloc(gpa, io, 0, 1 << 20);
+        defer gpa.free(got);
+        try testing.expectEqualStrings(body, got);
+    }
+
+    // Flip one byte inside the stored body and read again.
+    const raw = try std.Io.Dir.cwd().readFileAlloc(io, zpath, gpa, .unlimited);
+    defer gpa.free(raw);
+    const at = std.mem.indexOf(u8, raw, "cell bytes").? + 2;
+    raw[at] ^= 0x20;
+    try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = raw });
+
+    var arc2 = try Archive.open(gpa, io, zpath);
+    defer arc2.deinit();
+    try testing.expectError(Error.ChecksumMismatch, arc2.readAlloc(gpa, io, 0, 1 << 20));
+}