diff --git a/src/chart.zig b/src/chart.zig index 0e09ab5..42ac17c 100644 --- a/src/chart.zig +++ b/src/chart.zig @@ -336,10 +336,21 @@ const PathCtx = struct { io: std.Io, dir: std.Io.Dir, paths: [][]u8, // base .000 path per cell, relative to `dir` + /// CRC per file from the exchange set's catalogue, keyed by the same + /// relative path the reads use. Empty when the set has no catalogue, or + /// when its producer left the CRCs out. Keys are owned. + /// + /// The base cells are verified once at open. The update files are read + /// later, on demand, so their CRCs travel here to be checked at that point + /// rather than costing a second pass over the whole set. + crcs: std.StringHashMapUnmanaged(u32) = .empty, fn deinit(self: *PathCtx) void { for (self.paths) |p| gpa.free(p); gpa.free(self.paths); + var it = self.crcs.keyIterator(); + while (it.next()) |k| gpa.free(k.*); + self.crcs.deinit(gpa); self.dir.close(self.io); self.threaded.deinit(); gpa.destroy(self.threaded); @@ -388,12 +399,21 @@ fn cdup(bytes: []const u8) ?[*]u8 { // Peek `relpath`'s bbox+scale; on success append an index-aligned meta + a gpa-owned // copy of the path. Cells that don't read / have no coverage bbox are skipped (both // lists), keeping meta[i] and paths[i] aligned with the streaming cell index. -fn addPathCell(io: std.Io, dir: std.Io.Dir, relpath: []const u8, metas: *std.ArrayList(ChartMeta), paths: *std.ArrayList([]u8)) !bool { +fn addPathCell(io: std.Io, dir: std.Io.Dir, relpath: []const u8, metas: *std.ArrayList(ChartMeta), paths: *std.ArrayList([]u8), want_crc: ?u32) !bool { const bytes = dir.readFileAlloc(io, relpath, gpa, .limited(MAX_CELL_BYTES)) catch { std.debug.print("CHART LOST {s}: cell did not read\n", .{relpath}); return false; }; defer gpa.free(bytes); + // S-57 Part 3 3.4: the catalogue may include a CRC per file. A cell whose + // bytes disagree with it is not the cell the producer published. + if (want_crc) |want| { + const got = std.hash.Crc32.hash(bytes); + if (got != want) { + std.debug.print("CHART LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ relpath, want, got }); + return false; + } + } const m = peekAnyMeta(bytes) orelse { std.debug.print("CHART LOST {s}: cell did not parse\n", .{relpath}); return false; @@ -431,6 +451,16 @@ fn pathRead(user: ?*anyopaque, index: usize, out: *ChartBytes) callconv(.c) bool defer gpa.free(upn); const ub = ctx.dir.readFileAlloc(ctx.io, upn, gpa, .limited(MAX_CELL_BYTES)) catch break; defer gpa.free(ub); + // An update whose bytes disagree with the catalogue is damaged. Stop + // the chain here and keep what applied before it, the policy a corrupt + // update already follows. + if (ctx.crcs.get(upn)) |want| { + const got = std.hash.Crc32.hash(ub); + if (got != want) { + std.debug.print("UPDATE LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ upn, want, got }); + break; + } + } const cub = cdup(ub) orelse break; ulens.append(gpa, ub.len) catch { std.c.free(cub); @@ -1115,6 +1145,11 @@ const BakeFileCtx = struct { done: std.atomic.Value(u32), /// Set when a progress callback returned false; every worker drains out at its next cell. cancel: std.atomic.Value(bool), + /// CRC per file from the exchange set's catalogue, keyed as `in_paths` + /// names them. Empty when the set published none, or when reading from an + /// archive, where every entry is checked against its own CRC as it + /// inflates. Read-only once the workers start. + crcs: std.StringHashMapUnmanaged(u32) = .empty, /// Write the text and pictures each cell references beside its archive. aux: bool = true, }; @@ -1164,12 +1199,15 @@ fn bakeFileWorker(ctx: *BakeFileCtx) void { if (ctx.cancel.load(.monotonic)) return; // a peer's progress callback said stop const i = ctx.next.fetchAdd(1, .monotonic); if (i >= ctx.in_paths.len) return; - bakeOneToFile(ctx, i); + // A file that fails its catalogue CRC has already named itself, so it + // is not baked and does not draw the generic loss line below. + const verified = ctx.crcs.count() == 0 or catalogVerified(ctx.io, ctx.in_paths[i], &ctx.crcs); + if (verified) bakeOneToFile(ctx, i); // The label names a chart that was written. It fired for every cell // before, so a host printed a finished chart for one that failed. if (ctx.ok[i]) { if (ctx.label) |lb| lb(ctx.progress_ctx, @intCast(i)); - } else { + } else if (verified) { std.debug.print("CHART LOST {s}: bake produced no archive\n", .{ctx.in_paths[i]}); } const d = ctx.done.fetchAdd(1, .monotonic) + 1; // attempted count (smooth progress) @@ -1218,7 +1256,11 @@ fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []cons const cell_ms = gpa.alloc(i64, in_paths.len) catch return 0; defer gpa.free(cell_ms); @memset(cell_ms, 0); - var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux }; + // A zip entry is checked against its own CRC as it inflates, so the + // catalogue lookup is for the on-disk bake only. + var crcs = if (zip == null) catalogCrcsFor(io, in_paths) else std.StringHashMapUnmanaged(u32).empty; + defer freeCrcMap(&crcs); + var ctx = BakeFileCtx{ .next = std.atomic.Value(usize).init(0), .in_paths = in_paths, .out_paths = out_paths, .rules_dir = rules_dir, .zip = zip, .io = io, .ok = ok, .ms = cell_ms, .progress = progress, .progress_ctx = progress_ctx, .label = label, .done = std.atomic.Value(u32).init(0), .cancel = std.atomic.Value(bool).init(false), .aux = aux, .crcs = crcs }; var n = @min(@max(workers, 1), in_paths.len); if (n > MAX_BAKE_WORKERS) n = MAX_BAKE_WORKERS; // The comptime lhs prunes the spawn branch on a single-threaded build (wasm). @@ -1270,6 +1312,92 @@ fn bakeToFiles(io: std.Io, zip: ?*const zipsrc.Archive, in_paths: []const []cons /// input (.000 + update chain) is skipped, so a re-run over an unchanged tree bakes nothing — and a /// run that resumes a cancelled one only bakes what the cancel left undone. Returns the count baked /// THIS run; errors if `in_dir` is unreadable. +/// The CRCs an exchange set's catalogue gives, keyed by path the way +/// `in_paths` names each cell, so a bake can check a file against the value +/// the set publishes for it. +/// +/// S-57 Part 3 puts CATALOG.031 at the exchange set's root, with the cells +/// either beside it or one directory below, so a cell's own directory and its +/// parent are the two places to look. Each catalogue found is parsed once, +/// however many cells it covers. An empty result means the set published no +/// catalogue, which S-57 allows, and the bake proceeds unchecked as before. +fn catalogCrcsFor(io: std.Io, in_paths: []const []const u8) std.StringHashMapUnmanaged(u32) { + var out: std.StringHashMapUnmanaged(u32) = .empty; + var seen: std.StringHashMapUnmanaged(void) = .empty; + defer seen.deinit(gpa); + + for (in_paths) |p| { + const d1 = std.fs.path.dirname(p) orelse continue; + const roots = [_][]const u8{ d1, std.fs.path.dirname(d1) orelse d1 }; + for (roots) |root| { + if (seen.contains(root)) continue; + seen.put(gpa, root, {}) catch continue; + const cat = std.fs.path.join(gpa, &.{ root, "CATALOG.031" }) catch continue; + defer gpa.free(cat); + const bytes = std.Io.Dir.cwd().readFileAlloc(io, cat, gpa, .limited(MAX_CELL_BYTES)) catch continue; + defer gpa.free(bytes); + var carena = std.heap.ArenaAllocator.init(gpa); + defer carena.deinit(); + const entries = s57.parseCatalog(carena.allocator(), bytes) orelse continue; + for (entries) |e| { + const want = s57.catalogCrc(e.crcs) orelse continue; + const key = std.fs.path.join(gpa, &.{ root, e.path }) catch continue; + out.put(gpa, key, want) catch gpa.free(key); + } + } + } + return out; +} + +fn freeCrcMap(m: *std.StringHashMapUnmanaged(u32)) void { + var it = m.keyIterator(); + while (it.next()) |k| gpa.free(k.*); + m.deinit(gpa); +} + +/// True when the cell at `path` and every update beside it match the CRC the +/// catalogue gives. A file the catalogue gives no CRC for passes, the case +/// S-57 lets a producer leave out. +/// +/// A damaged update fails the whole cell rather than truncating its chain: the +/// bake writes one archive per cell, and an archive built to an earlier update +/// than the set names cannot be told apart from a complete one afterwards. The +/// serve path keeps the cell and stops the chain instead, because a mariner +/// underway needs the chart already in hand. +fn catalogVerified(io: std.Io, path: []const u8, crcs: *const std.StringHashMapUnmanaged(u32)) bool { + const check = struct { + fn one(io_: std.Io, p: []const u8, m: *const std.StringHashMapUnmanaged(u32)) bool { + const want = m.get(p) orelse return true; + const bytes = std.Io.Dir.cwd().readFileAlloc(io_, p, gpa, .limited(MAX_CELL_BYTES)) catch { + std.debug.print("CHART LOST {s}: did not read\n", .{p}); + return false; + }; + defer gpa.free(bytes); + const got = std.hash.Crc32.hash(bytes); + if (got != want) { + std.debug.print("CHART LOST {s}: catalogue CRC is {x:0>8}, file is {x:0>8}\n", .{ p, want, got }); + return false; + } + return true; + } + }.one; + + if (!check(io, path, crcs)) return false; + if (!std.mem.endsWith(u8, path, ".000")) return true; + const dir_path = std.fs.path.dirname(path) orelse "."; + var dir = std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }) catch return true; + defer dir.close(io); + var nums = updateNumbersFor(dir, io, std.fs.path.basename(path)) catch return true; + defer nums.deinit(gpa); + const stem = path[0 .. path.len - 4]; // strip ".000" + for (nums.items) |u| { + const upn = std.fmt.allocPrint(gpa, "{s}.{d:0>3}", .{ stem, u }) catch return true; + defer gpa.free(upn); + if (!check(io, upn, crcs)) return false; + } + return true; +} + pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir: ?[]const u8, workers: usize, progress: BakeProgress, progress_ctx: ?*anyopaque, label: BakeLabel) !usize { var arena = std.heap.ArenaAllocator.init(gpa); defer arena.deinit(); @@ -1280,6 +1408,7 @@ pub fn bakeTree(io: std.Io, in_dir: []const u8, out_dir: []const u8, rules_dir: var dir = try std.Io.Dir.cwd().openDir(io, in_dir, .{ .iterate = true }); defer dir.close(io); + var walker = try dir.walk(a); defer walker.deinit(); while (walker.next(io) catch null) |entry| { @@ -2574,17 +2703,31 @@ pub const Chart = struct { // Cells the walk found and could not use. The open succeeds on the // rest, so this is what tells a host the set has a gap. var skipped: u32 = 0; + var crcs: std.StringHashMapUnmanaged(u32) = .empty; + errdefer { + var cit = crcs.keyIterator(); + while (cit.next()) |k| gpa.free(k.*); + crcs.deinit(gpa); + } if (single_file) { - if (!try addPathCell(io, dir, std.fs.path.basename(path), &metas, &paths)) skipped += 1; + if (!try addPathCell(io, dir, std.fs.path.basename(path), &metas, &paths, null)) skipped += 1; } else if (dir.readFileAlloc(io, "CATALOG.031", gpa, .limited(MAX_CELL_BYTES))) |cbytes| { defer gpa.free(cbytes); var carena = std.heap.ArenaAllocator.init(gpa); defer carena.deinit(); if (s57.parseCatalog(carena.allocator(), cbytes)) |entries| { + // Keep every CRC the catalogue gives, cells and updates alike. + // A cell is verified below, as its bytes are already read; an + // update is verified when the chain reaches it. + for (entries) |e| { + const want = s57.catalogCrc(e.crcs) orelse continue; + const key = gpa.dupe(u8, e.path) catch continue; + crcs.put(gpa, key, want) catch gpa.free(key); + } for (entries) |e| { if (e.is_cell) { - if (!try addPathCell(io, dir, e.path, &metas, &paths)) skipped += 1; + if (!try addPathCell(io, dir, e.path, &metas, &paths, s57.catalogCrc(e.crcs))) skipped += 1; } } } @@ -2594,7 +2737,7 @@ pub const Chart = struct { while (try walker.next(io)) |entry| { if (entry.kind != .file) continue; if (!std.mem.endsWith(u8, entry.path, ".000")) continue; - if (!try addPathCell(io, dir, entry.path, &metas, &paths)) skipped += 1; + if (!try addPathCell(io, dir, entry.path, &metas, &paths, null)) skipped += 1; } } if (metas.items.len == 0) return error.OpenFailed; @@ -2606,7 +2749,7 @@ pub const Chart = struct { src.skipped_cells = skipped; const ctx = try gpa.create(PathCtx); errdefer gpa.destroy(ctx); - ctx.* = .{ .threaded = threaded, .io = io, .dir = dir, .paths = try paths.toOwnedSlice(gpa) }; + ctx.* = .{ .threaded = threaded, .io = io, .dir = dir, .paths = try paths.toOwnedSlice(gpa), .crcs = crcs }; src.backend.cells.reader_user = ctx; src.backend.cells.path_ctx = ctx; return src; diff --git a/src/s57/s57.zig b/src/s57/s57.zig index 31a30fb..0c4f6ca 100644 --- a/src/s57/s57.zig +++ b/src/s57/s57.zig @@ -1625,8 +1625,21 @@ pub const CatalogEntry = struct { impl: []const u8, // "BIN" (a cell) / "ASC" / "TXT" ("" when absent); allocator-owned bbox: ?[4]f64, // [west, south, east, north]; null for non-cell / no coverage is_cell: bool, // BIN .000 base cell + /// The CRC the catalogue gives for the file (S-57 Part 3 3.4, the CATD + /// CRCS subfield), as the hex string the field holds. "" when the producer + /// gave none, which the spec permits. + crcs: []const u8, }; +/// The CRC32 in a CATD CRCS subfield, or null when it has none or the +/// text is not eight hex digits. S-57 Part 3 7.4.1 table 7.11 types CRCS as +/// `A( )` holding hex, and every producer seen writes it big-endian-first. +pub fn catalogCrc(crcs: []const u8) ?u32 { + const t = std.mem.trim(u8, crcs, " "); + if (t.len != 8) return null; + return std.fmt.parseInt(u32, t, 16) catch null; +} + /// ASCII whitespace set matching Go's strings.TrimSpace over byte data: space, /// tab, LF, VT, FF, CR. The oracle TrimSpace-es every attribute value before /// strconv.ParseFloat (pkg/s57 parseFloat + the portrayal/bake float parses), so @@ -1699,7 +1712,8 @@ fn decodeCATD(a: Allocator, raw_in: []const u8) ?CatalogEntry { }; } const long_name = a.dupe(u8, parts[1]) catch return null; - return .{ .stem = stem, .path = norm, .long_name = long_name, .impl = impl, .bbox = bbox, .is_cell = is_cell }; + const crcs = a.dupe(u8, parts[7]) catch return null; + return .{ .stem = stem, .path = norm, .long_name = long_name, .impl = impl, .bbox = bbox, .is_cell = is_cell, .crcs = crcs }; } /// Parse an S-57 exchange-set catalogue (CATALOG.031): one CATD record per file, @@ -2882,6 +2896,13 @@ test "a catalogue entry naming a file outside the exchange set is dropped" { try std.testing.expectEqualStrings("US5MD12M", e.stem); try std.testing.expect(e.is_cell); + // The CRC the catalogue gives for the file (S-57 Part 3 3.4). + try std.testing.expectEqual(@as(?u32, 0x1A2B3C4D), catalogCrc("1A2B3C4D")); + try std.testing.expectEqual(@as(?u32, 0x1A2B3C4D), catalogCrc(" 1a2b3c4d ")); + try std.testing.expectEqual(@as(?u32, null), catalogCrc("")); // producers may omit it + try std.testing.expectEqual(@as(?u32, null), catalogCrc("1A2B")); + try std.testing.expectEqual(@as(?u32, null), catalogCrc("ZZZZZZZZ")); + try std.testing.expect(safeCatalogPath("ENC_ROOT/A/B.000")); try std.testing.expect(safeCatalogPath("..a/B.000")); try std.testing.expect(!safeCatalogPath("../B.000")); diff --git a/src/zipsrc.zig b/src/zipsrc.zig index 39b3949..54358cd 100644 --- a/src/zipsrc.zig +++ b/src/zipsrc.zig @@ -34,6 +34,8 @@ pub const Error = error{ BadLocalHeader, UnsupportedCompressionMethod, EntryTooLarge, + /// The entry's bytes do not match the CRC32 the archive holds for it. + ChecksumMismatch, }; pub const Entry = struct { @@ -191,6 +193,12 @@ pub const Archive = struct { var w: std.Io.Writer = .fixed(buf); try self.streamEntry(io, i, &w); if (w.end != buf.len) return error.EndOfStream; + // Every zip entry has a CRC32 of its uncompressed bytes. Raw deflate + // has no checksum of its own, so a flipped bit inside a literal run + // inflates to bytes that parse. On a chart that moves a boundary vertex + // and draws a chart nobody published. + const want = e.raw.crc32; + if (want != 0 and std.hash.Crc32.hash(buf) != want) return Error.ChecksumMismatch; return buf; } @@ -652,3 +660,38 @@ test "a file that is not a zip is refused at open" { try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = "0001 this is an S-57 cell, not an archive" }); try testing.expectError(error.ZipNoEndRecord, Archive.open(gpa, io, zpath)); } + +test "a corrupted entry body is refused" { + const gpa = testing.allocator; + const io = testIo(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir = try tmpPath(gpa, &tmp); + defer gpa.free(dir); + const zpath = try std.fs.path.join(gpa, &.{ dir, "t.zip" }); + defer gpa.free(zpath); + + // Stored, so a flipped byte in the file is a flipped byte in the entry. + // Deflate has no checksum of its own, so the entry CRC32 covers it. + const body = "US5MD12M cell bytes " ** 40; + try writeTestZip(gpa, io, zpath, &.{.{ .name = "ENC_ROOT/US5MD12M/US5MD12M.000", .data = body, .deflate = false }}); + + { + var arc = try Archive.open(gpa, io, zpath); + defer arc.deinit(); + const got = try arc.readAlloc(gpa, io, 0, 1 << 20); + defer gpa.free(got); + try testing.expectEqualStrings(body, got); + } + + // Flip one byte inside the stored body and read again. + const raw = try std.Io.Dir.cwd().readFileAlloc(io, zpath, gpa, .unlimited); + defer gpa.free(raw); + const at = std.mem.indexOf(u8, raw, "cell bytes").? + 2; + raw[at] ^= 0x20; + try std.Io.Dir.cwd().writeFile(io, .{ .sub_path = zpath, .data = raw }); + + var arc2 = try Archive.open(gpa, io, zpath); + defer arc2.deinit(); + try testing.expectError(Error.ChecksumMismatch, arc2.readAlloc(gpa, io, 0, 1 << 20)); +}